From 8d287d53d36e6f2ad857639d7e161d303769c817 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Mon, 19 Oct 2026 17:01:01 -0400 Subject: [PATCH 01/24] Keep the open book in BookService's cache beside the latest The open book, the one last outlined, is now kept alongside the latest book instead of being watched apart from the cache. Reading other books no longer evicts it, so it has one USJ subscription however it is reached, page fetches after a whole-book read reuse its tokenized text, and closing a project waits on one first delivery per book. Patch from the issue #439 discussion. Co-Authored-By: Claude Opus 5.5 --- src/__tests__/services/bookService.test.ts | 185 +++++++++++++-------- src/services/bookService.ts | 129 +++++++------- 2 files changed, 178 insertions(+), 136 deletions(-) diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index 3acdd691d..9d30cbbc1 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -539,12 +539,17 @@ describe('onOpenBookTextChanged', () => { const unsubscribers: Array<() => boolean> = []; /** A listener registered for the test, removed once it ends. */ - function listen() { - const listener = jest.fn(); + function listen(onCall?: () => void) { + const listener = jest.fn(onCall); unsubscribers.push(onOpenBookTextChanged(listener)); return listener; } + /** The subscriptions made to `bookId`, oldest first. */ + function subscriptionsTo(project: ReturnType, bookId: string) { + return project.subscriptions.filter((s) => s.bookId === bookId); + } + beforeEach(() => { segmentStandIns(); }); @@ -553,6 +558,69 @@ describe('onOpenBookTextChanged', () => { unsubscribers.splice(0).forEach((unsubscribe) => unsubscribe()); }); + it('subscribes once to a book that is outlined and read whole', async () => { + const project = serveProject((bookId) => usj(bookId)); + + await getBookOutline('src', 'GEN'); + await getTokenizedBook('src', 'GEN'); + await getSegments('src', 'GEN', 0, 1); + + expect(project.subscriptions).toHaveLength(1); + }); + + it('subscribes once to a book however often it is outlined', async () => { + const project = serveProject((bookId) => usj(bookId)); + + await getBookOutline('src', 'GEN'); + await getBookOutline('src', 'GEN'); + + expect(project.subscriptions).toHaveLength(1); + }); + + it('keeps the outlined book while other books are read whole', async () => { + const project = serveProject((bookId) => usj(bookId)); + await getBookOutline('src', 'GEN'); + await getTokenizedBook('src', 'EXO'); + await getTokenizedBook('src', 'LEV'); + await settle(); + jest.mocked(tokenizeBook).mockClear(); + + await getTokenizedBook('src', 'GEN'); + await getBookOutline('src', 'GEN'); + + expect(tokenizeBook).not.toHaveBeenCalled(); + expect(subscriptionsTo(project, 'GEN')).toHaveLength(1); + expect(subscriptionsTo(project, 'GEN')[0].unsubscribe).not.toHaveBeenCalled(); + }); + + it('serves pages of the outlined book after another book was read whole', async () => { + const project = serveProject((bookId) => usj(bookId)); + await getBookOutline('src', 'GEN'); + await getTokenizedBook('src', 'EXO'); + jest.mocked(tokenizeBook).mockClear(); + + const page = parsePage(await getSegments('src', 'GEN', 0, 2)); + + expect(page.segments.map((seg) => seg.id)).toEqual(['GEN 1:1', 'GEN 1:3']); + expect(tokenizeBook).not.toHaveBeenCalled(); + expect(subscriptionsTo(project, 'GEN')).toHaveLength(1); + }); + + it('lets go of a book read whole once another replaces it, but not the outlined book', async () => { + const project = serveProject((bookId) => usj(bookId)); + await getBookOutline('src', 'GEN'); + await getTokenizedBook('src', 'EXO'); + + await getTokenizedBook('src', 'LEV'); + await settle(); + + expect(project.subscriptions.map((s) => [s.bookId, s.unsubscribe.mock.calls.length])).toEqual([ + ['GEN', 0], + ['EXO', 1], + ['LEV', 0], + ]); + }); + it('reports an edit to the outlined book with its project and book', async () => { const project = serveProject((bookId) => usj(bookId)); const listener = listen(); @@ -574,35 +642,31 @@ describe('onOpenBookTextChanged', () => { listeners.forEach((listener) => expect(listener).toHaveBeenCalledTimes(1)); }); - it('reads the edited text when the book is next requested', async () => { - let version = 'v1'; - const project = serveProject((bookId) => usj(bookId, version)); + it('watches the book again once an edit to it is reported', async () => { + const project = serveProject((bookId) => usj(bookId)); await getBookOutline('src', 'GEN'); - version = 'v2'; + const [first] = project.subscriptions; - project.subscriptions[0].deliver(usj('GEN', version)); + first.deliver(usj('GEN', 'v2')); await settle(); - expect(parseBook(await getTokenizedBook('src', 'GEN')).textVersion).toBe('v2@en'); + expect(subscriptionsTo(project, 'GEN')).toHaveLength(2); + expect(first.unsubscribe).toHaveBeenCalled(); }); - it('reads the edited text when the held book heard of the edit first', async () => { + it('reads the edited text when the book is next requested', async () => { let version = 'v1'; const project = serveProject((bookId) => usj(bookId, version)); - const listener = listen(); await getBookOutline('src', 'GEN'); version = 'v2'; - project.subscriptions[1].deliver(usj('GEN', version)); project.subscriptions[0].deliver(usj('GEN', version)); await settle(); - expect(listener).toHaveBeenCalledTimes(1); - expect(listener).toHaveBeenCalledWith('src', 'GEN'); expect(parseBook(await getTokenizedBook('src', 'GEN')).textVersion).toBe('v2@en'); }); - it('still reports an edit after other books have been read in between', async () => { + it('reads the edited text after other books were read in between', async () => { let version = 'v1'; const project = serveProject((bookId) => usj(bookId, version)); const listener = listen(); @@ -614,104 +678,90 @@ describe('onOpenBookTextChanged', () => { project.subscriptions[0].deliver(usj('GEN', version)); await settle(); - expect(listener).toHaveBeenCalledTimes(1); expect(listener).toHaveBeenCalledWith('src', 'GEN'); expect(parseBook(await getTokenizedBook('src', 'GEN')).textVersion).toBe('v2@en'); + expect(parseOutline(await getBookOutline('src', 'GEN')).textVersion).toBe('v2@en'); }); - it('keeps the book held for another when the outlined book is edited', async () => { + it('reports later edits through the book watched again', async () => { const project = serveProject((bookId) => usj(bookId)); + const listener = listen(); await getBookOutline('src', 'GEN'); - await getTokenizedBook('src', 'EXO'); - const held = project.subscriptions.at(-1); - project.subscriptions[0].deliver(usj('GEN', 'v2')); await settle(); - expect(held?.unsubscribe).not.toHaveBeenCalled(); + subscriptionsTo(project, 'GEN')[1].deliver(usj('GEN', 'v3')); + + expect(listener).toHaveBeenCalledTimes(2); }); - it('stops a watch that fails to read its book', async () => { - const project = serveProject(); - const outline = getBookOutline('src', 'GEN'); - await settle(); - const [failed, held] = project.subscriptions; + it('keeps the book held for another when the outlined book is edited', async () => { + const project = serveProject((bookId) => usj(bookId)); + await getBookOutline('src', 'GEN'); + await getTokenizedBook('src', 'EXO'); + const [exo] = subscriptionsTo(project, 'EXO'); - failed.deliver(platformError('busy')); - held.deliver(usj('GEN')); - await outline; + project.subscriptions[0].deliver(usj('GEN', 'v2')); await settle(); - expect(failed.unsubscribe).toHaveBeenCalled(); + expect(exo.unsubscribe).not.toHaveBeenCalled(); }); - it('watches the book again on the next outline once its watch fails to read it', async () => { - const project = serveProject(); + it('stops watching an outlined book that fails to read, and watches it again on the next outline', async () => { + let delivery: Delivery = platformError('busy'); + const project = serveProject(() => delivery); const listener = listen(); - const outline = getBookOutline('src', 'GEN'); - await settle(); - const [failed, held] = project.subscriptions; - failed.deliver(platformError('busy')); - held.deliver(usj('GEN')); - await outline; + await expect(getBookOutline('src', 'GEN')).rejects.toThrow('busy'); await settle(); + delivery = usj('GEN'); await getBookOutline('src', 'GEN'); - await settle(); - const watch = project.subscriptions[2]; - watch.deliver(usj('GEN')); + const [failed, watch] = subscriptionsTo(project, 'GEN'); watch.deliver(usj('GEN', 'v2')); + expect(failed.unsubscribe).toHaveBeenCalled(); expect(listener).toHaveBeenCalledTimes(1); expect(listener).toHaveBeenCalledWith('src', 'GEN'); }); - it('keeps the newer watch when a replaced watch fails to read its book', async () => { + it('keeps the newer outlined book when the one it replaced fails to read', async () => { const project = serveProject(); getBookOutline('src', 'GEN').catch(() => undefined); await settle(); const replaced = project.subscriptions[0]; - getBookOutline('src', 'EXO').catch(() => undefined); + const outline = getBookOutline('src', 'EXO'); await settle(); - project.subscriptions.filter((s) => s.bookId === 'EXO').forEach((s) => s.deliver(usj('EXO'))); - const subscriptionCount = project.subscriptions.length; + subscriptionsTo(project, 'EXO')[0].deliver(usj('EXO')); + await outline; replaced.deliver(platformError('busy')); await settle(); await getBookOutline('src', 'EXO'); - expect(project.subscriptions).toHaveLength(subscriptionCount); + expect(subscriptionsTo(project, 'EXO')).toHaveLength(1); }); - it('reports nothing for a book only read whole', async () => { + it('reports nothing for an edit to a book only read whole', async () => { const project = serveProject((bookId) => usj(bookId)); const listener = listen(); await getTokenizedBook('src', 'GEN'); await getSegments('src', 'GEN', 0, 1); - project.subscriptions.forEach((subscription) => subscription.deliver(usj('GEN', 'v2'))); + project.subscriptions[0].deliver(usj('GEN', 'v2')); expect(listener).not.toHaveBeenCalled(); }); - it('watches an outlined book once however often it is outlined', async () => { + it('stops watching the previous outlined book once another is outlined', async () => { const project = serveProject((bookId) => usj(bookId)); - await getBookOutline('src', 'GEN'); - await getBookOutline('src', 'GEN'); - - expect(project.subscriptions).toHaveLength(2); - }); - - it('stops watching the previous book once another is outlined', async () => { - const project = serveProject((bookId) => usj(bookId)); - await getBookOutline('src', 'GEN'); - const [first] = project.subscriptions; + await getTokenizedBook('src', 'EXO'); + const [gen] = project.subscriptions; - await getBookOutline('src', 'EXO'); + await getBookOutline('src', 'LEV'); await settle(); - expect(first.unsubscribe).toHaveBeenCalled(); + expect(gen.unsubscribe).toHaveBeenCalled(); }); it('reports edits to the newly outlined book', async () => { @@ -719,38 +769,37 @@ describe('onOpenBookTextChanged', () => { const listener = listen(); await getBookOutline('src', 'GEN'); await getBookOutline('src', 'EXO'); - const exo = project.subscriptions.find((s) => s.bookId === 'EXO'); - exo?.deliver(usj('EXO', 'v2')); + subscriptionsTo(project, 'EXO')[0].deliver(usj('EXO', 'v2')); expect(listener).toHaveBeenCalledTimes(1); expect(listener).toHaveBeenCalledWith('src', 'EXO'); }); - it("stops watching the project's outlined book once it closes", async () => { + it("stops watching the project's books once it closes", async () => { const project = serveProject((bookId) => usj(bookId)); await getBookOutline('src', 'GEN'); + await getTokenizedBook('src', 'EXO'); await closeProject('src'); expect(project.subscriptions.every((s) => s.unsubscribe.mock.calls.length === 1)).toBe(true); }); - it('closes a project that has an outlined book but nothing held', async () => { + it('stops watching an outlined book that is also the latest once the project closes', async () => { const project = serveProject((bookId) => usj(bookId)); await getBookOutline('src', 'GEN'); - project.subscriptions[1].deliver(usj('GEN', 'v2')); - await settle(); await closeProject('src'); - expect(project.subscriptions.every((s) => s.unsubscribe.mock.calls.length === 1)).toBe(true); + expect(project.subscriptions[0].unsubscribe).toHaveBeenCalledTimes(1); }); it('stops watching every outlined book once all projects close', async () => { const project = serveProject((bookId) => usj(bookId)); await getBookOutline('src', 'GEN'); - await getBookOutline('other', 'EXO'); + await getTokenizedBook('src', 'EXO'); + await getBookOutline('other', 'LEV'); await closeAllProjects(); diff --git a/src/services/bookService.ts b/src/services/bookService.ts index 0a8f8f7a5..1e482ea92 100644 --- a/src/services/bookService.ts +++ b/src/services/bookService.ts @@ -30,16 +30,16 @@ interface HeldBook { } /** Each source project's most recently requested book. */ -const heldBooks = new Map(); - -/** Tells a request whether its project closed while it was under way. */ -const closeCounts = new Map(); +const latestBooks = new Map(); /** - * Each source project's open book, the one whose outline was last asked for, watched apart from - * {@link heldBooks} since reading other books replaces the held one. + * Each source project's open book, the one whose outline was last asked for. It stays kept while + * other books are read, and its edits are reported to {@link onOpenBookTextChanged}'s listeners. */ -const openBooks = new Map(); +const openBooks = new Map(); + +/** Tells a request whether its project closed while it was under way. */ +const closeCounts = new Map(); /** Receives the source project and book of each edit to an open book's text. */ type OpenBookTextListener = (sourceProjectId: string, bookId: string) => void; @@ -99,14 +99,30 @@ function watchBookText( }; } -/** Stops watching `watch`'s book, and forgets it unless another book has replaced it. */ +/** Stops watching `watch`'s book, and forgets it wherever the source project keeps it. */ function releaseBook(sourceProjectId: string, watch: BookTextWatch): Promise { - if (heldBooks.get(sourceProjectId)?.watch === watch) heldBooks.delete(sourceProjectId); + if (latestBooks.get(sourceProjectId)?.watch === watch) latestBooks.delete(sourceProjectId); + if (openBooks.get(sourceProjectId)?.watch === watch) openBooks.delete(sourceProjectId); return watch.stop(); } +/** Stops watching `held`'s book once the source project keeps it neither as latest nor as open. */ +function dropUnlessKept(sourceProjectId: string, held: HeldBook | undefined): void { + if (!held) return; + if (latestBooks.get(sourceProjectId) === held || openBooks.get(sourceProjectId) === held) return; + held.watch.stop(); +} + function holdBook(sourceProjectId: string, bookId: string, writingSystem: string): HeldBook { - const watch = watchBookText(sourceProjectId, bookId, () => releaseBook(sourceProjectId, watch)); + const watch = watchBookText(sourceProjectId, bookId, () => { + const isOpen = openBooks.get(sourceProjectId)?.watch === watch; + releaseBook(sourceProjectId, watch); + if (!isOpen) return; + // Read again at once, so the book stays watched and a refetch the listeners start reads the + // edit. + openBooks.set(sourceProjectId, holdBook(sourceProjectId, bookId, writingSystem)); + openBookTextListeners.forEach((listener) => listener(sourceProjectId, bookId)); + }); const book = perfTrack('getTokenizedBook:usj', watch.text).then( (usj) => usj && @@ -121,10 +137,15 @@ function holdBook(sourceProjectId: string, bookId: string, writingSystem: string } /** - * Finds the source project's book tokenized under the project's language tag. Each project's latest - * book is kept, and served again until its text changes or {@link closeProject} lets it go. + * Finds the source project's book tokenized under the project's language tag, and keeps it as the + * project's latest book, and as its open book too when `isOpen`. A kept book is served again until + * its text changes, another book takes its place, or {@link closeProject} lets it go. */ -async function heldBook(sourceProjectId: string, bookId: string): Promise { +async function heldBook( + sourceProjectId: string, + bookId: string, + isOpen = false, +): Promise { const closeCount = closeCounts.get(sourceProjectId); const writingSystem = await getWritingSystem(sourceProjectId); if (closeCounts.get(sourceProjectId) !== closeCount) { @@ -133,12 +154,17 @@ async function heldBook(sourceProjectId: string, bookId: string): Promise kept?.bookId === bookId && kept.writingSystem === writingSystem, + ) ?? holdBook(sourceProjectId, bookId, writingSystem); + latestBooks.set(sourceProjectId, held); + if (isOpen) openBooks.set(sourceProjectId, held); + dropUnlessKept(sourceProjectId, latest); + dropUnlessKept(sourceProjectId, open); + return held; } /** @@ -205,37 +231,6 @@ export function onOpenBookTextChanged(listener: OpenBookTextListener): () => boo return () => openBookTextListeners.delete(listener); } -/** - * Makes `bookId` the source project's open book, watching it in place of the previous open book. A - * no-op when it is already the open book. - */ -function watchOpenBook(sourceProjectId: string, bookId: string): void { - const open = openBooks.get(sourceProjectId); - if (open?.bookId === bookId) return; - open?.watch.stop(); - const watch = watchBookText(sourceProjectId, bookId, () => { - // Either watch may hear of the edit first, so the held text goes before any listener refetches. - const held = heldBooks.get(sourceProjectId); - if (held?.bookId === bookId) releaseBook(sourceProjectId, held.watch); - openBookTextListeners.forEach((listener) => listener(sourceProjectId, bookId)); - }); - // A read failure reaches whoever requests the text. Forgetting the watch lets the next outline - // watch again; stopping it ends a subscription an error delivery leaves open. - watch.text.catch(() => { - if (openBooks.get(sourceProjectId)?.watch !== watch) return; - openBooks.delete(sourceProjectId); - watch.stop(); - }); - openBooks.set(sourceProjectId, { bookId, watch }); -} - -function closeOpenBook(sourceProjectId: string): Promise { - const open = openBooks.get(sourceProjectId); - if (!open) return Promise.resolve(); - openBooks.delete(sourceProjectId); - return open.watch.stop(); -} - /** * Outlines a book of the source project: its segments, segmented by the boundaries * `segmentationJson` names, each without its text. The book becomes the project's open book, whose @@ -250,8 +245,7 @@ export async function getBookOutline( bookId: string, segmentationJson?: string, ): Promise { - watchOpenBook(sourceProjectId, bookId); - const held = await heldBook(sourceProjectId, bookId); + const held = await heldBook(sourceProjectId, bookId, true); const book = await held.book; if (!book) return undefined; const segments = segmentsOf(held, book, segmentationJson, 'getBookOutline'); @@ -276,8 +270,10 @@ export async function getSegments( end: number, segmentationJson?: string, ): Promise { - const kept = heldBooks.get(sourceProjectId); - const held = kept?.bookId === bookId ? kept : await heldBook(sourceProjectId, bookId); + const held = + [openBooks.get(sourceProjectId), latestBooks.get(sourceProjectId)].find( + (kept) => kept?.bookId === bookId, + ) ?? (await heldBook(sourceProjectId, bookId)); const book = await held.book; if (!book) return undefined; const segments = segmentsOf(held, book, segmentationJson, 'getSegments'); @@ -290,23 +286,20 @@ export async function getSegments( return perfTime('getSegments:serialize', () => JSON.stringify(page)); } -/** - * Lets go of the book kept for the source project, and of any a request under way would keep, and - * stops watching its open book. - */ +/** Lets go of the books kept for the source project. */ +async function releaseProject(sourceProjectId: string): Promise { + const kept = new Set([latestBooks.get(sourceProjectId), openBooks.get(sourceProjectId)]); + await Promise.all([...kept].map((held) => held && releaseBook(sourceProjectId, held.watch))); +} + +/** Lets go of the books kept for the source project, and of any a request under way would keep. */ export async function closeProject(sourceProjectId: string): Promise { closeCounts.set(sourceProjectId, (closeCounts.get(sourceProjectId) ?? 0) + 1); - const held = heldBooks.get(sourceProjectId); - await Promise.all([ - held && releaseBook(sourceProjectId, held.watch), - closeOpenBook(sourceProjectId), - ]); + await releaseProject(sourceProjectId); } -/** Lets go of every book kept, and stops watching every open book. */ +/** Lets go of every book kept. */ export async function closeAllProjects(): Promise { - await Promise.all([ - ...[...heldBooks].map(([sourceProjectId, { watch }]) => releaseBook(sourceProjectId, watch)), - ...[...openBooks.keys()].map(closeOpenBook), - ]); + const sourceProjectIds = new Set([...latestBooks.keys(), ...openBooks.keys()]); + await Promise.all([...sourceProjectIds].map(releaseProject)); } From 83b28414c7c68b100fb9f61649314651cdc24bca Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Mon, 19 Oct 2026 17:01:11 -0400 Subject: [PATCH 02/24] Pass the analyses to the bench's concordance step approvedAnalysisByToken gained an analyses-by-id parameter in #461, and perf:bench, outside the typecheck, still called it without one, so the run threw at concordance.build. It now builds the map as the concordance panel does. Co-Authored-By: Claude Opus 5.5 --- perf/bench/cli.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/perf/bench/cli.ts b/perf/bench/cli.ts index 6768b089d..76fbbb0ff 100644 --- a/perf/bench/cli.ts +++ b/perf/bench/cli.ts @@ -143,7 +143,10 @@ function benchTier(tier: TierSpec, allBooks: () => Book[]): Record [ta.id, ta])), + ), viewBookCode, ); }), From b0976fe3a66290340505ffa160b91ece54b244ea Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Mon, 19 Oct 2026 17:01:21 -0400 Subject: [PATCH 03/24] Read the whole text in the host with a form-in-text index (#439) BookService reads every book of a source project's text, a few at a time, and files each book's word forms in a FormIndex. The read starts with the first getTextReadStatus for the project and lasts until the project closes: - getTextReadStatus (also a command) reports the read's state, the books read and in the text, and the books whose latest read failed. - isFormInText answers whether the text holds a form, and answers nothing until every book is read or while any book has failed, so "not in the text" marking stays off. - The read watches each book it has read; an edit lets go of the cached copies, reads the book again, updates its forms, and is then reported. onOpenBookTextChanged becomes onBookTextChanged, reporting edits to the open book and to every book the read keeps, once each. A watch kept only to be stopped no longer holds its first delivery, so the read keeps no USJ once each book is indexed. perf:bench checks the index against the WebView's read: on every #407 tier both find the same catalog forms not in the text. perf:app gains a whole-text read scenario timing the host's read and its heap, for the 2e checkpoint. Co-Authored-By: Claude Opus 5.5 --- e2e-tests/tests/perf/app-perf.spec.ts | 46 ++ perf/bench/cli.ts | 26 ++ perf/summary/cli.ts | 17 + src/__tests__/main.test.ts | 23 +- src/__tests__/services/bookService.test.ts | 463 ++++++++++++++++++++- src/__tests__/utils/form-index.test.ts | 70 ++++ src/hooks/useSourceTextReader.ts | 16 +- src/main.ts | 39 +- src/services/bookService.ts | 220 +++++++++- src/types/interlinearizer.d.ts | 8 + src/types/text-read-status.ts | 14 + src/utils/books-present.ts | 15 + src/utils/form-index.ts | 39 ++ 13 files changed, 941 insertions(+), 55 deletions(-) create mode 100644 src/__tests__/utils/form-index.test.ts create mode 100644 src/types/text-read-status.ts create mode 100644 src/utils/books-present.ts create mode 100644 src/utils/form-index.ts diff --git a/e2e-tests/tests/perf/app-perf.spec.ts b/e2e-tests/tests/perf/app-perf.spec.ts index 38fbbb102..d3579aab0 100644 --- a/e2e-tests/tests/perf/app-perf.spec.ts +++ b/e2e-tests/tests/perf/app-perf.spec.ts @@ -81,6 +81,19 @@ async function waitForAutosave(frame: Frame): Promise { return measures; } +/** Starts the host's read of the source's whole text, and waits until it has read every book. */ +async function readWholeText(page: Page, projectId: string): Promise { + await expect + .poll( + async () => { + const json = await sendCommand(page, 'interlinearizer.getTextReadStatus', projectId); + return typeof json === 'string' ? JSON.parse(json).state : undefined; + }, + { timeout: 300_000, intervals: [250] }, + ) + .toBe('read'); +} + /** Waits for any autosave an edit or Save started to land. */ async function settleAutosave(): Promise { if (unsavedFrame) await waitForAutosave(unsavedFrame); @@ -433,6 +446,39 @@ async function measureTier( ); } result.concordance = concordance.summary(); + + // Whole-text read in the host, in a fresh view each run since the host keeps the read until the + // view closes. + const textRead = new Samples(); + for (let run = 0; run < RUNS; run += 1) { + const isLast = run === RUNS - 1; + // eslint-disable-next-line no-await-in-loop + await closeSettledView(page); + // eslint-disable-next-line no-await-in-loop + const { frame: view } = await openView(page, projectId, tier.viewBook); + // eslint-disable-next-line no-await-in-loop + await waitForMeasure(view, 'reanchor', tier.viewBook); + // eslint-disable-next-line no-await-in-loop + if (isLast) await recordMemory('beforeTextRead'); + // eslint-disable-next-line no-await-in-loop + await takeHostEntries(page); + // eslint-disable-next-line no-await-in-loop + await readWholeText(page, projectId); + // eslint-disable-next-line no-await-in-loop + const entries = await takeHostEntries(page); + entries + .filter(({ name }) => name === 'readText') + .forEach(({ duration }) => textRead.addValue('text-read', duration)); + textRead.addValue( + 'text-read-tokenize', + entries + .filter(({ name }) => name === 'readText:tokenize') + .reduce((total, { duration }) => total + duration, 0), + ); + // eslint-disable-next-line no-await-in-loop + if (isLast) await recordMemory('afterTextRead'); + } + result.textRead = textRead.summary(); } test('measure the app', async ({ mainPage }) => { diff --git a/perf/bench/cli.ts b/perf/bench/cli.ts index 76fbbb0ff..a74d42e47 100644 --- a/perf/bench/cli.ts +++ b/perf/bench/cli.ts @@ -20,6 +20,7 @@ import { deriveConcordanceRows, indexBook, } from '../../src/utils/concordance'; +import { FormIndex, formsOfBook } from '../../src/utils/form-index'; import { collatorForTag } from '../../src/utils/language-tags'; import { reanchorAnalysisToBook } from '../../src/utils/reanchor-analysis'; import { MAX_UNDO_STEPS } from '../../src/utils/undo-history'; @@ -152,6 +153,26 @@ function benchTier(tier: TierSpec, allBooks: () => Book[]): Record { + const index = new FormIndex(); + books.forEach((book) => index.setBook(book.bookRef, formsOfBook(book))); + return index; + }; + time('text.formIndex', measure(opts, noSetup, indexForms)); + // The host's form index has to find the catalog forms the WebView's whole-text read finds missing. + const index = indexForms(); + const textForms = new Set(books.flatMap((book) => [...indexBook(book).occurrencesByForm.keys()])); + const catalogForms = selectCatalogRows( + { analysis, analysisLanguage: language }, + viewBookCode, + ).map((row) => row.form); + const byWebViewRead = catalogForms.filter((form) => !textForms.has(form)); + const byHostIndex = catalogForms.filter((form) => !index.has(form)); + const disagreeing = catalogForms.filter((form) => textForms.has(form) !== index.has(form)).length; + console.log( + ` forms not in the text: ${byWebViewRead.length} by the WebView's read, ${byHostIndex.length} by the host's index, ${disagreeing} disagreeing`, + ); + // The undo history holds a snapshot of the analysis before each of its steps. const undoBytes = retainedBytes(() => { let state: AnalysisState = { analysis, analysisLanguage: language }; @@ -169,6 +190,11 @@ function benchTier(tier: TierSpec, allBooks: () => Book[]): Record = { "Committing one gloss to the store and rerunning the view's selectors, then serializing the draft to save it.", 'bench.timings.concordance': 'Indexing every book of the source and deriving the concordance rows.', + 'bench.timings.text': + "Filing every book's word forms in the host's form index, the books already tokenized.", + 'bench.formsNotInText': + "How many of the catalog's forms the text lacks, by the WebView's whole-text read and by the host's form index, and how many forms the two disagree on.", + 'app.textRead': + 'With a view open, the host reads every book of the source and files its word forms, from the first `getTextReadStatus` until it reports every book read. Tokenizing is summed over the books, which are read a few at a time.', 'bench.memory': 'Heap held by the parsed draft, and by an undo history filled to its limit.', }; @@ -77,10 +83,13 @@ const SECTION_LABELS: Record = { picker: 'List saved projects in the picker', save: 'Save the analysis to its project', concordance: 'Open the concordance', + textRead: 'Read the whole text in the host', + formsNotInText: 'Catalog forms not in the text', 'timings.load': 'Load', 'timings.storage': 'Storage', 'timings.edit': 'Edit one gloss', 'timings.concordance': 'Concordance', + 'timings.text': 'Whole-text read', }; /** Names for measured values, by their key; an unlisted value shows its key. */ @@ -110,6 +119,12 @@ const METRIC_LABELS: Record = { 'save-bytes': 'Analysis sent', 'concordance-read': 'Read every book', 'concordance-entries': 'Build entries', + 'text-read': 'Read every book', + 'text-read-tokenize': 'Tokenize every book, summed', + formIndex: "Index every book's forms", + byWebViewRead: "By the WebView's read", + byHostIndex: "By the host's index", + disagreeing: 'Disagreeing', parseDraft: 'Parse draft', storeMount: 'Mount analysis store', readShards: 'Parse and merge shards', @@ -139,6 +154,8 @@ const MEMORY_POINTS: Record = { withFullUndoHistory: 'full undo history', afterPicker: 'after picker', afterSave: 'after save', + beforeTextRead: 'before the whole-text read', + afterTextRead: 'with the whole text read', }; const isBytes = (name: string) => /bytes/i.test(name); diff --git a/src/__tests__/main.test.ts b/src/__tests__/main.test.ts index 245906c19..6b386ceb1 100644 --- a/src/__tests__/main.test.ts +++ b/src/__tests__/main.test.ts @@ -273,10 +273,10 @@ describe('main', () => { return emitters; } - it('emits the project and book of an edit to the open book', async () => { + it('emits the project and book of an edit to a watched book', async () => { const emitters = emittersByEvent(); await activate(createTestActivationContext()); - const [listener] = jest.mocked(bookService.onOpenBookTextChanged).mock.calls[0]; + const [listener] = jest.mocked(bookService.onBookTextChanged).mock.calls[0]; listener('src-project', 'GEN'); @@ -290,7 +290,7 @@ describe('main', () => { it('gives each emitted change its own id', async () => { const emitters = emittersByEvent(); await activate(createTestActivationContext()); - const [listener] = jest.mocked(bookService.onOpenBookTextChanged).mock.calls[0]; + const [listener] = jest.mocked(bookService.onBookTextChanged).mock.calls[0]; listener('src-project', 'GEN'); listener('src-project', 'GEN'); @@ -302,7 +302,7 @@ describe('main', () => { it('disposes the emitter and stops listening on deactivation', async () => { const emitters = emittersByEvent(); const unsubscribe = jest.fn(() => true); - jest.mocked(bookService.onOpenBookTextChanged).mockReturnValue(unsubscribe); + jest.mocked(bookService.onBookTextChanged).mockReturnValue(unsubscribe); const context = createTestActivationContext(); await activate(context); @@ -374,6 +374,7 @@ describe('main', () => { 'interlinearizer.getTokenizedBook', 'interlinearizer.getBookOutline', 'interlinearizer.getSegments', + 'interlinearizer.getTextReadStatus', 'interlinearizer.saveDraft', 'interlinearizer.openSelectProjectModal', 'interlinearizer.openNewProjectModal', @@ -1487,6 +1488,20 @@ describe('main', () => { }); }); + describe('interlinearizer.getTextReadStatus command', () => { + it('returns the status the service reports for the project', async () => { + jest.mocked(bookService.getTextReadStatus).mockReturnValue('{"state":"reading"}'); + const handler = await activateAndGetHandler<(sourceProjectId: string) => Promise>( + 'interlinearizer.getTextReadStatus', + ); + + const result = await handler('src-project'); + + expect(bookService.getTextReadStatus).toHaveBeenCalledWith('src-project'); + expect(result).toBe('{"state":"reading"}'); + }); + }); + describe('interlinearizer.setPerfMarks command', () => { it("turns the host's timing entries on or off as asked", async () => { const handler = await activateAndGetHandler<(enabled: boolean) => Promise>( diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index 9d30cbbc1..16aadf105 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -10,11 +10,14 @@ import { closeProject, getBookOutline, getSegments, + getTextReadStatus, getTokenizedBook, getWritingSystem, - onOpenBookTextChanged, + isFormInText, + onBookTextChanged, } from '../../services/bookService'; import type { BookOutline, SegmentPage } from '../../types/segment-pages'; +import type { TextReadStatus } from '../../types/text-read-status'; import { buildBookOutline } from '../../utils/book-outline'; import * as perfMarks from '../../utils/perf-marks'; import { reanchorSegmentation } from '../../utils/segmentation'; @@ -41,6 +44,7 @@ type Delivery = UsjDocument | undefined | { message: string; platformErrorVersio interface Subscription { bookId: string; deliver: (usj: Delivery) => void; + isDelivered: boolean; unsubscribe: jest.Mock; } @@ -55,25 +59,43 @@ function platformError(message: string) { /** * Wires the providers to serve one source project. Each subscription delivers `text` for its book - * at once unless `text` is omitted, which leaves delivering to the test. + * at once unless `text` is omitted, which leaves delivering to the test. Listing the project's + * books throws `booksPresent` when it is an error. */ function serveProject(text?: (bookId: string) => Delivery) { - const project: { languageTag: unknown; subscriptions: Subscription[] } = { + const project: { + languageTag: unknown; + booksPresent: string | Error; + subscriptions: Subscription[]; + } = { languageTag: 'en', + booksPresent: '', subscriptions: [], }; const subscribeBookUSJ = jest.fn( async ({ book }: { book: string }, callback: (usj: Delivery) => void) => { const unsubscribe = jest.fn(async () => true); - project.subscriptions.push({ bookId: book, deliver: callback, unsubscribe }); - if (text) queueMicrotask(() => callback(text(book))); + const subscription: Subscription = { + bookId: book, + deliver: (delivery) => { + subscription.isDelivered = true; + callback(delivery); + }, + isDelivered: false, + unsubscribe, + }; + project.subscriptions.push(subscription); + if (text) queueMicrotask(() => subscription.deliver(text(book))); return unsubscribe; }, ); + const getSetting = async (key: string) => { + if (key !== 'platformScripture.booksPresent') return project.languageTag; + if (project.booksPresent instanceof Error) throw project.booksPresent; + return project.booksPresent; + }; mockPdpGet.mockImplementation(async (type: string) => - type === 'platform.base' - ? { getSetting: async () => project.languageTag } - : { subscribeBookUSJ }, + type === 'platform.base' ? { getSetting } : { subscribeBookUSJ }, ); return project; } @@ -100,6 +122,46 @@ function parsePage(json: string | undefined): SegmentPage { return JSON.parse(json); } +function readStatus(sourceProjectId = 'src'): TextReadStatus { + return JSON.parse(getTextReadStatus(sourceProjectId)); +} + +/** + * Delivers each book a read awaits, and those it goes on to read, so a project can close without + * waiting on a delivery. + */ +async function finishReading(project: ReturnType) { + const pending = project.subscriptions.filter((s) => !s.isDelivered); + if (pending.length === 0) return; + pending.forEach((s) => s.deliver(usj(s.bookId))); + await settle(); + await finishReading(project); +} + +/** The subscriptions made to `bookId`, oldest first. */ +function subscriptionsTo(project: ReturnType, bookId: string) { + return project.subscriptions.filter((s) => s.bookId === bookId); +} + +/** + * Has each book tokenize to the word `shared` and a word naming its book and text version, such as + * `GEN-v1`. + */ +function tokenizeToWords() { + jest.mocked(tokenizeBook).mockImplementation((raw) => ({ + id: raw.bookCode, + bookRef: raw.bookCode, + textVersion: `${raw.contentHash}@${raw.writingSystem}`, + duplicateVerseIds: [], + segments: [ + makeSegment(`${raw.bookCode} 1:1`, 'word', [ + makeWordToken(`${raw.bookCode} 1:1:0`, 'shared'), + makeWordToken(`${raw.bookCode} 1:1:7`, `${raw.bookCode}-${raw.contentHash}`), + ]), + ], + })); +} + /** A tokenized Genesis 1:1–3, standing in for whatever text the project holds. */ const VERSE_BOOK: Book = { id: 'GEN', @@ -535,21 +597,16 @@ describe('closeAllProjects', () => { }); }); -describe('onOpenBookTextChanged', () => { +describe('onBookTextChanged', () => { const unsubscribers: Array<() => boolean> = []; /** A listener registered for the test, removed once it ends. */ function listen(onCall?: () => void) { const listener = jest.fn(onCall); - unsubscribers.push(onOpenBookTextChanged(listener)); + unsubscribers.push(onBookTextChanged(listener)); return listener; } - /** The subscriptions made to `bookId`, oldest first. */ - function subscriptionsTo(project: ReturnType, bookId: string) { - return project.subscriptions.filter((s) => s.bookId === bookId); - } - beforeEach(() => { segmentStandIns(); }); @@ -806,10 +863,123 @@ describe('onOpenBookTextChanged', () => { expect(project.subscriptions.every((s) => s.unsubscribe.mock.calls.length === 1)).toBe(true); }); + it('reports an edit to a book the whole-text read has read', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '11'; + const listener = listen(); + readStatus(); + await settle(); + + subscriptionsTo(project, 'EXO')[0].deliver(usj('EXO', 'v2')); + await settle(); + + expect(listener).toHaveBeenCalledTimes(1); + expect(listener).toHaveBeenCalledWith('src', 'EXO'); + }); + + it("reports an edit once the whole-text read's answers include it", async () => { + tokenizeToWords(); + let version = 'v1'; + const project = serveProject((bookId) => usj(bookId, version)); + project.booksPresent = '1'; + readStatus(); + await settle(); + const answers: unknown[] = []; + listen(() => answers.push(isFormInText('src', 'gen-v2'))); + + version = 'v2'; + project.subscriptions[0].deliver(usj('GEN', version)); + await settle(); + + expect(answers).toEqual([true]); + }); + + it('reports an edit to the outlined book once when the whole-text read has read it too', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + const listener = listen(); + await getBookOutline('src', 'GEN'); + readStatus(); + await settle(); + + subscriptionsTo(project, 'GEN').forEach((s) => s.deliver(usj('GEN', 'v2'))); + await settle(); + + expect(listener).toHaveBeenCalledTimes(1); + }); + + it('reports an edit to the outlined book the whole-text read has yet to read', async () => { + const project = serveProject(); + project.booksPresent = '1'; + const listener = listen(); + const outline = getBookOutline('src', 'GEN'); + await settle(); + project.subscriptions[0].deliver(usj('GEN')); + await outline; + readStatus(); + await settle(); + + project.subscriptions[0].deliver(usj('GEN', 'v2')); + + expect(listener).toHaveBeenCalledWith('src', 'GEN'); + await finishReading(project); + }); + + it('serves the edited outlined book when the whole-text read hears of the edit first', async () => { + let version = 'v1'; + const project = serveProject((bookId) => usj(bookId, version)); + project.booksPresent = '1'; + await getBookOutline('src', 'GEN'); + readStatus(); + await settle(); + const [, read] = subscriptionsTo(project, 'GEN'); + const listener = listen(); + + version = 'v2'; + read.deliver(usj('GEN', version)); + await settle(); + + expect(listener).toHaveBeenCalledTimes(1); + expect(parseOutline(await getBookOutline('src', 'GEN')).textVersion).toBe('v2@en'); + }); + + it('reports nothing for an edit that lands while the project closes', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + readStatus(); + await settle(); + const listener = listen(); + + const closing = closeProject('src'); + project.subscriptions[0].deliver(usj('GEN', 'v2')); + await closing; + await settle(); + + expect(listener).not.toHaveBeenCalled(); + expect(project.subscriptions).toHaveLength(1); + }); + + it('reports each edit once, through the watch that heard it', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + readStatus(); + await settle(); + const listener = listen(); + const [replaced] = project.subscriptions; + replaced.deliver(usj('GEN', 'v2')); + await settle(); + + replaced.deliver(usj('GEN', 'v3')); + await settle(); + + expect(listener).toHaveBeenCalledTimes(1); + expect(replaced.unsubscribe).toHaveBeenCalled(); + }); + it('does not call a listener once it is removed', async () => { const project = serveProject((bookId) => usj(bookId)); const listener = jest.fn(); - const unsubscribe = onOpenBookTextChanged(listener); + const unsubscribe = onBookTextChanged(listener); await getBookOutline('src', 'GEN'); unsubscribe(); @@ -818,3 +988,264 @@ describe('onOpenBookTextChanged', () => { expect(listener).not.toHaveBeenCalled(); }); }); + +describe('getTextReadStatus', () => { + it('reads nothing until the books are known', () => { + serveProject((bookId) => usj(bookId)); + + expect(readStatus()).toEqual({ + state: 'listing', + booksRead: 0, + bookCount: 0, + failedBookIds: [], + }); + }); + + it("reads every book the project's text holds, under its language tag", async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '101'; + + readStatus(); + await settle(); + + expect(readStatus()).toEqual({ state: 'read', booksRead: 2, bookCount: 2, failedBookIds: [] }); + expect(project.subscriptions.map((s) => s.bookId)).toEqual(['GEN', 'LEV']); + expect(extractBookFromUsj).toHaveBeenCalledWith(usj('LEV'), 'en'); + }); + + it('reports the books read so far while others are still being read', async () => { + const project = serveProject(); + project.booksPresent = '111'; + readStatus(); + await settle(); + + project.subscriptions[0].deliver(usj('GEN')); + project.subscriptions[1].deliver(usj('EXO')); + await settle(); + + expect(readStatus()).toEqual({ + state: 'reading', + booksRead: 2, + bookCount: 3, + failedBookIds: [], + }); + await finishReading(project); + }); + + it('reads a few books at a time', async () => { + const project = serveProject(); + project.booksPresent = '111111'; + readStatus(); + await settle(); + const started = project.subscriptions.length; + + project.subscriptions[0].deliver(usj('GEN')); + await settle(); + + expect(started).toBeLessThan(6); + expect(project.subscriptions).toHaveLength(started + 1); + await finishReading(project); + }); + + it('reads the text once however often its status is asked for', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + + readStatus(); + readStatus(); + await settle(); + readStatus(); + + expect(project.subscriptions).toHaveLength(1); + }); + + it('reports a book that fails to read, and reads the rest', async () => { + const project = serveProject((bookId) => + bookId === 'EXO' ? platformError('busy') : usj(bookId), + ); + project.booksPresent = '111'; + + readStatus(); + await settle(); + + expect(readStatus()).toEqual({ + state: 'read', + booksRead: 3, + bookCount: 3, + failedBookIds: ['EXO'], + }); + }); + + it('counts a book the project has no text for as read', async () => { + const project = serveProject(() => undefined); + project.booksPresent = '1'; + + readStatus(); + await settle(); + + expect(readStatus()).toEqual({ state: 'read', booksRead: 1, bookCount: 1, failedBookIds: [] }); + }); + + it('reports that the books could not be listed', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = new Error('offline'); + + readStatus(); + await settle(); + + expect(readStatus().state).toBe('unlisted'); + expect(project.subscriptions).toHaveLength(0); + }); + + it('clears a failed book once an edit lets it read', async () => { + let delivery: Delivery = platformError('busy'); + const project = serveProject(() => delivery); + project.booksPresent = '1'; + readStatus(); + await settle(); + + delivery = usj('GEN', 'v2'); + project.subscriptions[0].deliver(delivery); + await settle(); + + expect(readStatus().failedBookIds).toEqual([]); + }); + + it('stops watching the books once the project closes', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '11'; + readStatus(); + await settle(); + + await closeProject('src'); + + expect(project.subscriptions.every((s) => s.unsubscribe.mock.calls.length === 1)).toBe(true); + }); + + it('reads no further books once the project closes', async () => { + const project = serveProject(); + project.booksPresent = '111111'; + readStatus(); + await settle(); + const started = project.subscriptions.length; + + const closing = closeProject('src'); + project.subscriptions.forEach((s) => s.deliver(usj(s.bookId))); + await closing; + await settle(); + + expect(project.subscriptions).toHaveLength(started); + }); + + it('reads no book once the project closes while its books are listed', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + readStatus(); + + await closeProject('src'); + await settle(); + + expect(project.subscriptions).toHaveLength(0); + }); + + it('reads the text again once the project has closed', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + readStatus(); + await settle(); + await closeProject('src'); + + readStatus(); + await settle(); + + expect(readStatus().state).toBe('read'); + expect(project.subscriptions).toHaveLength(2); + }); + + it('stops watching the books of every project once all close', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + readStatus('src'); + readStatus('other'); + await settle(); + + await closeAllProjects(); + + expect(project.subscriptions).toHaveLength(2); + expect(project.subscriptions.every((s) => s.unsubscribe.mock.calls.length === 1)).toBe(true); + }); +}); + +describe('isFormInText', () => { + beforeEach(() => { + tokenizeToWords(); + }); + + it('answers nothing for a project whose text has not been read', () => { + expect(isFormInText('src', 'shared')).toBeUndefined(); + }); + + it('answers nothing until every book has been read', async () => { + const project = serveProject(); + project.booksPresent = '11'; + readStatus(); + await settle(); + + project.subscriptions[0].deliver(usj('GEN')); + await settle(); + + expect(isFormInText('src', 'shared')).toBeUndefined(); + await finishReading(project); + }); + + it('finds the forms any book of the text holds', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '11'; + readStatus(); + await settle(); + + expect(['shared', 'gen-v1', 'exo-v1', 'lev-v1'].map((f) => isFormInText('src', f))).toEqual([ + true, + true, + true, + false, + ]); + }); + + it('answers nothing while a book has failed to read', async () => { + const project = serveProject((bookId) => + bookId === 'EXO' ? platformError('busy') : usj(bookId), + ); + project.booksPresent = '11'; + readStatus(); + await settle(); + + expect(isFormInText('src', 'gen-v1')).toBeUndefined(); + }); + + it("answers from a book's edited text", async () => { + let version = 'v1'; + const project = serveProject((bookId) => usj(bookId, version)); + project.booksPresent = '1'; + readStatus(); + await settle(); + + version = 'v2'; + project.subscriptions[0].deliver(usj('GEN', version)); + await settle(); + + expect(isFormInText('src', 'gen-v1')).toBe(false); + expect(isFormInText('src', 'gen-v2')).toBe(true); + }); + + it('answers nothing once a project closes', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + readStatus(); + await settle(); + + await closeProject('src'); + + expect(isFormInText('src', 'shared')).toBeUndefined(); + }); +}); diff --git a/src/__tests__/utils/form-index.test.ts b/src/__tests__/utils/form-index.test.ts new file mode 100644 index 000000000..733979c70 --- /dev/null +++ b/src/__tests__/utils/form-index.test.ts @@ -0,0 +1,70 @@ +/// + +import type { Book } from 'interlinearizer'; +import { indexBook } from '../../utils/concordance'; +import { FormIndex, formsOfBook } from '../../utils/form-index'; +import { makePunctToken, makeSegment, makeWordToken } from '../test-helpers'; + +const BOOK: Book = { + id: 'GEN', + bookRef: 'GEN', + textVersion: 'v1@en', + duplicateVerseIds: [], + segments: [ + makeSegment('GEN 1:1', 'In the beginning.', [ + makeWordToken('GEN 1:1:0', 'In'), + makeWordToken('GEN 1:1:3', 'the'), + makeWordToken('GEN 1:1:7', 'beginning'), + makePunctToken('GEN 1:1:16'), + ]), + makeSegment('GEN 1:2', 'The earth', [ + makeWordToken('GEN 1:2:0', 'The'), + makeWordToken('GEN 1:2:4', 'earth'), + ]), + ], +}; + +describe('formsOfBook', () => { + it("holds each of the book's word forms once, normalized", () => { + expect([...formsOfBook(BOOK)]).toEqual(['in', 'the', 'beginning', 'earth']); + }); + + it('holds the forms the concordance files occurrences under', () => { + expect(formsOfBook(BOOK)).toEqual(new Set(indexBook(BOOK).occurrencesByForm.keys())); + }); +}); + +describe('FormIndex', () => { + it('holds the forms of every book filed', () => { + const index = new FormIndex(); + + index.setBook('GEN', new Set(['in', 'the'])); + index.setBook('EXO', new Set(['now'])); + + expect(['in', 'the', 'now', 'earth'].map((form) => index.has(form))).toEqual([ + true, + true, + true, + false, + ]); + }); + + it("drops forms only a refiled book's earlier forms held", () => { + const index = new FormIndex(); + index.setBook('GEN', new Set(['in', 'the'])); + + index.setBook('GEN', new Set(['the', 'earth'])); + + expect(['in', 'the', 'earth'].map((form) => index.has(form))).toEqual([false, true, true]); + }); + + it('keeps a form another book still holds once a book is refiled without it', () => { + const index = new FormIndex(); + index.setBook('GEN', new Set(['the'])); + index.setBook('EXO', new Set(['the'])); + + index.setBook('GEN', new Set()); + + expect(index.has('the')).toBe(true); + }); +}); diff --git a/src/hooks/useSourceTextReader.ts b/src/hooks/useSourceTextReader.ts index 040a7d4a6..4e6da0888 100644 --- a/src/hooks/useSourceTextReader.ts +++ b/src/hooks/useSourceTextReader.ts @@ -1,7 +1,7 @@ import papi, { logger } from '@papi/frontend'; -import { Canon } from '@sillsdev/scripture'; import type { Book } from 'interlinearizer'; import { useCallback, useEffect, useMemo, useState } from 'react'; +import { presentBookIds } from '../utils/books-present'; import { indexBook, type BookConcordance } from '../utils/concordance'; import { perfMark, perfMeasure } from '../utils/perf-marks'; import useLatestRef from './useLatestRef'; @@ -55,20 +55,6 @@ export interface SourceText { request: () => void; } -/** - * Reads the books a `platformScripture.booksPresent` flag string marks present — one `'1'` per - * present book, indexed by canonical book number — in canonical order. Flags past the end of the - * canon mark nothing. - */ -function presentBookIds(booksPresent: string): string[] { - const bookCount = Math.min(booksPresent.length, Canon.allBookIds.length); - const ids: string[] = []; - for (let index = 0; index < bookCount; index += 1) { - if (booksPresent[index] === '1') ids.push(Canon.bookNumberToId(index + 1)); - } - return ids; -} - /** * How many books are read at once. Enough to keep the provider busy while earlier books are * indexed, few enough that the whole canon's USJ is never in memory at the same time. diff --git a/src/main.ts b/src/main.ts index 94c9f0695..5c182d81f 100644 --- a/src/main.ts +++ b/src/main.ts @@ -553,6 +553,16 @@ async function getSegments( } } +/** + * Reports how far reading the source project's whole text has got, starting the read when the + * project has none. + * + * @returns The `TextReadStatus` as JSON. + */ +async function getTextReadStatus(sourceProjectId: string): Promise { + return bookService.getTextReadStatus(sourceProjectId); +} + /** * Persists the draft working buffer for a source project. Called from the WebView after every edit * (auto-save) and whenever the draft is reset (New), opened from a project, or wiped. @@ -974,6 +984,30 @@ export async function activate(context: ExecutionActivationContext): Promise + const bookTextUnsubscriber = bookService.onBookTextChanged((sourceProjectId, bookId) => analysisChangedEmitter.emit({ sourceProjectId, changeId: crypto.randomUUID(), @@ -1366,6 +1400,7 @@ export async function activate(context: ExecutionActivationContext): Promise(); /** * Each source project's open book, the one whose outline was last asked for. It stays kept while - * other books are read, and its edits are reported to {@link onOpenBookTextChanged}'s listeners. + * other books are read, and its edits are reported to {@link onBookTextChanged}'s listeners. */ const openBooks = new Map(); /** Tells a request whether its project closed while it was under way. */ const closeCounts = new Map(); -/** Receives the source project and book of each edit to an open book's text. */ -type OpenBookTextListener = (sourceProjectId: string, bookId: string) => void; +/** + * How many books a whole-text read reads at once. Enough to keep the provider busy while earlier + * books are tokenized, few enough that the whole canon's USJ is never in memory at the same time. + */ +const READ_CONCURRENCY = 4; + +/** A source project's whole text, read book by book into a form index, each book watched for edits. */ +interface TextRead { + state: TextReadStatus['state']; + /** The text's books in canonical order; empty until they are known. */ + bookIds: string[]; + /** The tag the books are tokenized under; `''` until the books are known. */ + writingSystem: string; + /** Books read at least once, whether or not they failed. */ + readBookIds: Set; + /** Books whose latest read failed. */ + failedBookIds: Set; + forms: FormIndex; + /** Stops watching each book, by book id. */ + watches: Map Promise>; +} + +/** Each source project's whole-text read, from the first status asked for until the project closes. */ +const textReads = new Map(); -const openBookTextListeners = new Set(); +/** Receives the source project and book of each edit to a watched book's text. */ +type BookTextListener = (sourceProjectId: string, bookId: string) => void; + +const bookTextListeners = new Set(); + +function reportEdit(sourceProjectId: string, bookId: string): void { + bookTextListeners.forEach((listener) => listener(sourceProjectId, bookId)); +} /** * Resolves the writing system tag of the source project's text, `und` when the project declares @@ -84,12 +116,17 @@ function watchBookText( }), ); subscription.catch(rejectText); + // Awaited in place of `text` itself, so a watch kept only to be stopped holds no text. + const delivered = text.then( + () => undefined, + () => undefined, + ); return { text, stop: async () => { // Unsubscribing ahead of the first delivery would leave `text` unsettled. - await text.catch(() => undefined); + await delivered; const unsubscribe = await subscription.catch(() => undefined); await unsubscribe?.().catch( /* v8 ignore next -- defensive: core's unsubscriber never rejects */ @@ -121,7 +158,9 @@ function holdBook(sourceProjectId: string, bookId: string, writingSystem: string // Read again at once, so the book stays watched and a refetch the listeners start reads the // edit. openBooks.set(sourceProjectId, holdBook(sourceProjectId, bookId, writingSystem)); - openBookTextListeners.forEach((listener) => listener(sourceProjectId, bookId)); + // A whole-text read that has the book reports the edit once it has read it again. + if (!textReads.get(sourceProjectId)?.readBookIds.has(bookId)) + reportEdit(sourceProjectId, bookId); }); const book = perfTrack('getTokenizedBook:usj', watch.text).then( (usj) => @@ -222,19 +261,20 @@ function segmentsOf( } /** - * Calls `listener` whenever the Scripture text of a source project's open book changes, until the - * returned function is called. A project's open book is the one whose outline was last asked for, - * and a book requested once the listener is called reads the edit. + * Calls `listener` whenever the Scripture text of a book the service watches changes, until the + * returned function is called. A source project's watched books are its open book, the one whose + * outline was last asked for, and every book its whole-text read has read. A book requested once + * the listener is called reads the edit, and the whole-text read's answers include it. */ -export function onOpenBookTextChanged(listener: OpenBookTextListener): () => boolean { - openBookTextListeners.add(listener); - return () => openBookTextListeners.delete(listener); +export function onBookTextChanged(listener: BookTextListener): () => boolean { + bookTextListeners.add(listener); + return () => bookTextListeners.delete(listener); } /** * Outlines a book of the source project: its segments, segmented by the boundaries * `segmentationJson` names, each without its text. The book becomes the project's open book, whose - * edits {@link onOpenBookTextChanged} reports. + * edits {@link onBookTextChanged} reports. * * @returns The `BookOutline` as JSON, or `undefined` when the project has no text for the book. * @throws If the book cannot be read or tokenized, or `segmentationJson` is not a JSON @@ -286,20 +326,164 @@ export async function getSegments( return perfTime('getSegments:serialize', () => JSON.stringify(page)); } -/** Lets go of the books kept for the source project. */ +/** Lets go of the source project's kept copies of `bookId`, holding its open book again at once. */ +function rereadHeldBook(sourceProjectId: string, bookId: string): void { + const open = openBooks.get(sourceProjectId); + new Set([latestBooks.get(sourceProjectId), open]).forEach((held) => { + if (held?.bookId === bookId) releaseBook(sourceProjectId, held.watch); + }); + if (open?.bookId === bookId) + openBooks.set(sourceProjectId, holdBook(sourceProjectId, bookId, open.writingSystem)); +} + +/** + * Reads `bookId` of `read`'s text into its form index, and watches the book until the read ends. An + * edit to the book lets go of its kept copies, reads it again, and is then reported. Resolves once + * the book's forms are filed or its read has failed. + */ +async function readTextBook( + sourceProjectId: string, + read: TextRead, + bookId: string, +): Promise { + read.watches.get(bookId)?.(); + const { text, stop } = watchBookText(sourceProjectId, bookId, async () => { + if (textReads.get(sourceProjectId) !== read || read.watches.get(bookId) !== stop) return; + rereadHeldBook(sourceProjectId, bookId); + await readTextBook(sourceProjectId, read, bookId); + reportEdit(sourceProjectId, bookId); + }); + read.watches.set(bookId, stop); + const forms = await perfTrack('readText:usj', text) + .then((usj) => + usj + ? perfTime('readText:tokenize', () => + formsOfBook(tokenizeBook(extractBookFromUsj(usj, read.writingSystem))), + ) + : new Set(), + ) + .catch((e) => { + logger.warn(`Could not read ${bookId} of project ${sourceProjectId}: ${e}`); + return undefined; + }); + if (textReads.get(sourceProjectId) !== read) return; + if (forms) { + read.forms.setBook(bookId, forms); + read.failedBookIds.delete(bookId); + } else { + read.failedBookIds.add(bookId); + } + read.readBookIds.add(bookId); +} + +/** Lists the source project's books in canonical order, with the tag their text is written in. */ +async function listBooks( + sourceProjectId: string, +): Promise<{ bookIds: string[]; writingSystem: string }> { + const basePdp = await papi.projectDataProviders.get('platform.base', sourceProjectId); + const [booksPresent, writingSystem] = await Promise.all([ + basePdp.getSetting('platformScripture.booksPresent'), + getWritingSystem(sourceProjectId), + ]); + return { bookIds: presentBookIds(booksPresent), writingSystem }; +} + +/** Lists the books of `read`'s text, then reads each of them. Resolves once every book is read. */ +async function readText(sourceProjectId: string, read: TextRead): Promise { + const isCurrent = () => textReads.get(sourceProjectId) === read; + const listed = await listBooks(sourceProjectId).catch((e) => { + logger.warn(`Could not list the books of project ${sourceProjectId}: ${e}`); + return undefined; + }); + if (!isCurrent()) return; + if (!listed) { + read.state = 'unlisted'; + return; + } + read.bookIds = listed.bookIds; + read.writingSystem = listed.writingSystem; + read.state = 'reading'; + const unread = [...read.bookIds]; + const readNext = async (): Promise => { + const bookId = unread.shift(); + if (bookId === undefined || !isCurrent()) return; + await readTextBook(sourceProjectId, read, bookId); + await readNext(); + }; + await Promise.all(Array.from({ length: READ_CONCURRENCY }, readNext)); + if (isCurrent()) read.state = 'read'; +} + +/** + * Reports how far reading the source project's whole text has got, starting the read when the + * project has none. The read keeps a form index that {@link isFormInText} answers from, and watches + * every book it has read for edits, which {@link onBookTextChanged} reports, until the project + * closes. + * + * @returns The `TextReadStatus` as JSON. + */ +export function getTextReadStatus(sourceProjectId: string): string { + let read = textReads.get(sourceProjectId); + if (!read) { + read = { + state: 'listing', + bookIds: [], + writingSystem: '', + readBookIds: new Set(), + failedBookIds: new Set(), + forms: new FormIndex(), + watches: new Map(), + }; + textReads.set(sourceProjectId, read); + perfTrack('readText', readText(sourceProjectId, read)); + } + const { state, bookIds, readBookIds, failedBookIds } = read; + const status: TextReadStatus = { + state, + booksRead: readBookIds.size, + bookCount: bookIds.length, + failedBookIds: bookIds.filter((bookId) => failedBookIds.has(bookId)), + }; + return JSON.stringify(status); +} + +/** + * Whether the source project's whole text holds `form`, normalized as analyses match forms. + * + * @returns `undefined` until every book has been read, and while any book's latest read has failed. + */ +export function isFormInText(sourceProjectId: string, form: string): boolean | undefined { + const read = textReads.get(sourceProjectId); + if (read?.state !== 'read' || read.failedBookIds.size > 0) return undefined; + return read.forms.has(form); +} + +/** Lets go of the books kept for the source project, and ends its whole-text read. */ async function releaseProject(sourceProjectId: string): Promise { const kept = new Set([latestBooks.get(sourceProjectId), openBooks.get(sourceProjectId)]); - await Promise.all([...kept].map((held) => held && releaseBook(sourceProjectId, held.watch))); + const read = textReads.get(sourceProjectId); + textReads.delete(sourceProjectId); + await Promise.all([ + ...[...kept].map((held) => held && releaseBook(sourceProjectId, held.watch)), + ...[...(read?.watches.values() ?? [])].map((stop) => stop()), + ]); } -/** Lets go of the books kept for the source project, and of any a request under way would keep. */ +/** + * Lets go of the books kept for the source project, and of any a request under way would keep, and + * ends its whole-text read. + */ export async function closeProject(sourceProjectId: string): Promise { closeCounts.set(sourceProjectId, (closeCounts.get(sourceProjectId) ?? 0) + 1); await releaseProject(sourceProjectId); } -/** Lets go of every book kept. */ +/** Lets go of every book kept, and ends every whole-text read. */ export async function closeAllProjects(): Promise { - const sourceProjectIds = new Set([...latestBooks.keys(), ...openBooks.keys()]); + const sourceProjectIds = new Set([ + ...latestBooks.keys(), + ...openBooks.keys(), + ...textReads.keys(), + ]); await Promise.all([...sourceProjectIds].map(releaseProject)); } diff --git a/src/types/interlinearizer.d.ts b/src/types/interlinearizer.d.ts index eea1c47af..924aa6418 100644 --- a/src/types/interlinearizer.d.ts +++ b/src/types/interlinearizer.d.ts @@ -278,6 +278,14 @@ declare module 'papi-shared-types' { segmentationJson?: string, ) => Promise; + /** + * Reports how far reading the source project's whole text has got, starting the read when the + * project has none. + * + * @returns The `TextReadStatus` as a JSON string. + */ + 'interlinearizer.getTextReadStatus': (sourceProjectId: string) => Promise; + /** * Persists the draft working buffer for the given source project. Called from the WebView after * every edit (auto-save) and whenever the draft is reset (New), opened from a project, or diff --git a/src/types/text-read-status.ts b/src/types/text-read-status.ts new file mode 100644 index 000000000..3ac463919 --- /dev/null +++ b/src/types/text-read-status.ts @@ -0,0 +1,14 @@ +/** How far reading a source project's whole text has got. */ +export type TextReadStatus = Readonly<{ + /** + * `listing` until the text's books are known, `unlisted` when they could not be, `reading` until + * every book has been read once, then `read`. + */ + state: 'listing' | 'unlisted' | 'reading' | 'read'; + /** How many of the text's books have been read at least once, those that failed included. */ + booksRead: number; + /** How many books the text holds; `0` until they are known. */ + bookCount: number; + /** The books whose latest read failed, in canonical order. */ + failedBookIds: readonly string[]; +}>; diff --git a/src/utils/books-present.ts b/src/utils/books-present.ts new file mode 100644 index 000000000..e2c712ef1 --- /dev/null +++ b/src/utils/books-present.ts @@ -0,0 +1,15 @@ +import { Canon } from '@sillsdev/scripture'; + +/** + * Reads the books a `platformScripture.booksPresent` flag string marks present — one `'1'` per + * present book, indexed by canonical book number — in canonical order. Flags past the end of the + * canon mark nothing. + */ +export function presentBookIds(booksPresent: string): string[] { + const bookCount = Math.min(booksPresent.length, Canon.allBookIds.length); + const ids: string[] = []; + for (let index = 0; index < bookCount; index += 1) { + if (booksPresent[index] === '1') ids.push(Canon.bookNumberToId(index + 1)); + } + return ids; +} diff --git a/src/utils/form-index.ts b/src/utils/form-index.ts new file mode 100644 index 000000000..dff721675 --- /dev/null +++ b/src/utils/form-index.ts @@ -0,0 +1,39 @@ +import type { Book } from 'interlinearizer'; +import { isWordToken } from '../types/type-guards'; +import { normalizeSurfaceForm } from './analysis-identity'; + +/** Every word form a tokenized book holds, normalized as analyses match them. */ +export function formsOfBook(book: Book): Set { + const forms = new Set(); + book.segments.forEach((segment) => + segment.tokens.forEach((token) => { + if (isWordToken(token)) forms.add(normalizeSurfaceForm(token.surfaceText)); + }), + ); + return forms; +} + +/** The word forms a text holds, filed book by book so that one book's can be replaced alone. */ +export class FormIndex { + private readonly formsByBook = new Map>(); + + /** How many filed books hold each form. */ + private readonly bookCounts = new Map(); + + /** Files `forms` as the forms `bookId` holds, in place of any filed for it before. */ + setBook(bookId: string, forms: ReadonlySet): void { + this.formsByBook.get(bookId)?.forEach((form) => { + /* v8 ignore next -- defensive: a filed form always has a count */ + const count = (this.bookCounts.get(form) ?? 1) - 1; + if (count === 0) this.bookCounts.delete(form); + else this.bookCounts.set(form, count); + }); + this.formsByBook.set(bookId, forms); + forms.forEach((form) => this.bookCounts.set(form, (this.bookCounts.get(form) ?? 0) + 1)); + } + + /** Whether a filed book holds `form`, normalized as {@link formsOfBook} normalizes forms. */ + has(form: string): boolean { + return this.bookCounts.has(form); + } +} From 519345fb45aacb1054fd7f6d0c0d592268c7e9dc Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Mon, 19 Oct 2026 17:11:43 -0400 Subject: [PATCH 04/24] Correct what the whole-text read holds, and match forms as analyses do Each book subscription keeps its last delivery in the platform for as long as it is open, so the read holds every book's USJ once it has read them, whatever the watch keeps. The concurrency comment and the watch's stop now say only what is true, and the perf summary notes that the host heap with the whole text read includes that USJ. isFormInText normalizes the form it is asked about, and the app perf scenario fails a read in which any book failed, rather than timing it. Co-Authored-By: Claude Opus 5.5 --- e2e-tests/tests/perf/app-perf.spec.ts | 25 +++++++++++++++------- perf/summary/cli.ts | 2 +- src/__tests__/services/bookService.test.ts | 9 ++++++++ src/services/bookService.ts | 16 ++++++-------- 4 files changed, 33 insertions(+), 19 deletions(-) diff --git a/e2e-tests/tests/perf/app-perf.spec.ts b/e2e-tests/tests/perf/app-perf.spec.ts index d3579aab0..4ca4a016d 100644 --- a/e2e-tests/tests/perf/app-perf.spec.ts +++ b/e2e-tests/tests/perf/app-perf.spec.ts @@ -5,6 +5,7 @@ import { readDraftJson, readManifest } from '../../../perf/dataset/store'; import { TIERS, type TierSpec } from '../../../perf/dataset/tiers'; import { provenance, writeResults } from '../../../perf/results'; import { addSavedProject, readDraft, writeDraft } from '../../../perf/storage'; +import type { TextReadStatus } from '../../../src/types/text-read-status'; import { readCaptureMeta } from '../../../perf/usj-cache'; import { expect, test } from '../../fixtures/cdp.fixture'; import { waitForAppAndInterlinearizerReady } from '../../fixtures/helpers'; @@ -81,17 +82,25 @@ async function waitForAutosave(frame: Frame): Promise { return measures; } -/** Starts the host's read of the source's whole text, and waits until it has read every book. */ +/** Reads the host's whole-text read status for the source, starting the read when it has none. */ +async function textReadStatus(page: Page, projectId: string): Promise { + const json = await sendCommand(page, 'interlinearizer.getTextReadStatus', projectId); + return typeof json === 'string' ? JSON.parse(json) : undefined; +} + +/** + * Starts the host's read of the source's whole text, and waits until it has read every book. + * + * @throws If any book failed to read, since the read's timing would then leave it out. + */ async function readWholeText(page: Page, projectId: string): Promise { await expect - .poll( - async () => { - const json = await sendCommand(page, 'interlinearizer.getTextReadStatus', projectId); - return typeof json === 'string' ? JSON.parse(json).state : undefined; - }, - { timeout: 300_000, intervals: [250] }, - ) + .poll(async () => (await textReadStatus(page, projectId))?.state, { + timeout: 300_000, + intervals: [250], + }) .toBe('read'); + expect((await textReadStatus(page, projectId))?.failedBookIds).toEqual([]); } /** Waits for any autosave an edit or Save started to land. */ diff --git a/perf/summary/cli.ts b/perf/summary/cli.ts index 36ebb0194..8be6dbfde 100644 --- a/perf/summary/cli.ts +++ b/perf/summary/cli.ts @@ -65,7 +65,7 @@ const SECTION_DESCRIPTIONS: Record = { 'bench.formsNotInText': "How many of the catalog's forms the text lacks, by the WebView's whole-text read and by the host's form index, and how many forms the two disagree on.", 'app.textRead': - 'With a view open, the host reads every book of the source and files its word forms, from the first `getTextReadStatus` until it reports every book read. Tokenizing is summed over the books, which are read a few at a time.', + "With a view open, the host reads every book of the source and files its word forms, from the first `getTextReadStatus` until it reports every book read. Tokenizing is summed over the books, which are read a few at a time. The host heap with the whole text read includes each book's USJ, which the platform keeps for every book subscription.", 'bench.memory': 'Heap held by the parsed draft, and by an undo history filled to its limit.', }; diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index 16aadf105..d4918f973 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -1212,6 +1212,15 @@ describe('isFormInText', () => { ]); }); + it('matches a form however it is cased', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + readStatus(); + await settle(); + + expect(isFormInText('src', 'Shared')).toBe(true); + }); + it('answers nothing while a book has failed to read', async () => { const project = serveProject((bookId) => bookId === 'EXO' ? platformError('busy') : usj(bookId), diff --git a/src/services/bookService.ts b/src/services/bookService.ts index 5a489360b..e62732b66 100644 --- a/src/services/bookService.ts +++ b/src/services/bookService.ts @@ -9,6 +9,7 @@ import type { SegmentPage } from '../types/segment-pages'; import type { TextReadStatus } from '../types/text-read-status'; import { buildBookOutline } from '../utils/book-outline'; import { presentBookIds } from '../utils/books-present'; +import { normalizeSurfaceForm } from '../utils/analysis-identity'; import { FormIndex, formsOfBook } from '../utils/form-index'; import { perfTime, perfTrack } from '../utils/perf-marks'; import { reanchorSegmentation } from '../utils/segmentation'; @@ -46,7 +47,7 @@ const closeCounts = new Map(); /** * How many books a whole-text read reads at once. Enough to keep the provider busy while earlier - * books are tokenized, few enough that the whole canon's USJ is never in memory at the same time. + * books are tokenized, few enough to bound the fetches in flight. */ const READ_CONCURRENCY = 4; @@ -116,17 +117,12 @@ function watchBookText( }), ); subscription.catch(rejectText); - // Awaited in place of `text` itself, so a watch kept only to be stopped holds no text. - const delivered = text.then( - () => undefined, - () => undefined, - ); return { text, stop: async () => { // Unsubscribing ahead of the first delivery would leave `text` unsettled. - await delivered; + await text.catch(() => undefined); const unsubscribe = await subscription.catch(() => undefined); await unsubscribe?.().catch( /* v8 ignore next -- defensive: core's unsubscriber never rejects */ @@ -388,7 +384,7 @@ async function listBooks( return { bookIds: presentBookIds(booksPresent), writingSystem }; } -/** Lists the books of `read`'s text, then reads each of them. Resolves once every book is read. */ +/** Lists the books of `read`'s text, then reads each of them. Resolves once the read ends. */ async function readText(sourceProjectId: string, read: TextRead): Promise { const isCurrent = () => textReads.get(sourceProjectId) === read; const listed = await listBooks(sourceProjectId).catch((e) => { @@ -448,14 +444,14 @@ export function getTextReadStatus(sourceProjectId: string): string { } /** - * Whether the source project's whole text holds `form`, normalized as analyses match forms. + * Whether the source project's whole text holds `form`, matched as analyses match forms. * * @returns `undefined` until every book has been read, and while any book's latest read has failed. */ export function isFormInText(sourceProjectId: string, form: string): boolean | undefined { const read = textReads.get(sourceProjectId); if (read?.state !== 'read' || read.failedBookIds.size > 0) return undefined; - return read.forms.has(form); + return read.forms.has(normalizeSurfaceForm(form)); } /** Lets go of the books kept for the source project, and ends its whole-text read. */ From bc197076e69b7874f216b433b60d61c5a5c1f3ae Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Mon, 19 Oct 2026 17:52:40 -0400 Subject: [PATCH 05/24] Say onAnalysisChanged covers every book the whole-text read has read Co-Authored-By: Claude Opus 5.5 --- src/types/interlinearizer.d.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/types/interlinearizer.d.ts b/src/types/interlinearizer.d.ts index 924aa6418..4f5ab6880 100644 --- a/src/types/interlinearizer.d.ts +++ b/src/types/interlinearizer.d.ts @@ -11,8 +11,9 @@ declare module 'papi-shared-types' { */ 'interlinearizer.onUndoFromNotification': { notificationId: string | number }; /** - * Emitted when the text of a source project's open book (the one last outlined) changes, so - * views showing it refetch it. Edits to other books emit nothing. + * Emitted when the text of a book the extension watches changes, so views showing it refetch + * it: a source project's open book (the one last outlined), and every book its whole-text read + * has read. Edits to other books emit nothing. */ 'interlinearizer.onAnalysisChanged': { sourceProjectId: string; From b4a12d7d7a1c35a01f788400d7eb7bfb69a15d11 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Mon, 19 Oct 2026 16:46:18 -0600 Subject: [PATCH 06/24] Keep the latest outline's book open, and report no edit once a read has ended --- src/__tests__/services/bookService.test.ts | 35 ++++++++++++++++++++++ src/services/bookService.ts | 19 ++++++++---- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index d4918f973..eef744fd8 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -833,6 +833,23 @@ describe('onBookTextChanged', () => { expect(listener).toHaveBeenCalledWith('src', 'EXO'); }); + it('keeps the later outlined book open when an earlier outline reads its language tag last', async () => { + const project = serveProject((bookId) => usj(bookId)); + const listener = listen(); + const tag = Promise.withResolvers(); + project.languageTag = tag.promise; + const earlier = getBookOutline('src', 'GEN'); + await settle(); + project.languageTag = 'en'; + await getBookOutline('src', 'EXO'); + tag.resolve('en'); + await earlier; + + subscriptionsTo(project, 'EXO')[0].deliver(usj('EXO', 'v2')); + + expect(listener).toHaveBeenCalledWith('src', 'EXO'); + }); + it("stops watching the project's books once it closes", async () => { const project = serveProject((bookId) => usj(bookId)); await getBookOutline('src', 'GEN'); @@ -959,6 +976,24 @@ describe('onBookTextChanged', () => { expect(project.subscriptions).toHaveLength(1); }); + it('reports nothing for an edit read again only after the project closes', async () => { + const project = serveProject(); + project.booksPresent = '1'; + readStatus(); + await settle(); + await finishReading(project); + const listener = listen(); + project.subscriptions[0].deliver(usj('GEN', 'v2')); + await settle(); + + const closing = closeProject('src'); + project.subscriptions[1].deliver(usj('GEN', 'v2')); + await closing; + await settle(); + + expect(listener).not.toHaveBeenCalled(); + }); + it('reports each edit once, through the watch that heard it', async () => { const project = serveProject((bookId) => usj(bookId)); project.booksPresent = '1'; diff --git a/src/services/bookService.ts b/src/services/bookService.ts index e62732b66..89cbbf86e 100644 --- a/src/services/bookService.ts +++ b/src/services/bookService.ts @@ -45,6 +45,9 @@ const openBooks = new Map(); /** Tells a request whether its project closed while it was under way. */ const closeCounts = new Map(); +/** Tells an outline request whether a later one has started, which picks the open book instead. */ +const outlineCounts = new Map(); + /** * How many books a whole-text read reads at once. Enough to keep the provider busy while earlier * books are tokenized, few enough to bound the fetches in flight. @@ -173,8 +176,9 @@ function holdBook(sourceProjectId: string, bookId: string, writingSystem: string /** * Finds the source project's book tokenized under the project's language tag, and keeps it as the - * project's latest book, and as its open book too when `isOpen`. A kept book is served again until - * its text changes, another book takes its place, or {@link closeProject} lets it go. + * project's latest book, and as its open book too when `isOpen` and no later outline request has + * started. A kept book is served again until its text changes, another book takes its place, or + * {@link closeProject} lets it go. */ async function heldBook( sourceProjectId: string, @@ -182,6 +186,8 @@ async function heldBook( isOpen = false, ): Promise { const closeCount = closeCounts.get(sourceProjectId); + const outlineCount = (outlineCounts.get(sourceProjectId) ?? 0) + 1; + if (isOpen) outlineCounts.set(sourceProjectId, outlineCount); const writingSystem = await getWritingSystem(sourceProjectId); if (closeCounts.get(sourceProjectId) !== closeCount) { // The project closed while its tag was read, so the book is served but not kept. @@ -196,7 +202,8 @@ async function heldBook( (kept) => kept?.bookId === bookId && kept.writingSystem === writingSystem, ) ?? holdBook(sourceProjectId, bookId, writingSystem); latestBooks.set(sourceProjectId, held); - if (isOpen) openBooks.set(sourceProjectId, held); + if (isOpen && outlineCounts.get(sourceProjectId) === outlineCount) + openBooks.set(sourceProjectId, held); dropUnlessKept(sourceProjectId, latest); dropUnlessKept(sourceProjectId, open); return held; @@ -334,8 +341,8 @@ function rereadHeldBook(sourceProjectId: string, bookId: string): void { /** * Reads `bookId` of `read`'s text into its form index, and watches the book until the read ends. An - * edit to the book lets go of its kept copies, reads it again, and is then reported. Resolves once - * the book's forms are filed or its read has failed. + * edit to the book lets go of its kept copies, reads it again, and is then reported unless the read + * has ended. Resolves once the book's forms are filed or its read has failed. */ async function readTextBook( sourceProjectId: string, @@ -347,7 +354,7 @@ async function readTextBook( if (textReads.get(sourceProjectId) !== read || read.watches.get(bookId) !== stop) return; rereadHeldBook(sourceProjectId, bookId); await readTextBook(sourceProjectId, read, bookId); - reportEdit(sourceProjectId, bookId); + if (textReads.get(sourceProjectId) === read) reportEdit(sourceProjectId, bookId); }); read.watches.set(bookId, stop); const forms = await perfTrack('readText:usj', text) From decc5b0779bd2b842238668abbfc3334e4d16d9e Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 20 Oct 2026 08:53:13 -0600 Subject: [PATCH 07/24] Report an edit to the open book the whole-text read could not watch --- e2e-tests/tests/perf/app-perf.spec.ts | 6 +++++- src/__tests__/services/bookService.test.ts | 22 +++++++++++++++++++++- src/services/bookService.ts | 20 +++++++++++++++----- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/e2e-tests/tests/perf/app-perf.spec.ts b/e2e-tests/tests/perf/app-perf.spec.ts index 4ca4a016d..83421c8eb 100644 --- a/e2e-tests/tests/perf/app-perf.spec.ts +++ b/e2e-tests/tests/perf/app-perf.spec.ts @@ -82,7 +82,11 @@ async function waitForAutosave(frame: Frame): Promise { return measures; } -/** Reads the host's whole-text read status for the source, starting the read when it has none. */ +/** + * Reads the host's whole-text read status for the source, starting the read when it has none. + * + * @returns The status, or `undefined` when the command answers with something other than a string. + */ async function textReadStatus(page: Page, projectId: string): Promise { const json = await sendCommand(page, 'interlinearizer.getTextReadStatus', projectId); return typeof json === 'string' ? JSON.parse(json) : undefined; diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index eef744fd8..83dfaf56f 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -60,20 +60,24 @@ function platformError(message: string) { /** * Wires the providers to serve one source project. Each subscription delivers `text` for its book * at once unless `text` is omitted, which leaves delivering to the test. Listing the project's - * books throws `booksPresent` when it is an error. + * books throws `booksPresent` when it is an error, and subscribing throws `subscribeError` while it + * is set. */ function serveProject(text?: (bookId: string) => Delivery) { const project: { languageTag: unknown; booksPresent: string | Error; + subscribeError: Error | undefined; subscriptions: Subscription[]; } = { languageTag: 'en', booksPresent: '', + subscribeError: undefined, subscriptions: [], }; const subscribeBookUSJ = jest.fn( async ({ book }: { book: string }, callback: (usj: Delivery) => void) => { + if (project.subscribeError) throw project.subscribeError; const unsubscribe = jest.fn(async () => true); const subscription: Subscription = { bookId: book, @@ -942,6 +946,22 @@ describe('onBookTextChanged', () => { await finishReading(project); }); + it('reports an edit to the outlined book the whole-text read could not watch', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + const listener = listen(); + await getBookOutline('src', 'GEN'); + project.subscribeError = new Error('no provider'); + readStatus(); + await settle(); + project.subscribeError = undefined; + + project.subscriptions[0].deliver(usj('GEN', 'v2')); + + expect(listener).toHaveBeenCalledTimes(1); + expect(listener).toHaveBeenCalledWith('src', 'GEN'); + }); + it('serves the edited outlined book when the whole-text read hears of the edit first', async () => { let version = 'v1'; const project = serveProject((bookId) => usj(bookId, version)); diff --git a/src/services/bookService.ts b/src/services/bookService.ts index 89cbbf86e..585b612fb 100644 --- a/src/services/bookService.ts +++ b/src/services/bookService.ts @@ -17,6 +17,8 @@ import { reanchorSegmentation } from '../utils/segmentation'; interface BookTextWatch { /** The text as first delivered; `undefined` when the project has no text for the book. */ text: Promise; + /** `false` once subscribing to the book's text has failed. */ + isWatching: () => boolean; stop: () => Promise; } @@ -66,7 +68,7 @@ interface TextRead { /** Books whose latest read failed. */ failedBookIds: Set; forms: FormIndex; - /** Stops watching each book, by book id. */ + /** Stops watching each book, by book id; a book that could not be subscribed to has none. */ watches: Map Promise>; } @@ -104,6 +106,7 @@ function watchBookText( reject: rejectText, } = Promise.withResolvers(); let isFirstDelivery = true; + let isSubscribed = true; const subscription = papi.projectDataProviders .get('platformScripture.USJ_Book', sourceProjectId) .then((pdp) => @@ -119,10 +122,14 @@ function watchBookText( else resolveText(usj); }), ); - subscription.catch(rejectText); + subscription.catch((e) => { + isSubscribed = false; + rejectText(e); + }); return { text, + isWatching: () => isSubscribed, stop: async () => { // Unsubscribing ahead of the first delivery would leave `text` unsettled. await text.catch(() => undefined); @@ -157,8 +164,10 @@ function holdBook(sourceProjectId: string, bookId: string, writingSystem: string // Read again at once, so the book stays watched and a refetch the listeners start reads the // edit. openBooks.set(sourceProjectId, holdBook(sourceProjectId, bookId, writingSystem)); - // A whole-text read that has the book reports the edit once it has read it again. - if (!textReads.get(sourceProjectId)?.readBookIds.has(bookId)) + // A whole-text read that has read the book and still watches it reports the edit once it has + // read it again. + const read = textReads.get(sourceProjectId); + if (!read?.readBookIds.has(bookId) || !read.watches.has(bookId)) reportEdit(sourceProjectId, bookId); }); const book = perfTrack('getTokenizedBook:usj', watch.text).then( @@ -350,7 +359,7 @@ async function readTextBook( bookId: string, ): Promise { read.watches.get(bookId)?.(); - const { text, stop } = watchBookText(sourceProjectId, bookId, async () => { + const { text, isWatching, stop } = watchBookText(sourceProjectId, bookId, async () => { if (textReads.get(sourceProjectId) !== read || read.watches.get(bookId) !== stop) return; rereadHeldBook(sourceProjectId, bookId); await readTextBook(sourceProjectId, read, bookId); @@ -375,6 +384,7 @@ async function readTextBook( read.failedBookIds.delete(bookId); } else { read.failedBookIds.add(bookId); + if (!isWatching()) read.watches.delete(bookId); } read.readBookIds.add(bookId); } From 331b2e379520a6b23dc021f747168290519af768 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 20 Oct 2026 10:47:29 -0600 Subject: [PATCH 08/24] Record the 2e checkpoint run, and name each result for what measured it The checkpoint ran on the reference machine the #407 baseline used. Each committed result now ends with the issue it was measured for, so a run's origin is clear without opening it. Co-Authored-By: Claude Opus 5.5 --- perf/README.md | 2 +- ...6-10-15T154959-10d81320-407-baseline.json} | 0 ...-10-20T164125-257ed462-439-checkpoint.json | 2241 +++++++++++++++++ ...6-10-15T155002-10d81320-407-baseline.json} | 0 4 files changed, 2242 insertions(+), 1 deletion(-) rename perf/results/{app-2026-10-15T154959-10d81320.json => app-2026-10-15T154959-10d81320-407-baseline.json} (100%) create mode 100644 perf/results/app-2026-10-20T164125-257ed462-439-checkpoint.json rename perf/results/{bench-2026-10-15T155002-10d81320.json => bench-2026-10-15T155002-10d81320-407-baseline.json} (100%) diff --git a/perf/README.md b/perf/README.md index b8f55a337..82c681835 100644 --- a/perf/README.md +++ b/perf/README.md @@ -13,7 +13,7 @@ Measures the extension rather than tests it: the data layer's pure functions in `PERF_TIERS` (comma-separated tier names) and `PERF_RUNS` (10 by default) narrow a run. The `large` tier's draft exceeds one websocket message, so only the Node benchmarks can measure it. -Results land in `perf/results/` as JSON, each recording the machine, its load average, and both commits. +Results land in `perf/results/` as JSON, each recording the machine, its load average, and both commits. Before committing one, suffix its name with the issue it was measured for and why, such as `-407-baseline`; `perf:summary` reads the time from the file, not its name. ## Dataset shape diff --git a/perf/results/app-2026-10-15T154959-10d81320.json b/perf/results/app-2026-10-15T154959-10d81320-407-baseline.json similarity index 100% rename from perf/results/app-2026-10-15T154959-10d81320.json rename to perf/results/app-2026-10-15T154959-10d81320-407-baseline.json diff --git a/perf/results/app-2026-10-20T164125-257ed462-439-checkpoint.json b/perf/results/app-2026-10-20T164125-257ed462-439-checkpoint.json new file mode 100644 index 000000000..371cdac14 --- /dev/null +++ b/perf/results/app-2026-10-20T164125-257ed462-439-checkpoint.json @@ -0,0 +1,2241 @@ +{ + "kind": "app", + "commit": "257ed46262677b135c975881b180ada13d731070", + "dirty": false, + "coreCommit": "97bc33e814438d22668d2b58f9bb78c4ad420ccd", + "recordedAt": "2026-10-20T16:41:25.329Z", + "machine": { + "cpu": "12th Gen Intel(R) Core(TM) i7-12700KF", + "cores": 20, + "memoryGb": 34, + "os": "Windows_NT 10.0.26200", + "node": "v25.8.1", + "loadAverage": [ + 0, + 0, + 0 + ] + }, + "appBuild": "packaged", + "extensionBuild": "production", + "runs": 10, + "tiers": { + "small": { + "viewBook": "PHP", + "dataset": { + "counts": { + "segments": 104, + "wordTokens": 2219, + "tokenAnalyses": 757, + "tokenAnalysisLinks": { + "approved": 2150, + "suggested": 105, + "rejected": 112, + "stale": 69 + }, + "phraseAnalyses": 42, + "phraseAnalysisLinks": 42, + "segmentAnalyses": 25 + }, + "bytes": { + "draftJson": 805671, + "largestShard": 805471 + } + }, + "wireBytes": { + "draft": 903815, + "projectsForSource": 4519075 + }, + "cold": { + "webView": { + "app-startup": { + "runs": 10, + "medianMs": 2323, + "p90Ms": 2583, + "minMs": 2287, + "maxMs": 3126 + }, + "book-render": { + "runs": 10, + "medianMs": 326, + "p90Ms": 346, + "minMs": 309.1, + "maxMs": 388.5 + }, + "draft-fetch": { + "runs": 10, + "medianMs": 58.9, + "p90Ms": 64.8, + "minMs": 55.5, + "maxMs": 73.2 + }, + "draft-parse": { + "runs": 10, + "medianMs": 1.1, + "p90Ms": 1.2, + "minMs": 1, + "maxMs": 1.3 + }, + "store-mount": { + "runs": 10, + "medianMs": 5, + "p90Ms": 6.2, + "minMs": 4.7, + "maxMs": 7.6 + }, + "book-fetch": { + "runs": 10, + "medianMs": 97, + "p90Ms": 107.6, + "minMs": 90.7, + "maxMs": 108.3 + }, + "book-parse": { + "runs": 10, + "medianMs": 0.5, + "p90Ms": 0.6, + "minMs": 0.5, + "maxMs": 0.6 + }, + "reanchor": { + "runs": 10, + "medianMs": 9.6, + "p90Ms": 11.8, + "minMs": 8.5, + "maxMs": 13.2 + }, + "outline-fetch": { + "runs": 10, + "medianMs": 13.8, + "p90Ms": 18.4, + "minMs": 12.7, + "maxMs": 18.9 + }, + "outline-parse": { + "runs": 10, + "medianMs": 0.1, + "p90Ms": 0.2, + "minMs": 0, + "maxMs": 0.2 + }, + "page-fetch": { + "runs": 10, + "medianMs": 36.9, + "p90Ms": 40.2, + "minMs": 35, + "maxMs": 40.7 + }, + "page-parse": { + "runs": 10, + "medianMs": 0.3, + "p90Ms": 0.4, + "minMs": 0.2, + "maxMs": 0.5 + }, + "open-to-interactive": { + "runs": 10, + "medianMs": 1677.4, + "p90Ms": 1861.8, + "minMs": 716.4, + "maxMs": 2687.1 + }, + "draft-fetch:transport": { + "runs": 10, + "medianMs": 49.589, + "p90Ms": 53.765, + "minMs": 45.132, + "maxMs": 62.607 + }, + "book-fetch:transport": { + "runs": 10, + "medianMs": 14.032, + "p90Ms": 14.863, + "minMs": 10.5, + "maxMs": 15.09 + }, + "outline-fetch:transport": { + "runs": 10, + "medianMs": 7.581, + "p90Ms": 9.355, + "minMs": 6.321, + "maxMs": 11.246 + } + }, + "host": { + "draft-fetch": { + "runs": 10, + "medianMs": 10.231, + "p90Ms": 11.811, + "minMs": 9.311, + "maxMs": 13.368 + }, + "draft-fetch:read": { + "runs": 10, + "medianMs": 2.192, + "p90Ms": 2.619, + "minMs": 1.849, + "maxMs": 6.09 + }, + "draft-fetch:parse": { + "runs": 10, + "medianMs": 4.994, + "p90Ms": 5.657, + "minMs": 4.466, + "maxMs": 6.13 + }, + "draft-fetch:compare": { + "runs": 10, + "medianMs": 1.174, + "p90Ms": 1.329, + "minMs": 1.134, + "maxMs": 1.568 + }, + "draft-fetch:serialize": { + "runs": 10, + "medianMs": 1.076, + "p90Ms": 1.228, + "minMs": 0.988, + "maxMs": 1.239 + }, + "book-fetch": { + "runs": 10, + "medianMs": 83.361, + "p90Ms": 92.819, + "minMs": 76.702, + "maxMs": 97.8 + }, + "book-fetch:usj": { + "runs": 10, + "medianMs": 65.137, + "p90Ms": 73.102, + "minMs": 63.141, + "maxMs": 80.316 + }, + "book-fetch:tokenize": { + "runs": 10, + "medianMs": 8.126, + "p90Ms": 8.837, + "minMs": 7.619, + "maxMs": 9.743 + }, + "book-fetch:serialize": { + "runs": 10, + "medianMs": 0.815, + "p90Ms": 0.854, + "minMs": 0.789, + "maxMs": 0.867 + }, + "outline-fetch": { + "runs": 10, + "medianMs": 6.539, + "p90Ms": 8.979, + "minMs": 5.223, + "maxMs": 9.654 + }, + "outline-fetch:resegment": { + "runs": 10, + "medianMs": 0.1, + "p90Ms": 0.148, + "minMs": 0.095, + "maxMs": 0.169 + }, + "outline-fetch:serialize": { + "runs": 10, + "medianMs": 0.227, + "p90Ms": 0.287, + "minMs": 0.197, + "maxMs": 0.292 + }, + "page-fetch": { + "runs": 10, + "medianMs": 0.606, + "p90Ms": 0.724, + "minMs": 0.458, + "maxMs": 0.88 + } + } + }, + "memory": { + "webView": { + "heapBeforeOpenBytes": 17371636, + "viewLoadedBytes": 82924436, + "withFullUndoHistoryBytes": 40299012, + "afterPickerBytes": 40745676, + "afterSaveBytes": 23408720, + "beforeTextReadBytes": 127574284, + "afterTextReadBytes": 121209604 + }, + "host": { + "heapBeforeOpenBytes": 88502076, + "viewLoadedBytes": 4370552, + "viewLoadedExternalBytes": 40, + "viewLoadedRssBytes": 5607424, + "withFullUndoHistoryBytes": 4460132, + "withFullUndoHistoryExternalBytes": 40, + "withFullUndoHistoryRssBytes": 6254592, + "afterPickerBytes": 5670536, + "afterPickerExternalBytes": 1211712, + "afterPickerRssBytes": 9940992, + "afterSaveBytes": 5346356, + "afterSaveExternalBytes": 1211712, + "afterSaveRssBytes": 10002432, + "beforeTextReadBytes": 6648476, + "beforeTextReadExternalBytes": 1211712, + "beforeTextReadRssBytes": 13963264, + "afterTextReadBytes": 44757192, + "afterTextReadExternalBytes": 1211712, + "afterTextReadRssBytes": 54771712 + } + }, + "open": { + "webView": { + "book-render": { + "runs": 10, + "medianMs": 242.1, + "p90Ms": 256.1, + "minMs": 238.2, + "maxMs": 342.1 + }, + "draft-fetch": { + "runs": 10, + "medianMs": 62.6, + "p90Ms": 66.4, + "minMs": 57.3, + "maxMs": 77.6 + }, + "draft-parse": { + "runs": 10, + "medianMs": 1.2, + "p90Ms": 1.3, + "minMs": 1, + "maxMs": 1.3 + }, + "store-mount": { + "runs": 10, + "medianMs": 5.2, + "p90Ms": 5.8, + "minMs": 4.8, + "maxMs": 8.6 + }, + "book-fetch": { + "runs": 10, + "medianMs": 68, + "p90Ms": 76.6, + "minMs": 34.8, + "maxMs": 99.3 + }, + "book-parse": { + "runs": 10, + "medianMs": 0.5, + "p90Ms": 0.6, + "minMs": 0.4, + "maxMs": 0.7 + }, + "reanchor": { + "runs": 10, + "medianMs": 8.7, + "p90Ms": 9.3, + "minMs": 8.4, + "maxMs": 9.7 + }, + "outline-fetch": { + "runs": 10, + "medianMs": 6.1, + "p90Ms": 14.3, + "minMs": 5.4, + "maxMs": 15.8 + }, + "outline-parse": { + "runs": 10, + "medianMs": 0.1, + "p90Ms": 0.1, + "minMs": 0, + "maxMs": 0.1 + }, + "page-fetch": { + "runs": 10, + "medianMs": 4.2, + "p90Ms": 6.6, + "minMs": 4, + "maxMs": 38.6 + }, + "page-parse": { + "runs": 10, + "medianMs": 0.3, + "p90Ms": 0.4, + "minMs": 0.3, + "maxMs": 0.4 + }, + "open-to-interactive": { + "runs": 10, + "medianMs": 558.8, + "p90Ms": 570.9, + "minMs": 551.6, + "maxMs": 1693.8 + }, + "draft-fetch:transport": { + "runs": 10, + "medianMs": 53.724, + "p90Ms": 57.735, + "minMs": 49.142, + "maxMs": 64.266 + }, + "book-fetch:transport": { + "runs": 10, + "medianMs": 47.254, + "p90Ms": 52.926, + "minMs": 12.392, + "maxMs": 60.932 + }, + "outline-fetch:transport": { + "runs": 10, + "medianMs": 1.165, + "p90Ms": 1.328, + "minMs": 1.068, + "maxMs": 8.714 + } + }, + "host": { + "draft-fetch": { + "runs": 10, + "medianMs": 8.515, + "p90Ms": 9.271, + "minMs": 7.597, + "maxMs": 13.334 + }, + "draft-fetch:read": { + "runs": 10, + "medianMs": 1.779, + "p90Ms": 2.373, + "minMs": 1.705, + "maxMs": 2.4 + }, + "draft-fetch:parse": { + "runs": 10, + "medianMs": 4.157, + "p90Ms": 4.396, + "minMs": 3.583, + "maxMs": 6.823 + }, + "draft-fetch:compare": { + "runs": 10, + "medianMs": 1.325, + "p90Ms": 1.786, + "minMs": 1.171, + "maxMs": 2.42 + }, + "draft-fetch:serialize": { + "runs": 10, + "medianMs": 1.009, + "p90Ms": 1.138, + "minMs": 0.941, + "maxMs": 1.29 + }, + "book-fetch": { + "runs": 10, + "medianMs": 18.074, + "p90Ms": 23.217, + "minMs": 15.668, + "maxMs": 84.537 + }, + "book-fetch:usj": { + "runs": 10, + "medianMs": 8.277, + "p90Ms": 12.074, + "minMs": 6.79, + "maxMs": 67.445 + }, + "book-fetch:tokenize": { + "runs": 10, + "medianMs": 2.328, + "p90Ms": 7.044, + "minMs": 2.017, + "maxMs": 8.401 + }, + "book-fetch:serialize": { + "runs": 10, + "medianMs": 0.846, + "p90Ms": 1.03, + "minMs": 0.796, + "maxMs": 1.129 + }, + "outline-fetch": { + "runs": 10, + "medianMs": 4.911, + "p90Ms": 7.086, + "minMs": 4.211, + "maxMs": 13.162 + }, + "outline-fetch:resegment": { + "runs": 10, + "medianMs": 0.004, + "p90Ms": 0.007, + "minMs": 0.003, + "maxMs": 0.125 + }, + "outline-fetch:serialize": { + "runs": 10, + "medianMs": 0.138, + "p90Ms": 0.211, + "minMs": 0.122, + "maxMs": 0.233 + }, + "page-fetch": { + "runs": 10, + "medianMs": 0.403, + "p90Ms": 0.525, + "minMs": 0.362, + "maxMs": 0.585 + } + } + }, + "switchBook": { + "book-render": { + "runs": 10, + "medianMs": 82.1, + "p90Ms": 88.1, + "minMs": 78.3, + "maxMs": 93.7 + }, + "book-fetch": { + "runs": 10, + "medianMs": 22.9, + "p90Ms": 25.7, + "minMs": 20.6, + "maxMs": 26.8 + }, + "book-parse": { + "runs": 10, + "medianMs": 0.5, + "p90Ms": 0.5, + "minMs": 0.4, + "maxMs": 0.6 + }, + "reanchor": { + "runs": 10, + "medianMs": 5.6, + "p90Ms": 6.9, + "minMs": 5, + "maxMs": 7.1 + }, + "outline-fetch": { + "runs": 10, + "medianMs": 5.7, + "p90Ms": 6.4, + "minMs": 5.3, + "maxMs": 9.4 + }, + "outline-parse": { + "runs": 10, + "medianMs": 0.1, + "p90Ms": 0.1, + "minMs": 0, + "maxMs": 0.2 + }, + "page-fetch": { + "runs": 10, + "medianMs": 3.8, + "p90Ms": 4.6, + "minMs": 3.6, + "maxMs": 5.3 + }, + "page-parse": { + "runs": 10, + "medianMs": 0.3, + "p90Ms": 0.3, + "minMs": 0.3, + "maxMs": 0.4 + } + }, + "edit": { + "webView": { + "dispatch-render": { + "runs": 10, + "medianMs": 5.4, + "p90Ms": 7.7, + "minMs": 4.7, + "maxMs": 9.3 + }, + "autosave-serialize": { + "runs": 10, + "medianMs": 1.7, + "p90Ms": 2.1, + "minMs": 1.6, + "maxMs": 2.6 + }, + "autosave": { + "runs": 10, + "medianMs": 18.5, + "p90Ms": 27.6, + "minMs": 17.7, + "maxMs": 30.5 + }, + "autosave:transport": { + "runs": 10, + "medianMs": 12.107, + "p90Ms": 13.544, + "minMs": 11.42, + "maxMs": 23.935 + } + }, + "host": { + "autosave": { + "runs": 10, + "medianMs": 6.363, + "p90Ms": 13.056, + "minMs": 6.025, + "maxMs": 15.088 + }, + "autosave:parse": { + "runs": 10, + "medianMs": 1.13, + "p90Ms": 1.274, + "minMs": 1.073, + "maxMs": 1.401 + }, + "autosave:read": { + "runs": 10, + "medianMs": 0.515, + "p90Ms": 0.711, + "minMs": 0.469, + "maxMs": 0.731 + }, + "autosave:compare": { + "runs": 10, + "medianMs": 1.71, + "p90Ms": 2.06, + "minMs": 1.491, + "maxMs": 2.339 + }, + "autosave:write": { + "runs": 10, + "medianMs": 2.511, + "p90Ms": 9.209, + "minMs": 2.125, + "maxMs": 11.444 + } + } + }, + "picker": { + "webView": { + "projects-fetch": { + "runs": 10, + "medianMs": 62, + "p90Ms": 64.5, + "minMs": 59.5, + "maxMs": 74.9 + }, + "projects-fetch:transport": { + "runs": 10, + "medianMs": 44.065, + "p90Ms": 45.809, + "minMs": 38.172, + "maxMs": 55.528 + } + }, + "host": { + "projects-fetch": { + "runs": 10, + "medianMs": 17.091, + "p90Ms": 19.372, + "minMs": 16.24, + "maxMs": 26.328 + }, + "projects-fetch:read": { + "runs": 10, + "medianMs": 3.889, + "p90Ms": 3.946, + "minMs": 3.725, + "maxMs": 7.073 + }, + "projects-fetch:parse": { + "runs": 10, + "medianMs": 12.795, + "p90Ms": 14.737, + "minMs": 11.953, + "maxMs": 18.203 + }, + "projects-fetch:serialize": { + "runs": 10, + "medianMs": 0.567, + "p90Ms": 0.698, + "minMs": 0.529, + "maxMs": 0.915 + } + } + }, + "save": { + "webView": { + "save": { + "runs": 10, + "medianMs": 65.7, + "p90Ms": 66.3, + "minMs": 64.5, + "maxMs": 71.6 + }, + "save-bytes": { + "runs": 10, + "medianMs": 805471, + "p90Ms": 805471, + "minMs": 805471, + "maxMs": 805471 + }, + "save:transport": { + "runs": 10, + "medianMs": 54.48, + "p90Ms": 55.325, + "minMs": 48.481, + "maxMs": 61.15 + } + }, + "host": { + "save": { + "runs": 10, + "medianMs": 10.52, + "p90Ms": 16.62, + "minMs": 10.091, + "maxMs": 17.447 + }, + "save:parse": { + "runs": 10, + "medianMs": 5.095, + "p90Ms": 5.495, + "minMs": 4.985, + "maxMs": 6.071 + }, + "save:read": { + "runs": 10, + "medianMs": 1.099, + "p90Ms": 2.893, + "minMs": 1.066, + "maxMs": 7.564 + }, + "save:write": { + "runs": 10, + "medianMs": 3.037, + "p90Ms": 3.126, + "minMs": 2.842, + "maxMs": 6.254 + }, + "save:serialize": { + "runs": 10, + "medianMs": 0.945, + "p90Ms": 1.211, + "minMs": 0.914, + "maxMs": 3.317 + } + } + }, + "concordance": { + "concordance-read": { + "runs": 10, + "medianMs": 3764.6, + "p90Ms": 3861.7, + "minMs": 3727.2, + "maxMs": 3877.3 + }, + "concordance-entries": { + "runs": 10, + "medianMs": 46.7, + "p90Ms": 47.6, + "minMs": 45.2, + "maxMs": 48.2 + } + }, + "textRead": { + "text-read": { + "runs": 10, + "medianMs": 1376.25, + "p90Ms": 1422.251, + "minMs": 1337.438, + "maxMs": 1480.072 + }, + "text-read-tokenize": { + "runs": 10, + "medianMs": 604.149, + "p90Ms": 621.291, + "minMs": 588.354, + "maxMs": 630.292 + } + } + }, + "medium": { + "viewBook": "PSA", + "dataset": { + "counts": { + "segments": 2583, + "wordTokens": 41324, + "tokenAnalyses": 4797, + "tokenAnalysisLinks": { + "approved": 40059, + "suggested": 1963, + "rejected": 1965, + "stale": 1265 + }, + "phraseAnalyses": 756, + "phraseAnalysisLinks": 786, + "segmentAnalyses": 718 + }, + "bytes": { + "draftJson": 13270273, + "largestShard": 13267903 + } + }, + "wireBytes": { + "draft": 14851957, + "projectsForSource": 74259785 + }, + "cold": { + "webView": { + "app-startup": { + "runs": 10, + "medianMs": 2310, + "p90Ms": 2318, + "minMs": 2278, + "maxMs": 2324 + }, + "book-render": { + "runs": 10, + "medianMs": 1133.4, + "p90Ms": 1155.2, + "minMs": 1104.7, + "maxMs": 1162.2 + }, + "draft-fetch": { + "runs": 10, + "medianMs": 367.6, + "p90Ms": 372.8, + "minMs": 350.7, + "maxMs": 387 + }, + "draft-parse": { + "runs": 10, + "medianMs": 17.6, + "p90Ms": 18, + "minMs": 17.3, + "maxMs": 18 + }, + "store-mount": { + "runs": 10, + "medianMs": 18.9, + "p90Ms": 19.4, + "minMs": 18.4, + "maxMs": 19.6 + }, + "book-fetch": { + "runs": 10, + "medianMs": 371.5, + "p90Ms": 381.5, + "minMs": 362.8, + "maxMs": 405.2 + }, + "book-parse": { + "runs": 10, + "medianMs": 9.5, + "p90Ms": 9.9, + "minMs": 9.3, + "maxMs": 9.9 + }, + "reanchor": { + "runs": 10, + "medianMs": 142.9, + "p90Ms": 144.3, + "minMs": 140.9, + "maxMs": 145.5 + }, + "outline-fetch": { + "runs": 10, + "medianMs": 15.5, + "p90Ms": 21, + "minMs": 15.1, + "maxMs": 27.6 + }, + "outline-parse": { + "runs": 10, + "medianMs": 1.2, + "p90Ms": 1.2, + "minMs": 1.1, + "maxMs": 1.2 + }, + "page-fetch": { + "runs": 10, + "medianMs": 4, + "p90Ms": 4.1, + "minMs": 3.8, + "maxMs": 4.1 + }, + "page-parse": { + "runs": 10, + "medianMs": 0.3, + "p90Ms": 0.3, + "minMs": 0.2, + "maxMs": 0.4 + }, + "open-to-interactive": { + "runs": 10, + "medianMs": 2501.4, + "p90Ms": 3476.4, + "minMs": 2478.2, + "maxMs": 3522.6 + }, + "draft-fetch:transport": { + "runs": 10, + "medianMs": 244.566, + "p90Ms": 251.381, + "minMs": 230.45, + "maxMs": 266.721 + }, + "book-fetch:transport": { + "runs": 10, + "medianMs": 123.353, + "p90Ms": 127.799, + "minMs": 121.503, + "maxMs": 147.808 + }, + "outline-fetch:transport": { + "runs": 10, + "medianMs": 7.438, + "p90Ms": 12.982, + "minMs": 7.294, + "maxMs": 19.891 + } + }, + "host": { + "draft-fetch": { + "runs": 10, + "medianMs": 122.615, + "p90Ms": 125.903, + "minMs": 120.25, + "maxMs": 135.194 + }, + "draft-fetch:read": { + "runs": 10, + "medianMs": 13.732, + "p90Ms": 15.727, + "minMs": 13.144, + "maxMs": 17.821 + }, + "draft-fetch:parse": { + "runs": 10, + "medianMs": 62.731, + "p90Ms": 65.857, + "minMs": 61.343, + "maxMs": 68.329 + }, + "draft-fetch:compare": { + "runs": 10, + "medianMs": 24.619, + "p90Ms": 26.654, + "minMs": 19.709, + "maxMs": 28.834 + }, + "draft-fetch:serialize": { + "runs": 10, + "medianMs": 19.306, + "p90Ms": 26.226, + "minMs": 18.629, + "maxMs": 26.397 + }, + "book-fetch": { + "runs": 10, + "medianMs": 247.163, + "p90Ms": 257.928, + "minMs": 239.5, + "maxMs": 258.697 + }, + "book-fetch:usj": { + "runs": 10, + "medianMs": 177.607, + "p90Ms": 193.524, + "minMs": 171.918, + "maxMs": 195.166 + }, + "book-fetch:tokenize": { + "runs": 10, + "medianMs": 45.413, + "p90Ms": 47.177, + "minMs": 42.212, + "maxMs": 48.184 + }, + "book-fetch:serialize": { + "runs": 10, + "medianMs": 14.5, + "p90Ms": 15.038, + "minMs": 14.085, + "maxMs": 16.084 + }, + "outline-fetch": { + "runs": 10, + "medianMs": 8.015, + "p90Ms": 9.242, + "minMs": 7.593, + "maxMs": 10.906 + }, + "outline-fetch:resegment": { + "runs": 10, + "medianMs": 0.096, + "p90Ms": 0.105, + "minMs": 0.09, + "maxMs": 0.118 + }, + "outline-fetch:serialize": { + "runs": 10, + "medianMs": 2.633, + "p90Ms": 2.793, + "minMs": 2.559, + "maxMs": 3.047 + }, + "page-fetch": { + "runs": 10, + "medianMs": 0.417, + "p90Ms": 0.47, + "minMs": 0.406, + "maxMs": 0.483 + } + } + }, + "memory": { + "webView": { + "heapBeforeOpenBytes": 17422652, + "viewLoadedBytes": 198630264, + "withFullUndoHistoryBytes": 252319344, + "afterPickerBytes": 252745520, + "afterSaveBytes": 66020232, + "beforeTextReadBytes": 243316176, + "afterTextReadBytes": 227956672 + }, + "host": { + "heapBeforeOpenBytes": 88487296, + "viewLoadedBytes": 48606860, + "viewLoadedExternalBytes": 1408, + "viewLoadedRssBytes": 52846592, + "withFullUndoHistoryBytes": 49546276, + "withFullUndoHistoryExternalBytes": 1211712, + "withFullUndoHistoryRssBytes": 56410112, + "afterPickerBytes": 49178460, + "afterPickerExternalBytes": 1211712, + "afterPickerRssBytes": 57311232, + "afterSaveBytes": 49205216, + "afterSaveExternalBytes": 1211712, + "afterSaveRssBytes": 56127488, + "beforeTextReadBytes": 50656524, + "beforeTextReadExternalBytes": 1211712, + "beforeTextReadRssBytes": 59482112, + "afterTextReadBytes": 88800784, + "afterTextReadExternalBytes": 1211712, + "afterTextReadRssBytes": 101355520 + } + }, + "open": { + "webView": { + "book-render": { + "runs": 10, + "medianMs": 1002.2, + "p90Ms": 1059.3, + "minMs": 958.5, + "maxMs": 1153.5 + }, + "draft-fetch": { + "runs": 10, + "medianMs": 372.5, + "p90Ms": 380.6, + "minMs": 350.6, + "maxMs": 384.5 + }, + "draft-parse": { + "runs": 10, + "medianMs": 17.4, + "p90Ms": 25.1, + "minMs": 16.9, + "maxMs": 26 + }, + "store-mount": { + "runs": 10, + "medianMs": 17.4, + "p90Ms": 25.6, + "minMs": 17, + "maxMs": 34.3 + }, + "book-fetch": { + "runs": 10, + "medianMs": 255.5, + "p90Ms": 294.5, + "minMs": 244.7, + "maxMs": 400.6 + }, + "book-parse": { + "runs": 10, + "medianMs": 8.9, + "p90Ms": 9.2, + "minMs": 8.7, + "maxMs": 14.9 + }, + "reanchor": { + "runs": 10, + "medianMs": 146.1, + "p90Ms": 175.1, + "minMs": 140.2, + "maxMs": 175.3 + }, + "outline-fetch": { + "runs": 10, + "medianMs": 13.9, + "p90Ms": 15.4, + "minMs": 12.6, + "maxMs": 16.7 + }, + "outline-parse": { + "runs": 10, + "medianMs": 1.2, + "p90Ms": 1.3, + "minMs": 1, + "maxMs": 1.3 + }, + "page-fetch": { + "runs": 10, + "medianMs": 3.7, + "p90Ms": 3.9, + "minMs": 3.6, + "maxMs": 4 + }, + "page-parse": { + "runs": 10, + "medianMs": 0.2, + "p90Ms": 0.4, + "minMs": 0.2, + "maxMs": 0.4 + }, + "open-to-interactive": { + "runs": 10, + "medianMs": 1317.2, + "p90Ms": 1378.1, + "minMs": 1270.8, + "maxMs": 3527.2 + }, + "draft-fetch:transport": { + "runs": 10, + "medianMs": 230.337, + "p90Ms": 245.132, + "minMs": 225.3, + "maxMs": 250.183 + }, + "book-fetch:transport": { + "runs": 10, + "medianMs": 115.767, + "p90Ms": 124.49, + "minMs": 106.337, + "maxMs": 124.519 + }, + "outline-fetch:transport": { + "runs": 10, + "medianMs": 6.948, + "p90Ms": 7.297, + "minMs": 6.647, + "maxMs": 7.65 + } + }, + "host": { + "draft-fetch": { + "runs": 10, + "medianMs": 134.317, + "p90Ms": 143.124, + "minMs": 114.916, + "maxMs": 145.463 + }, + "draft-fetch:read": { + "runs": 10, + "medianMs": 21.193, + "p90Ms": 25.486, + "minMs": 12.57, + "maxMs": 33.111 + }, + "draft-fetch:parse": { + "runs": 10, + "medianMs": 69.617, + "p90Ms": 77.591, + "minMs": 54.184, + "maxMs": 81.31 + }, + "draft-fetch:compare": { + "runs": 10, + "medianMs": 20.413, + "p90Ms": 26.7, + "minMs": 18.344, + "maxMs": 28.631 + }, + "draft-fetch:serialize": { + "runs": 10, + "medianMs": 19.666, + "p90Ms": 22.422, + "minMs": 18.477, + "maxMs": 24.3 + }, + "book-fetch": { + "runs": 10, + "medianMs": 141.218, + "p90Ms": 188.163, + "minMs": 132.586, + "maxMs": 276.082 + }, + "book-fetch:usj": { + "runs": 10, + "medianMs": 81.905, + "p90Ms": 112.027, + "minMs": 71.655, + "maxMs": 209.474 + }, + "book-fetch:tokenize": { + "runs": 10, + "medianMs": 39.066, + "p90Ms": 42.536, + "minMs": 34.65, + "maxMs": 53.739 + }, + "book-fetch:serialize": { + "runs": 10, + "medianMs": 15.754, + "p90Ms": 16.676, + "minMs": 14.405, + "maxMs": 16.965 + }, + "outline-fetch": { + "runs": 10, + "medianMs": 6.803, + "p90Ms": 8.185, + "minMs": 5.953, + "maxMs": 9.051 + }, + "outline-fetch:resegment": { + "runs": 10, + "medianMs": 0.004, + "p90Ms": 0.006, + "minMs": 0.003, + "maxMs": 0.099 + }, + "outline-fetch:serialize": { + "runs": 10, + "medianMs": 1.841, + "p90Ms": 2.309, + "minMs": 1.529, + "maxMs": 2.997 + }, + "page-fetch": { + "runs": 10, + "medianMs": 0.367, + "p90Ms": 0.385, + "minMs": 0.347, + "maxMs": 0.436 + } + } + }, + "switchBook": { + "outline-fetch": { + "runs": 18, + "medianMs": 12.6, + "p90Ms": 24.3, + "minMs": 6, + "maxMs": 26.1 + }, + "outline-parse": { + "runs": 18, + "medianMs": 1.1, + "p90Ms": 1.3, + "minMs": 0, + "maxMs": 1.3 + }, + "page-fetch": { + "runs": 19, + "medianMs": 3.7, + "p90Ms": 6.2, + "minMs": 3.3, + "maxMs": 21.3 + }, + "book-fetch": { + "runs": 10, + "medianMs": 224.6, + "p90Ms": 249.7, + "minMs": 212.7, + "maxMs": 251.7 + }, + "book-render": { + "runs": 10, + "medianMs": 466.2, + "p90Ms": 489.5, + "minMs": 437.1, + "maxMs": 500.2 + }, + "page-parse": { + "runs": 19, + "medianMs": 0.2, + "p90Ms": 0.3, + "minMs": 0, + "maxMs": 0.3 + }, + "book-parse": { + "runs": 10, + "medianMs": 8.5, + "p90Ms": 9.1, + "minMs": 8.5, + "maxMs": 13.5 + }, + "reanchor": { + "runs": 10, + "medianMs": 128.1, + "p90Ms": 131.2, + "minMs": 120, + "maxMs": 138 + } + }, + "edit": { + "webView": { + "dispatch-render": { + "runs": 10, + "medianMs": 78.2, + "p90Ms": 80.4, + "minMs": 70, + "maxMs": 83.1 + }, + "autosave-serialize": { + "runs": 10, + "medianMs": 28.5, + "p90Ms": 29.4, + "minMs": 27.4, + "maxMs": 29.8 + }, + "autosave": { + "runs": 10, + "medianMs": 266.8, + "p90Ms": 280.6, + "minMs": 255.6, + "maxMs": 297.5 + }, + "autosave:transport": { + "runs": 10, + "medianMs": 188.143, + "p90Ms": 201.653, + "minMs": 185.762, + "maxMs": 210.342 + } + }, + "host": { + "autosave": { + "runs": 10, + "medianMs": 78.338, + "p90Ms": 87.158, + "minMs": 68.457, + "maxMs": 87.703 + }, + "autosave:parse": { + "runs": 10, + "medianMs": 21.004, + "p90Ms": 25.509, + "minMs": 17.69, + "maxMs": 37.106 + }, + "autosave:read": { + "runs": 10, + "medianMs": 0.8, + "p90Ms": 12.966, + "minMs": 0.651, + "maxMs": 13.581 + }, + "autosave:compare": { + "runs": 10, + "medianMs": 26.797, + "p90Ms": 29.005, + "minMs": 25.646, + "maxMs": 29.484 + }, + "autosave:write": { + "runs": 10, + "medianMs": 19.956, + "p90Ms": 20.231, + "minMs": 19.406, + "maxMs": 21.01 + } + } + }, + "picker": { + "webView": { + "projects-fetch": { + "runs": 10, + "medianMs": 354.5, + "p90Ms": 356.5, + "minMs": 351.5, + "maxMs": 367.8 + }, + "projects-fetch:transport": { + "runs": 10, + "medianMs": 14.372, + "p90Ms": 14.776, + "minMs": 13.696, + "maxMs": 16.191 + } + }, + "host": { + "projects-fetch": { + "runs": 10, + "medianMs": 339.607, + "p90Ms": 341.955, + "minMs": 336.824, + "maxMs": 353.526 + }, + "projects-fetch:read": { + "runs": 10, + "medianMs": 67.553, + "p90Ms": 73.588, + "minMs": 53.32, + "maxMs": 73.772 + }, + "projects-fetch:parse": { + "runs": 10, + "medianMs": 262.612, + "p90Ms": 272.63, + "minMs": 256.383, + "maxMs": 273.946 + }, + "projects-fetch:serialize": { + "runs": 10, + "medianMs": 8.74, + "p90Ms": 9.404, + "minMs": 8.351, + "maxMs": 9.415 + } + } + }, + "save": { + "webView": { + "save": { + "runs": 10, + "medianMs": 593.5, + "p90Ms": 610, + "minMs": 586.8, + "maxMs": 611.8 + }, + "save-bytes": { + "runs": 10, + "medianMs": 13267903, + "p90Ms": 13267903, + "minMs": 13267903, + "maxMs": 13267903 + }, + "save:transport": { + "runs": 10, + "medianMs": 402.831, + "p90Ms": 415.373, + "minMs": 396.987, + "maxMs": 416.285 + } + }, + "host": { + "save": { + "runs": 10, + "medianMs": 193.09, + "p90Ms": 196.427, + "minMs": 187.269, + "maxMs": 197.862 + }, + "save:parse": { + "runs": 10, + "medianMs": 103.453, + "p90Ms": 106.5, + "minMs": 93.404, + "maxMs": 107.365 + }, + "save:read": { + "runs": 10, + "medianMs": 28.97, + "p90Ms": 31.576, + "minMs": 14.751, + "maxMs": 32.881 + }, + "save:write": { + "runs": 10, + "medianMs": 40.705, + "p90Ms": 51.919, + "minMs": 37.683, + "maxMs": 53.053 + }, + "save:serialize": { + "runs": 10, + "medianMs": 19.455, + "p90Ms": 22.587, + "minMs": 18.764, + "maxMs": 23.914 + } + } + }, + "concordance": { + "concordance-read": { + "runs": 10, + "medianMs": 3702.6, + "p90Ms": 3802.6, + "minMs": 3626.1, + "maxMs": 3859.2 + }, + "concordance-entries": { + "runs": 10, + "medianMs": 47.1, + "p90Ms": 48.4, + "minMs": 42.1, + "maxMs": 48.9 + } + }, + "textRead": { + "text-read": { + "runs": 10, + "medianMs": 1327.226, + "p90Ms": 1376.466, + "minMs": 1298.338, + "maxMs": 1432.438 + }, + "text-read-tokenize": { + "runs": 10, + "medianMs": 586.437, + "p90Ms": 602.643, + "minMs": 579.301, + "maxMs": 606.826 + } + } + }, + "large-loadable": { + "viewBook": "PSA", + "dataset": { + "counts": { + "segments": 37691, + "wordTokens": 905971, + "tokenAnalyses": 16325, + "tokenAnalysisLinks": { + "approved": 262670, + "stale": 8121, + "suggested": 13072, + "rejected": 13201 + }, + "phraseAnalyses": 4725, + "phraseAnalysisLinks": 5197, + "segmentAnalyses": 11443 + }, + "bytes": { + "draftJson": 88502437, + "largestShard": 4534660 + } + }, + "wireBytes": { + "draft": 98742725, + "projectsForSource": 98742725 + }, + "cold": { + "webView": { + "app-startup": { + "runs": 10, + "medianMs": 2303, + "p90Ms": 2356, + "minMs": 2292, + "maxMs": 3082 + }, + "book-render": { + "runs": 10, + "medianMs": 3453.5, + "p90Ms": 3815, + "minMs": 3357.5, + "maxMs": 3837.1 + }, + "draft-fetch": { + "runs": 10, + "medianMs": 2471.5, + "p90Ms": 2492.8, + "minMs": 2407.2, + "maxMs": 2497.3 + }, + "book-fetch": { + "runs": 10, + "medianMs": 2369.6, + "p90Ms": 2437.2, + "minMs": 354.4, + "maxMs": 2461.9 + }, + "draft-parse": { + "runs": 10, + "medianMs": 160.6, + "p90Ms": 162.6, + "minMs": 159.5, + "maxMs": 163.3 + }, + "store-mount": { + "runs": 10, + "medianMs": 87.6, + "p90Ms": 90.2, + "minMs": 86.9, + "maxMs": 92.2 + }, + "book-parse": { + "runs": 10, + "medianMs": 8.8, + "p90Ms": 8.9, + "minMs": 8.5, + "maxMs": 9 + }, + "reanchor": { + "runs": 10, + "medianMs": 346.8, + "p90Ms": 363.5, + "minMs": 337.7, + "maxMs": 364.1 + }, + "outline-fetch": { + "runs": 10, + "medianMs": 17.8, + "p90Ms": 28.8, + "minMs": 14.8, + "maxMs": 29.2 + }, + "outline-parse": { + "runs": 10, + "medianMs": 1.1, + "p90Ms": 1.2, + "minMs": 1, + "maxMs": 1.2 + }, + "page-fetch": { + "runs": 10, + "medianMs": 4, + "p90Ms": 4.2, + "minMs": 3.9, + "maxMs": 4.2 + }, + "page-parse": { + "runs": 10, + "medianMs": 0.2, + "p90Ms": 0.3, + "minMs": 0.2, + "maxMs": 0.3 + }, + "open-to-interactive": { + "runs": 10, + "medianMs": 4800.3, + "p90Ms": 5170.2, + "minMs": 4715.6, + "maxMs": 6202.5 + }, + "draft-fetch:transport": { + "runs": 10, + "medianMs": 1571.376, + "p90Ms": 1618.752, + "minMs": 1488.89, + "maxMs": 1627.038 + }, + "book-fetch:transport": { + "runs": 10, + "medianMs": 290.447, + "p90Ms": 1384.983, + "minMs": 113.682, + "maxMs": 1413.153 + }, + "outline-fetch:transport": { + "runs": 10, + "medianMs": 7.454, + "p90Ms": 20.981, + "minMs": 6.987, + "maxMs": 21.354 + } + }, + "host": { + "draft-fetch": { + "runs": 10, + "medianMs": 891.158, + "p90Ms": 907.395, + "minMs": 870.262, + "maxMs": 925.51 + }, + "draft-fetch:read": { + "runs": 10, + "medianMs": 125.258, + "p90Ms": 133.304, + "minMs": 109.31, + "maxMs": 137.65 + }, + "draft-fetch:parse": { + "runs": 10, + "medianMs": 472.645, + "p90Ms": 495.393, + "minMs": 446.309, + "maxMs": 504.82 + }, + "draft-fetch:compare": { + "runs": 10, + "medianMs": 140.627, + "p90Ms": 171.865, + "minMs": 132.185, + "maxMs": 171.919 + }, + "draft-fetch:serialize": { + "runs": 10, + "medianMs": 140.561, + "p90Ms": 145.36, + "minMs": 137.544, + "maxMs": 146.84 + }, + "book-fetch": { + "runs": 10, + "medianMs": 1019.147, + "p90Ms": 2079.869, + "minMs": 239.836, + "maxMs": 2171.453 + }, + "book-fetch:usj": { + "runs": 10, + "medianMs": 185.856, + "p90Ms": 196.892, + "minMs": 172.588, + "maxMs": 199.414 + }, + "book-fetch:tokenize": { + "runs": 10, + "medianMs": 48.29, + "p90Ms": 49.17, + "minMs": 46.047, + "maxMs": 49.382 + }, + "book-fetch:serialize": { + "runs": 10, + "medianMs": 14.241, + "p90Ms": 15.021, + "minMs": 13.747, + "maxMs": 15.238 + }, + "outline-fetch": { + "runs": 10, + "medianMs": 7.646, + "p90Ms": 10.663, + "minMs": 7.2, + "maxMs": 10.811 + }, + "outline-fetch:resegment": { + "runs": 10, + "medianMs": 0.092, + "p90Ms": 0.115, + "minMs": 0.088, + "maxMs": 0.162 + }, + "outline-fetch:serialize": { + "runs": 10, + "medianMs": 2.248, + "p90Ms": 2.649, + "minMs": 2.12, + "maxMs": 2.773 + }, + "page-fetch": { + "runs": 10, + "medianMs": 0.397, + "p90Ms": 0.415, + "minMs": 0.381, + "maxMs": 0.452 + } + } + }, + "memory": { + "webView": { + "heapBeforeOpenBytes": 17334424, + "viewLoadedBytes": 255883004, + "withFullUndoHistoryBytes": 1417312856, + "afterPickerBytes": 1417812196, + "afterSaveBytes": 424045708, + "beforeTextReadBytes": 655435960, + "afterTextReadBytes": 554185564 + }, + "host": { + "heapBeforeOpenBytes": 88462776, + "viewLoadedBytes": 224412036, + "viewLoadedExternalBytes": 1211712, + "viewLoadedRssBytes": 232591360, + "withFullUndoHistoryBytes": 224637752, + "withFullUndoHistoryExternalBytes": 1211712, + "withFullUndoHistoryRssBytes": 233406464, + "afterPickerBytes": 224661984, + "afterPickerExternalBytes": 1211712, + "afterPickerRssBytes": 237309952, + "afterSaveBytes": 224674924, + "afterSaveExternalBytes": 1211712, + "afterSaveRssBytes": 233779200, + "beforeTextReadBytes": 225917984, + "beforeTextReadExternalBytes": 1213086, + "beforeTextReadRssBytes": 240730112, + "afterTextReadBytes": 263814012, + "afterTextReadExternalBytes": 1211712, + "afterTextReadRssBytes": 281403392 + } + }, + "open": { + "webView": { + "book-render": { + "runs": 10, + "medianMs": 3605.5, + "p90Ms": 3855.5, + "minMs": 3394.4, + "maxMs": 3889 + }, + "draft-fetch": { + "runs": 10, + "medianMs": 2448.5, + "p90Ms": 2461.3, + "minMs": 2376.9, + "maxMs": 2468.8 + }, + "book-fetch": { + "runs": 10, + "medianMs": 2417.9, + "p90Ms": 2463, + "minMs": 250.3, + "maxMs": 2480.6 + }, + "draft-parse": { + "runs": 10, + "medianMs": 164.3, + "p90Ms": 179.4, + "minMs": 161.5, + "maxMs": 180.3 + }, + "store-mount": { + "runs": 10, + "medianMs": 89.3, + "p90Ms": 91.2, + "minMs": 87, + "maxMs": 91.3 + }, + "book-parse": { + "runs": 10, + "medianMs": 8.9, + "p90Ms": 9, + "minMs": 8.7, + "maxMs": 9.1 + }, + "reanchor": { + "runs": 10, + "medianMs": 398.3, + "p90Ms": 404.2, + "minMs": 353.5, + "maxMs": 406.6 + }, + "outline-fetch": { + "runs": 10, + "medianMs": 16.2, + "p90Ms": 23.4, + "minMs": 15, + "maxMs": 37.5 + }, + "outline-parse": { + "runs": 10, + "medianMs": 1.6, + "p90Ms": 1.9, + "minMs": 1.1, + "maxMs": 2.2 + }, + "page-fetch": { + "runs": 10, + "medianMs": 5.4, + "p90Ms": 6.9, + "minMs": 4, + "maxMs": 40.8 + }, + "page-parse": { + "runs": 10, + "medianMs": 0.4, + "p90Ms": 0.7, + "minMs": 0.3, + "maxMs": 0.7 + }, + "open-to-interactive": { + "runs": 10, + "medianMs": 3926.4, + "p90Ms": 4205.2, + "minMs": 3839.1, + "maxMs": 5769.5 + }, + "draft-fetch:transport": { + "runs": 10, + "medianMs": 1576.632, + "p90Ms": 1591.817, + "minMs": 1519.928, + "maxMs": 1600.63 + }, + "book-fetch:transport": { + "runs": 10, + "medianMs": 403.441, + "p90Ms": 1525.484, + "minMs": 109.981, + "maxMs": 1544.364 + }, + "outline-fetch:transport": { + "runs": 10, + "medianMs": 8.784, + "p90Ms": 11.429, + "minMs": 7.903, + "maxMs": 25.756 + } + }, + "host": { + "draft-fetch": { + "runs": 10, + "medianMs": 870.823, + "p90Ms": 885.307, + "minMs": 856.083, + "maxMs": 897.122 + }, + "draft-fetch:read": { + "runs": 10, + "medianMs": 110.27, + "p90Ms": 134.167, + "minMs": 99.549, + "maxMs": 143.29 + }, + "draft-fetch:parse": { + "runs": 10, + "medianMs": 479.944, + "p90Ms": 491.57, + "minMs": 448.022, + "maxMs": 499.05 + }, + "draft-fetch:compare": { + "runs": 10, + "medianMs": 135.801, + "p90Ms": 137.359, + "minMs": 134.268, + "maxMs": 140.328 + }, + "draft-fetch:serialize": { + "runs": 10, + "medianMs": 137.892, + "p90Ms": 142.228, + "minMs": 132.541, + "maxMs": 142.441 + }, + "book-fetch": { + "runs": 10, + "medianMs": 902.116, + "p90Ms": 2060.275, + "minMs": 138.758, + "maxMs": 2104.339 + }, + "book-fetch:usj": { + "runs": 10, + "medianMs": 82.866, + "p90Ms": 150.927, + "minMs": 76.359, + "maxMs": 182.156 + }, + "book-fetch:tokenize": { + "runs": 10, + "medianMs": 44.927, + "p90Ms": 50.419, + "minMs": 37.669, + "maxMs": 55.761 + }, + "book-fetch:serialize": { + "runs": 10, + "medianMs": 14.674, + "p90Ms": 16.784, + "minMs": 13.588, + "maxMs": 17.733 + }, + "outline-fetch": { + "runs": 10, + "medianMs": 7.516, + "p90Ms": 11.744, + "minMs": 7.097, + "maxMs": 12.613 + }, + "outline-fetch:resegment": { + "runs": 10, + "medianMs": 0.004, + "p90Ms": 0.008, + "minMs": 0.003, + "maxMs": 0.092 + }, + "outline-fetch:serialize": { + "runs": 10, + "medianMs": 2.413, + "p90Ms": 2.779, + "minMs": 1.735, + "maxMs": 4.373 + }, + "page-fetch": { + "runs": 10, + "medianMs": 0.409, + "p90Ms": 0.496, + "minMs": 0.39, + "maxMs": 0.64 + } + } + }, + "switchBook": { + "page-fetch": { + "runs": 20, + "medianMs": 3.9, + "p90Ms": 4.6, + "minMs": 2.1, + "maxMs": 5.5 + }, + "page-parse": { + "runs": 20, + "medianMs": 0.2, + "p90Ms": 0.3, + "minMs": 0.1, + "maxMs": 0.4 + }, + "book-render": { + "runs": 10, + "medianMs": 659.8, + "p90Ms": 713, + "minMs": 625.4, + "maxMs": 764.7 + }, + "book-fetch": { + "runs": 10, + "medianMs": 220.6, + "p90Ms": 267.2, + "minMs": 215.8, + "maxMs": 268.3 + }, + "book-parse": { + "runs": 10, + "medianMs": 8.7, + "p90Ms": 9.2, + "minMs": 8.4, + "maxMs": 13.2 + }, + "reanchor": { + "runs": 10, + "medianMs": 324.6, + "p90Ms": 363.3, + "minMs": 309, + "maxMs": 403.3 + }, + "outline-fetch": { + "runs": 19, + "medianMs": 13.9, + "p90Ms": 26.3, + "minMs": 6.1, + "maxMs": 30.6 + }, + "outline-parse": { + "runs": 19, + "medianMs": 1, + "p90Ms": 1.1, + "minMs": 0, + "maxMs": 1.2 + } + }, + "edit": { + "webView": { + "dispatch-render": { + "runs": 10, + "medianMs": 570.2, + "p90Ms": 592.4, + "minMs": 507.1, + "maxMs": 597.6 + }, + "autosave-serialize": { + "runs": 10, + "medianMs": 206, + "p90Ms": 211, + "minMs": 199.9, + "maxMs": 212.4 + }, + "autosave": { + "runs": 10, + "medianMs": 1733.9, + "p90Ms": 1758.4, + "minMs": 1682.1, + "maxMs": 1761.2 + }, + "autosave:transport": { + "runs": 10, + "medianMs": 1325.761, + "p90Ms": 1348.426, + "minMs": 1263.559, + "maxMs": 1351.55 + } + }, + "host": { + "autosave": { + "runs": 10, + "medianMs": 408.139, + "p90Ms": 419.193, + "minMs": 401.74, + "maxMs": 429.541 + }, + "autosave:parse": { + "runs": 10, + "medianMs": 173.743, + "p90Ms": 177.675, + "minMs": 171.098, + "maxMs": 185.941 + }, + "autosave:read": { + "runs": 10, + "medianMs": 1.167, + "p90Ms": 1.67, + "minMs": 0.747, + "maxMs": 1.882 + }, + "autosave:compare": { + "runs": 10, + "medianMs": 213.799, + "p90Ms": 219.698, + "minMs": 208.329, + "maxMs": 221.97 + }, + "autosave:write": { + "runs": 10, + "medianMs": 7.989, + "p90Ms": 8.365, + "minMs": 7.733, + "maxMs": 8.622 + } + } + }, + "picker": { + "webView": { + "projects-fetch": { + "runs": 10, + "medianMs": 598.4, + "p90Ms": 608.9, + "minMs": 550.1, + "maxMs": 625.1 + }, + "projects-fetch:transport": { + "runs": 10, + "medianMs": 14.627, + "p90Ms": 15.071, + "minMs": 13.792, + "maxMs": 15.18 + } + }, + "host": { + "projects-fetch": { + "runs": 10, + "medianMs": 583.54, + "p90Ms": 595.075, + "minMs": 535.896, + "maxMs": 609.92 + }, + "projects-fetch:read": { + "runs": 10, + "medianMs": 80.039, + "p90Ms": 85.789, + "minMs": 78.568, + "maxMs": 114.331 + }, + "projects-fetch:parse": { + "runs": 10, + "medianMs": 487.084, + "p90Ms": 501.707, + "minMs": 403.956, + "maxMs": 506.636 + }, + "projects-fetch:serialize": { + "runs": 10, + "medianMs": 13.936, + "p90Ms": 20.752, + "minMs": 12.222, + "maxMs": 21.845 + } + } + }, + "save": { + "webView": { + "save": { + "runs": 10, + "medianMs": 4206.9, + "p90Ms": 4231.1, + "minMs": 4144.3, + "maxMs": 4345.2 + }, + "save-bytes": { + "runs": 10, + "medianMs": 88459053, + "p90Ms": 88459053, + "minMs": 88459053, + "maxMs": 88459053 + }, + "save:transport": { + "runs": 10, + "medianMs": 2830.705, + "p90Ms": 2852.292, + "minMs": 2767.369, + "maxMs": 2920.202 + } + }, + "host": { + "save": { + "runs": 10, + "medianMs": 1377.091, + "p90Ms": 1390.095, + "minMs": 1360.282, + "maxMs": 1424.998 + }, + "save:parse": { + "runs": 10, + "medianMs": 825.484, + "p90Ms": 844.582, + "minMs": 815.209, + "maxMs": 871.645 + }, + "save:read": { + "runs": 10, + "medianMs": 78.975, + "p90Ms": 80.786, + "minMs": 76.891, + "maxMs": 83.378 + }, + "save:write": { + "runs": 10, + "medianMs": 305.969, + "p90Ms": 306.789, + "minMs": 303.208, + "maxMs": 306.792 + }, + "save:serialize": { + "runs": 10, + "medianMs": 155.447, + "p90Ms": 157.823, + "minMs": 147.258, + "maxMs": 158.026 + } + } + }, + "concordance": { + "concordance-read": { + "runs": 10, + "medianMs": 28008.2, + "p90Ms": 28749.4, + "minMs": 23573.5, + "maxMs": 29002.7 + }, + "concordance-entries": { + "runs": 10, + "medianMs": 50, + "p90Ms": 53.8, + "minMs": 47.3, + "maxMs": 54.5 + } + }, + "textRead": { + "text-read": { + "runs": 10, + "medianMs": 1449.106, + "p90Ms": 1487.539, + "minMs": 1414.228, + "maxMs": 1506.549 + }, + "text-read-tokenize": { + "runs": 10, + "medianMs": 613.77, + "p90Ms": 648.228, + "minMs": 595.196, + "maxMs": 651.966 + } + } + } + } +} diff --git a/perf/results/bench-2026-10-15T155002-10d81320.json b/perf/results/bench-2026-10-15T155002-10d81320-407-baseline.json similarity index 100% rename from perf/results/bench-2026-10-15T155002-10d81320.json rename to perf/results/bench-2026-10-15T155002-10d81320-407-baseline.json From bc338f1e6dbb94ad5054660596cad5789f9a4b81 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 20 Oct 2026 10:54:54 -0600 Subject: [PATCH 09/24] Run Prettier --- .../app-2026-10-20T164125-257ed462-439-checkpoint.json | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/perf/results/app-2026-10-20T164125-257ed462-439-checkpoint.json b/perf/results/app-2026-10-20T164125-257ed462-439-checkpoint.json index 371cdac14..73acc52b6 100644 --- a/perf/results/app-2026-10-20T164125-257ed462-439-checkpoint.json +++ b/perf/results/app-2026-10-20T164125-257ed462-439-checkpoint.json @@ -10,11 +10,7 @@ "memoryGb": 34, "os": "Windows_NT 10.0.26200", "node": "v25.8.1", - "loadAverage": [ - 0, - 0, - 0 - ] + "loadAverage": [0, 0, 0] }, "appBuild": "packaged", "extensionBuild": "production", From 96c8ab2398cba1e37ae8c0915bf47cc743ce03e7 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 20 Oct 2026 11:22:28 -0600 Subject: [PATCH 10/24] Head the bench's form-index timing for what it measures --- perf/summary/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf/summary/cli.ts b/perf/summary/cli.ts index 8be6dbfde..92e24a4b3 100644 --- a/perf/summary/cli.ts +++ b/perf/summary/cli.ts @@ -89,7 +89,7 @@ const SECTION_LABELS: Record = { 'timings.storage': 'Storage', 'timings.edit': 'Edit one gloss', 'timings.concordance': 'Concordance', - 'timings.text': 'Whole-text read', + 'timings.text': 'Form-in-text index', }; /** Names for measured values, by their key; an unlisted value shows its key. */ From f1381eb64835e3f8f19c778b0f716df72721fd59 Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 20 Oct 2026 11:22:28 -0600 Subject: [PATCH 11/24] List a text's books again after reporting they could not be listed --- src/__tests__/services/bookService.test.ts | 14 ++++++++++++++ src/services/bookService.ts | 9 +++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index 83dfaf56f..c21346b3d 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -1152,6 +1152,20 @@ describe('getTextReadStatus', () => { expect(project.subscriptions).toHaveLength(0); }); + it('lists the books again on the request after reporting they could not be', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = new Error('offline'); + readStatus(); + await settle(); + readStatus(); + + project.booksPresent = '1'; + readStatus(); + await settle(); + + expect(readStatus()).toEqual({ state: 'read', booksRead: 1, bookCount: 1, failedBookIds: [] }); + }); + it('clears a failed book once an edit lets it read', async () => { let delivery: Delivery = platformError('busy'); const project = serveProject(() => delivery); diff --git a/src/services/bookService.ts b/src/services/bookService.ts index 585b612fb..d8ba3cefc 100644 --- a/src/services/bookService.ts +++ b/src/services/bookService.ts @@ -72,7 +72,10 @@ interface TextRead { watches: Map Promise>; } -/** Each source project's whole-text read, from the first status asked for until the project closes. */ +/** + * Each source project's whole-text read, from the first status asked for until the project closes + * or reports its books unlisted. + */ const textReads = new Map(); /** Receives the source project and book of each edit to a watched book's text. */ @@ -429,7 +432,8 @@ async function readText(sourceProjectId: string, read: TextRead): Promise /** * Reports how far reading the source project's whole text has got, starting the read when the - * project has none. The read keeps a form index that {@link isFormInText} answers from, and watches + * project has none. A read whose books could not be listed is reported once, then started again on + * the next request. The read keeps a form index that {@link isFormInText} answers from, and watches * every book it has read for edits, which {@link onBookTextChanged} reports, until the project * closes. * @@ -451,6 +455,7 @@ export function getTextReadStatus(sourceProjectId: string): string { perfTrack('readText', readText(sourceProjectId, read)); } const { state, bookIds, readBookIds, failedBookIds } = read; + if (state === 'unlisted') textReads.delete(sourceProjectId); const status: TextReadStatus = { state, booksRead: readBookIds.size, From 9d953e2dd2e6a43eedc23e4ec7073d30f9f445aa Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 20 Oct 2026 11:48:17 -0600 Subject: [PATCH 12/24] Report an edit to the open book once, whichever watch hears it first --- src/__tests__/services/bookService.test.ts | 85 ++++++++++++++++++++++ src/services/bookService.ts | 43 ++++++----- 2 files changed, 108 insertions(+), 20 deletions(-) diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index c21346b3d..691df51a4 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -941,9 +941,52 @@ describe('onBookTextChanged', () => { await settle(); project.subscriptions[0].deliver(usj('GEN', 'v2')); + await finishReading(project); expect(listener).toHaveBeenCalledWith('src', 'GEN'); + }); + + it('reports an edit to the outlined book once when the whole-text read first reads the text from before it', async () => { + const project = serveProject(); + project.booksPresent = '1'; + const listener = listen(); + const outline = getBookOutline('src', 'GEN'); + await settle(); + const [open] = project.subscriptions; + open.deliver(usj('GEN')); + await outline; + readStatus(); + await settle(); + const [, read] = subscriptionsTo(project, 'GEN'); + + open.deliver(usj('GEN', 'v2')); + read.deliver(usj('GEN')); + await settle(); + read.deliver(usj('GEN', 'v2')); await finishReading(project); + + expect(listener).toHaveBeenCalledTimes(1); + }); + + it('reports an edit to the outlined book the whole-text read first read with the edit in it', async () => { + const project = serveProject(); + project.booksPresent = '1'; + const listener = listen(); + const outline = getBookOutline('src', 'GEN'); + await settle(); + const [open] = project.subscriptions; + open.deliver(usj('GEN')); + await outline; + readStatus(); + await settle(); + subscriptionsTo(project, 'GEN')[1].deliver(usj('GEN', 'v2')); + await settle(); + + open.deliver(usj('GEN', 'v2')); + await settle(); + await finishReading(project); + + expect(listener).toHaveBeenCalledTimes(1); }); it('reports an edit to the outlined book the whole-text read could not watch', async () => { @@ -957,6 +1000,7 @@ describe('onBookTextChanged', () => { project.subscribeError = undefined; project.subscriptions[0].deliver(usj('GEN', 'v2')); + await settle(); expect(listener).toHaveBeenCalledTimes(1); expect(listener).toHaveBeenCalledWith('src', 'GEN'); @@ -1180,6 +1224,22 @@ describe('getTextReadStatus', () => { expect(readStatus().failedBookIds).toEqual([]); }); + it('reads a book it could not watch again once the outlined book is edited', async () => { + segmentStandIns(); + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + await getBookOutline('src', 'GEN'); + project.subscribeError = new Error('no provider'); + readStatus(); + await settle(); + project.subscribeError = undefined; + + project.subscriptions[0].deliver(usj('GEN', 'v2')); + await settle(); + + expect(readStatus().failedBookIds).toEqual([]); + }); + it('stops watching the books once the project closes', async () => { const project = serveProject((bookId) => usj(bookId)); project.booksPresent = '11'; @@ -1316,6 +1376,31 @@ describe('isFormInText', () => { expect(isFormInText('src', 'gen-v2')).toBe(true); }); + it('answers from the edited text when a read it replaced delivers older text last', async () => { + segmentStandIns(); + tokenizeToWords(); + const project = serveProject(); + project.booksPresent = '1'; + const outline = getBookOutline('src', 'GEN'); + await settle(); + const [open] = project.subscriptions; + open.deliver(usj('GEN')); + await outline; + readStatus(); + await settle(); + const [, replaced] = subscriptionsTo(project, 'GEN'); + open.deliver(usj('GEN', 'v2')); + await settle(); + + subscriptionsTo(project, 'GEN').at(-1)?.deliver(usj('GEN', 'v2')); + await settle(); + replaced.deliver(usj('GEN')); + await finishReading(project); + + expect(isFormInText('src', 'gen-v1')).toBe(false); + expect(isFormInText('src', 'gen-v2')).toBe(true); + }); + it('answers nothing once a project closes', async () => { const project = serveProject((bookId) => usj(bookId)); project.booksPresent = '1'; diff --git a/src/services/bookService.ts b/src/services/bookService.ts index d8ba3cefc..6961e3632 100644 --- a/src/services/bookService.ts +++ b/src/services/bookService.ts @@ -17,8 +17,6 @@ import { reanchorSegmentation } from '../utils/segmentation'; interface BookTextWatch { /** The text as first delivered; `undefined` when the project has no text for the book. */ text: Promise; - /** `false` once subscribing to the book's text has failed. */ - isWatching: () => boolean; stop: () => Promise; } @@ -68,7 +66,7 @@ interface TextRead { /** Books whose latest read failed. */ failedBookIds: Set; forms: FormIndex; - /** Stops watching each book, by book id; a book that could not be subscribed to has none. */ + /** Stops watching each book the read has started, by book id. */ watches: Map Promise>; } @@ -109,7 +107,6 @@ function watchBookText( reject: rejectText, } = Promise.withResolvers(); let isFirstDelivery = true; - let isSubscribed = true; const subscription = papi.projectDataProviders .get('platformScripture.USJ_Book', sourceProjectId) .then((pdp) => @@ -125,14 +122,10 @@ function watchBookText( else resolveText(usj); }), ); - subscription.catch((e) => { - isSubscribed = false; - rejectText(e); - }); + subscription.catch(rejectText); return { text, - isWatching: () => isSubscribed, stop: async () => { // Unsubscribing ahead of the first delivery would leave `text` unsettled. await text.catch(() => undefined); @@ -167,11 +160,11 @@ function holdBook(sourceProjectId: string, bookId: string, writingSystem: string // Read again at once, so the book stays watched and a refetch the listeners start reads the // edit. openBooks.set(sourceProjectId, holdBook(sourceProjectId, bookId, writingSystem)); - // A whole-text read that has read the book and still watches it reports the edit once it has - // read it again. + // The whole-text read's watch may have first read the book before the edit or after it, so the + // read reads the book again over a new watch and reports the edit itself. const read = textReads.get(sourceProjectId); - if (!read?.readBookIds.has(bookId) || !read.watches.has(bookId)) - reportEdit(sourceProjectId, bookId); + if (read?.watches.has(bookId)) rereadTextBook(sourceProjectId, read, bookId); + else reportEdit(sourceProjectId, bookId); }); const book = perfTrack('getTokenizedBook:usj', watch.text).then( (usj) => @@ -352,9 +345,9 @@ function rereadHeldBook(sourceProjectId: string, bookId: string): void { } /** - * Reads `bookId` of `read`'s text into its form index, and watches the book until the read ends. An - * edit to the book lets go of its kept copies, reads it again, and is then reported unless the read - * has ended. Resolves once the book's forms are filed or its read has failed. + * Reads `bookId` of `read`'s text into its form index, and watches the book until the read ends or + * reads it again. An edit to the book lets go of its kept copies, and is reported once the book is + * read again. Resolves once the book's text has arrived and the book is filed as read or failed. */ async function readTextBook( sourceProjectId: string, @@ -362,11 +355,10 @@ async function readTextBook( bookId: string, ): Promise { read.watches.get(bookId)?.(); - const { text, isWatching, stop } = watchBookText(sourceProjectId, bookId, async () => { + const { text, stop } = watchBookText(sourceProjectId, bookId, () => { if (textReads.get(sourceProjectId) !== read || read.watches.get(bookId) !== stop) return; rereadHeldBook(sourceProjectId, bookId); - await readTextBook(sourceProjectId, read, bookId); - if (textReads.get(sourceProjectId) === read) reportEdit(sourceProjectId, bookId); + rereadTextBook(sourceProjectId, read, bookId); }); read.watches.set(bookId, stop); const forms = await perfTrack('readText:usj', text) @@ -382,16 +374,27 @@ async function readTextBook( return undefined; }); if (textReads.get(sourceProjectId) !== read) return; + // A later read of the book reads newer text, so this one files only a book none has filed. + if (read.watches.get(bookId) !== stop && read.readBookIds.has(bookId)) return; if (forms) { read.forms.setBook(bookId, forms); read.failedBookIds.delete(bookId); } else { read.failedBookIds.add(bookId); - if (!isWatching()) read.watches.delete(bookId); } read.readBookIds.add(bookId); } +/** Reads `bookId` of `read`'s text again, then reports its edit unless the read has ended. */ +async function rereadTextBook( + sourceProjectId: string, + read: TextRead, + bookId: string, +): Promise { + await readTextBook(sourceProjectId, read, bookId); + if (textReads.get(sourceProjectId) === read) reportEdit(sourceProjectId, bookId); +} + /** Lists the source project's books in canonical order, with the tag their text is written in. */ async function listBooks( sourceProjectId: string, From 62278100f42c1384a120e878158793e6101aacfc Mon Sep 17 00:00:00 2001 From: Alex Rawlings Date: Tue, 20 Oct 2026 12:13:26 -0600 Subject: [PATCH 13/24] Report a reread edit only once its read has filed the book --- src/__tests__/services/bookService.test.ts | 23 ++++++++++++++++++++++ src/services/bookService.ts | 19 +++++++++++------- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index 691df51a4..fb1a8e4c2 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -915,6 +915,29 @@ describe('onBookTextChanged', () => { expect(answers).toEqual([true]); }); + it("reports an edit once the whole-text read's answers include it when a later edit replaces its read", async () => { + tokenizeToWords(); + const project = serveProject(); + project.booksPresent = '1'; + readStatus(); + await settle(); + subscriptionsTo(project, 'GEN')[0].deliver(usj('GEN')); + await settle(); + const answers: unknown[] = []; + listen(() => answers.push(isFormInText('src', 'gen-v1'))); + + subscriptionsTo(project, 'GEN')[0].deliver(usj('GEN', 'v2')); + await settle(); + const replaced = subscriptionsTo(project, 'GEN')[1]; + replaced.deliver(usj('GEN', 'v2')); + replaced.deliver(usj('GEN', 'v3')); + await settle(); + subscriptionsTo(project, 'GEN')[2].deliver(usj('GEN', 'v3')); + await settle(); + + expect(answers).toEqual([false]); + }); + it('reports an edit to the outlined book once when the whole-text read has read it too', async () => { const project = serveProject((bookId) => usj(bookId)); project.booksPresent = '1'; diff --git a/src/services/bookService.ts b/src/services/bookService.ts index 6961e3632..3c8817e5d 100644 --- a/src/services/bookService.ts +++ b/src/services/bookService.ts @@ -347,13 +347,15 @@ function rereadHeldBook(sourceProjectId: string, bookId: string): void { /** * Reads `bookId` of `read`'s text into its form index, and watches the book until the read ends or * reads it again. An edit to the book lets go of its kept copies, and is reported once the book is - * read again. Resolves once the book's text has arrived and the book is filed as read or failed. + * read again. + * + * @returns Once the book's text has arrived, whether this read filed the book as read or failed. */ async function readTextBook( sourceProjectId: string, read: TextRead, bookId: string, -): Promise { +): Promise { read.watches.get(bookId)?.(); const { text, stop } = watchBookText(sourceProjectId, bookId, () => { if (textReads.get(sourceProjectId) !== read || read.watches.get(bookId) !== stop) return; @@ -373,9 +375,9 @@ async function readTextBook( logger.warn(`Could not read ${bookId} of project ${sourceProjectId}: ${e}`); return undefined; }); - if (textReads.get(sourceProjectId) !== read) return; + if (textReads.get(sourceProjectId) !== read) return false; // A later read of the book reads newer text, so this one files only a book none has filed. - if (read.watches.get(bookId) !== stop && read.readBookIds.has(bookId)) return; + if (read.watches.get(bookId) !== stop && read.readBookIds.has(bookId)) return false; if (forms) { read.forms.setBook(bookId, forms); read.failedBookIds.delete(bookId); @@ -383,16 +385,19 @@ async function readTextBook( read.failedBookIds.add(bookId); } read.readBookIds.add(bookId); + return true; } -/** Reads `bookId` of `read`'s text again, then reports its edit unless the read has ended. */ +/** + * Reads `bookId` of `read`'s text again, then reports its edit unless the read has ended or a later + * read of the book will report it. + */ async function rereadTextBook( sourceProjectId: string, read: TextRead, bookId: string, ): Promise { - await readTextBook(sourceProjectId, read, bookId); - if (textReads.get(sourceProjectId) === read) reportEdit(sourceProjectId, bookId); + if (await readTextBook(sourceProjectId, read, bookId)) reportEdit(sourceProjectId, bookId); } /** Lists the source project's books in canonical order, with the tag their text is written in. */ From 5de4622664a4fe438e84ed5494d474f645cbb0c1 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 20 Oct 2026 15:13:06 -0400 Subject: [PATCH 14/24] Reword comments that read one way and mean another Co-Authored-By: Claude Opus 5.5 --- e2e-tests/tests/perf/app-perf.spec.ts | 4 ++-- perf/bench/cli.ts | 2 +- src/services/bookService.ts | 20 ++++++++++---------- src/types/interlinearizer.d.ts | 2 +- src/utils/books-present.ts | 2 +- src/utils/form-index.ts | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/e2e-tests/tests/perf/app-perf.spec.ts b/e2e-tests/tests/perf/app-perf.spec.ts index 83421c8eb..4d45b0b2f 100644 --- a/e2e-tests/tests/perf/app-perf.spec.ts +++ b/e2e-tests/tests/perf/app-perf.spec.ts @@ -460,8 +460,8 @@ async function measureTier( } result.concordance = concordance.summary(); - // Whole-text read in the host, in a fresh view each run since the host keeps the read until the - // view closes. + // Whole-text read in the host, in a fresh view each run, since the host keeps the read until + // the view closes. const textRead = new Samples(); for (let run = 0; run < RUNS; run += 1) { const isLast = run === RUNS - 1; diff --git a/perf/bench/cli.ts b/perf/bench/cli.ts index a74d42e47..ff22d26a8 100644 --- a/perf/bench/cli.ts +++ b/perf/bench/cli.ts @@ -159,7 +159,7 @@ function benchTier(tier: TierSpec, allBooks: () => Book[]): Record [...indexBook(book).occurrencesByForm.keys()])); const catalogForms = selectCatalogRows( diff --git a/src/services/bookService.ts b/src/services/bookService.ts index 3c8817e5d..063657210 100644 --- a/src/services/bookService.ts +++ b/src/services/bookService.ts @@ -45,7 +45,7 @@ const openBooks = new Map(); /** Tells a request whether its project closed while it was under way. */ const closeCounts = new Map(); -/** Tells an outline request whether a later one has started, which picks the open book instead. */ +/** Tells an outline request whether a later one, which picks the open book instead, has started. */ const outlineCounts = new Map(); /** @@ -71,8 +71,8 @@ interface TextRead { } /** - * Each source project's whole-text read, from the first status asked for until the project closes - * or reports its books unlisted. + * Each source project's whole-text read, from the first status asked for until the project closes, + * or until the read reports its books unlisted. */ const textReads = new Map(); @@ -157,11 +157,11 @@ function holdBook(sourceProjectId: string, bookId: string, writingSystem: string const isOpen = openBooks.get(sourceProjectId)?.watch === watch; releaseBook(sourceProjectId, watch); if (!isOpen) return; - // Read again at once, so the book stays watched and a refetch the listeners start reads the - // edit. + // Read again at once, so the book stays watched and a refetch started by the listeners sees + // the edit. openBooks.set(sourceProjectId, holdBook(sourceProjectId, bookId, writingSystem)); // The whole-text read's watch may have first read the book before the edit or after it, so the - // read reads the book again over a new watch and reports the edit itself. + // read itself reads the book again over a new watch and reports the edit. const read = textReads.get(sourceProjectId); if (read?.watches.has(bookId)) rereadTextBook(sourceProjectId, read, bookId); else reportEdit(sourceProjectId, bookId); @@ -269,10 +269,10 @@ function segmentsOf( } /** - * Calls `listener` whenever the Scripture text of a book the service watches changes, until the + * Calls `listener` whenever the Scripture text changes in a book the service watches, until the * returned function is called. A source project's watched books are its open book, the one whose - * outline was last asked for, and every book its whole-text read has read. A book requested once - * the listener is called reads the edit, and the whole-text read's answers include it. + * outline was last asked for, and every book its whole-text read has read. Once the listener is + * called, a request for the book reads the edit, and the whole-text read's answers include it. */ export function onBookTextChanged(listener: BookTextListener): () => boolean { bookTextListeners.add(listener); @@ -442,7 +442,7 @@ async function readText(sourceProjectId: string, read: TextRead): Promise * Reports how far reading the source project's whole text has got, starting the read when the * project has none. A read whose books could not be listed is reported once, then started again on * the next request. The read keeps a form index that {@link isFormInText} answers from, and watches - * every book it has read for edits, which {@link onBookTextChanged} reports, until the project + * each book it has read, reporting its edits through {@link onBookTextChanged}, until the project * closes. * * @returns The `TextReadStatus` as JSON. diff --git a/src/types/interlinearizer.d.ts b/src/types/interlinearizer.d.ts index 4f5ab6880..1461c5e03 100644 --- a/src/types/interlinearizer.d.ts +++ b/src/types/interlinearizer.d.ts @@ -11,7 +11,7 @@ declare module 'papi-shared-types' { */ 'interlinearizer.onUndoFromNotification': { notificationId: string | number }; /** - * Emitted when the text of a book the extension watches changes, so views showing it refetch + * Emitted when the text changes in a book the extension watches, so views showing it refetch * it: a source project's open book (the one last outlined), and every book its whole-text read * has read. Edits to other books emit nothing. */ diff --git a/src/utils/books-present.ts b/src/utils/books-present.ts index e2c712ef1..dca3a92c8 100644 --- a/src/utils/books-present.ts +++ b/src/utils/books-present.ts @@ -1,7 +1,7 @@ import { Canon } from '@sillsdev/scripture'; /** - * Reads the books a `platformScripture.booksPresent` flag string marks present — one `'1'` per + * Reads the books that a `platformScripture.booksPresent` flag string marks present — one `'1'` per * present book, indexed by canonical book number — in canonical order. Flags past the end of the * canon mark nothing. */ diff --git a/src/utils/form-index.ts b/src/utils/form-index.ts index dff721675..99c6ded6c 100644 --- a/src/utils/form-index.ts +++ b/src/utils/form-index.ts @@ -13,7 +13,7 @@ export function formsOfBook(book: Book): Set { return forms; } -/** The word forms a text holds, filed book by book so that one book's can be replaced alone. */ +/** The word forms a text holds, filed book by book so one book's forms can be replaced alone. */ export class FormIndex { private readonly formsByBook = new Map>(); From 5675828d1bb08ba2409ce280b954654323542d1f Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 20 Oct 2026 15:29:59 -0400 Subject: [PATCH 15/24] Sort imports --- e2e-tests/tests/perf/app-perf.spec.ts | 2 +- src/services/bookService.ts | 4 ++-- src/utils/books-present.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/e2e-tests/tests/perf/app-perf.spec.ts b/e2e-tests/tests/perf/app-perf.spec.ts index 4d45b0b2f..db4d9a7e9 100644 --- a/e2e-tests/tests/perf/app-perf.spec.ts +++ b/e2e-tests/tests/perf/app-perf.spec.ts @@ -5,8 +5,8 @@ import { readDraftJson, readManifest } from '../../../perf/dataset/store'; import { TIERS, type TierSpec } from '../../../perf/dataset/tiers'; import { provenance, writeResults } from '../../../perf/results'; import { addSavedProject, readDraft, writeDraft } from '../../../perf/storage'; -import type { TextReadStatus } from '../../../src/types/text-read-status'; import { readCaptureMeta } from '../../../perf/usj-cache'; +import type { TextReadStatus } from '../../../src/types/text-read-status'; import { expect, test } from '../../fixtures/cdp.fixture'; import { waitForAppAndInterlinearizerReady } from '../../fixtures/helpers'; import { diff --git a/src/services/bookService.ts b/src/services/bookService.ts index 063657210..8912d403a 100644 --- a/src/services/bookService.ts +++ b/src/services/bookService.ts @@ -4,12 +4,12 @@ import { tokenizeBook } from 'parsers/papi/bookTokenizer'; import { resegmentBook } from 'parsers/papi/resegmentBook'; import { extractBookFromUsj, type UsjDocument } from 'parsers/papi/usjBookExtractor'; import { isPlatformError } from 'platform-bible-utils'; -import { isSegmentationDelta } from '../types/type-guards'; import type { SegmentPage } from '../types/segment-pages'; import type { TextReadStatus } from '../types/text-read-status'; +import { isSegmentationDelta } from '../types/type-guards'; +import { normalizeSurfaceForm } from '../utils/analysis-identity'; import { buildBookOutline } from '../utils/book-outline'; import { presentBookIds } from '../utils/books-present'; -import { normalizeSurfaceForm } from '../utils/analysis-identity'; import { FormIndex, formsOfBook } from '../utils/form-index'; import { perfTime, perfTrack } from '../utils/perf-marks'; import { reanchorSegmentation } from '../utils/segmentation'; diff --git a/src/utils/books-present.ts b/src/utils/books-present.ts index dca3a92c8..fb9e35c15 100644 --- a/src/utils/books-present.ts +++ b/src/utils/books-present.ts @@ -1,8 +1,8 @@ import { Canon } from '@sillsdev/scripture'; /** - * Reads the books that a `platformScripture.booksPresent` flag string marks present — one `'1'` per - * present book, indexed by canonical book number — in canonical order. Flags past the end of the + * Reads the books that a `platformScripture.booksPresent` flag string marks present (one `'1'` per + * present book, indexed by canonical book number), in canonical order. Flags past the end of the * canon mark nothing. */ export function presentBookIds(booksPresent: string): string[] { From 0049419b03f0994bb364b1d0bb88da91741d5766 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 20 Oct 2026 15:46:46 -0400 Subject: [PATCH 16/24] Drop redundant tests and assertions on stand-ins Co-Authored-By: Claude Opus 5.5 --- src/__tests__/main.test.ts | 12 ++++-- src/__tests__/services/bookService.test.ts | 44 ++++++---------------- src/__tests__/utils/form-index.test.ts | 2 +- 3 files changed, 21 insertions(+), 37 deletions(-) diff --git a/src/__tests__/main.test.ts b/src/__tests__/main.test.ts index 6b386ceb1..4ea2438d3 100644 --- a/src/__tests__/main.test.ts +++ b/src/__tests__/main.test.ts @@ -1489,15 +1489,19 @@ describe('main', () => { }); describe('interlinearizer.getTextReadStatus command', () => { - it('returns the status the service reports for the project', async () => { - jest.mocked(bookService.getTextReadStatus).mockReturnValue('{"state":"reading"}'); - const handler = await activateAndGetHandler<(sourceProjectId: string) => Promise>( + const mockGetTextReadStatus = jest.mocked(bookService.getTextReadStatus); + const getHandler = () => + activateAndGetHandler<(sourceProjectId: string) => Promise>( 'interlinearizer.getTextReadStatus', ); + it('returns the status the service reports for the project', async () => { + mockGetTextReadStatus.mockReturnValue('{"state":"reading"}'); + const handler = await getHandler(); + const result = await handler('src-project'); - expect(bookService.getTextReadStatus).toHaveBeenCalledWith('src-project'); + expect(mockGetTextReadStatus).toHaveBeenCalledWith('src-project'); expect(result).toBe('{"state":"reading"}'); }); }); diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index fb1a8e4c2..5de371b65 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -648,21 +648,9 @@ describe('onBookTextChanged', () => { await getTokenizedBook('src', 'GEN'); await getBookOutline('src', 'GEN'); - - expect(tokenizeBook).not.toHaveBeenCalled(); - expect(subscriptionsTo(project, 'GEN')).toHaveLength(1); - expect(subscriptionsTo(project, 'GEN')[0].unsubscribe).not.toHaveBeenCalled(); - }); - - it('serves pages of the outlined book after another book was read whole', async () => { - const project = serveProject((bookId) => usj(bookId)); - await getBookOutline('src', 'GEN'); - await getTokenizedBook('src', 'EXO'); - jest.mocked(tokenizeBook).mockClear(); - const page = parsePage(await getSegments('src', 'GEN', 0, 2)); - expect(page.segments.map((seg) => seg.id)).toEqual(['GEN 1:1', 'GEN 1:3']); + expect(page.segments).toHaveLength(2); expect(tokenizeBook).not.toHaveBeenCalled(); expect(subscriptionsTo(project, 'GEN')).toHaveLength(1); }); @@ -768,7 +756,16 @@ describe('onBookTextChanged', () => { expect(exo.unsubscribe).not.toHaveBeenCalled(); }); - it('stops watching an outlined book that fails to read, and watches it again on the next outline', async () => { + it('stops watching an outlined book that fails to read', async () => { + const project = serveProject(() => platformError('busy')); + + await expect(getBookOutline('src', 'GEN')).rejects.toThrow('busy'); + await settle(); + + expect(project.subscriptions[0].unsubscribe).toHaveBeenCalled(); + }); + + it('watches an outlined book again on the next outline once it fails to read', async () => { let delivery: Delivery = platformError('busy'); const project = serveProject(() => delivery); const listener = listen(); @@ -777,10 +774,8 @@ describe('onBookTextChanged', () => { delivery = usj('GEN'); await getBookOutline('src', 'GEN'); - const [failed, watch] = subscriptionsTo(project, 'GEN'); - watch.deliver(usj('GEN', 'v2')); + subscriptionsTo(project, 'GEN')[1].deliver(usj('GEN', 'v2')); - expect(failed.unsubscribe).toHaveBeenCalled(); expect(listener).toHaveBeenCalledTimes(1); expect(listener).toHaveBeenCalledWith('src', 'GEN'); }); @@ -1384,21 +1379,6 @@ describe('isFormInText', () => { expect(isFormInText('src', 'gen-v1')).toBeUndefined(); }); - it("answers from a book's edited text", async () => { - let version = 'v1'; - const project = serveProject((bookId) => usj(bookId, version)); - project.booksPresent = '1'; - readStatus(); - await settle(); - - version = 'v2'; - project.subscriptions[0].deliver(usj('GEN', version)); - await settle(); - - expect(isFormInText('src', 'gen-v1')).toBe(false); - expect(isFormInText('src', 'gen-v2')).toBe(true); - }); - it('answers from the edited text when a read it replaced delivers older text last', async () => { segmentStandIns(); tokenizeToWords(); diff --git a/src/__tests__/utils/form-index.test.ts b/src/__tests__/utils/form-index.test.ts index 733979c70..0c68e2c0f 100644 --- a/src/__tests__/utils/form-index.test.ts +++ b/src/__tests__/utils/form-index.test.ts @@ -26,7 +26,7 @@ const BOOK: Book = { describe('formsOfBook', () => { it("holds each of the book's word forms once, normalized", () => { - expect([...formsOfBook(BOOK)]).toEqual(['in', 'the', 'beginning', 'earth']); + expect(formsOfBook(BOOK)).toEqual(new Set(['in', 'the', 'beginning', 'earth'])); }); it('holds the forms the concordance files occurrences under', () => { From 488ad4ec2b28db9ada52f2980b10e1e3b9af436d Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 20 Oct 2026 17:53:20 -0400 Subject: [PATCH 17/24] Test missing forms on their own and build the form-index book with makeVerseBook Co-Authored-By: Claude Opus 5.5 --- src/__tests__/services/bookService.test.ts | 12 +++++-- src/__tests__/utils/form-index.test.ts | 42 ++++++++-------------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index 5de371b65..05fd42689 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -1351,14 +1351,22 @@ describe('isFormInText', () => { readStatus(); await settle(); - expect(['shared', 'gen-v1', 'exo-v1', 'lev-v1'].map((f) => isFormInText('src', f))).toEqual([ + expect(['shared', 'gen-v1', 'exo-v1'].map((f) => isFormInText('src', f))).toEqual([ true, true, true, - false, ]); }); + it('misses a form no book of the text holds', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '11'; + readStatus(); + await settle(); + + expect(isFormInText('src', 'lev-v1')).toBe(false); + }); + it('matches a form however it is cased', async () => { const project = serveProject((bookId) => usj(bookId)); project.booksPresent = '1'; diff --git a/src/__tests__/utils/form-index.test.ts b/src/__tests__/utils/form-index.test.ts index 0c68e2c0f..8f2fdc264 100644 --- a/src/__tests__/utils/form-index.test.ts +++ b/src/__tests__/utils/form-index.test.ts @@ -1,28 +1,13 @@ /// -import type { Book } from 'interlinearizer'; import { indexBook } from '../../utils/concordance'; import { FormIndex, formsOfBook } from '../../utils/form-index'; -import { makePunctToken, makeSegment, makeWordToken } from '../test-helpers'; - -const BOOK: Book = { - id: 'GEN', - bookRef: 'GEN', - textVersion: 'v1@en', - duplicateVerseIds: [], - segments: [ - makeSegment('GEN 1:1', 'In the beginning.', [ - makeWordToken('GEN 1:1:0', 'In'), - makeWordToken('GEN 1:1:3', 'the'), - makeWordToken('GEN 1:1:7', 'beginning'), - makePunctToken('GEN 1:1:16'), - ]), - makeSegment('GEN 1:2', 'The earth', [ - makeWordToken('GEN 1:2:0', 'The'), - makeWordToken('GEN 1:2:4', 'earth'), - ]), - ], -}; +import { makeVerseBook } from '../test-helpers'; + +const BOOK = makeVerseBook([ + { sid: 'GEN 1:1', text: 'In the beginning.' }, + { sid: 'GEN 1:2', text: 'The earth' }, +]); describe('formsOfBook', () => { it("holds each of the book's word forms once, normalized", () => { @@ -41,12 +26,15 @@ describe('FormIndex', () => { index.setBook('GEN', new Set(['in', 'the'])); index.setBook('EXO', new Set(['now'])); - expect(['in', 'the', 'now', 'earth'].map((form) => index.has(form))).toEqual([ - true, - true, - true, - false, - ]); + expect(['in', 'the', 'now'].map((form) => index.has(form))).toEqual([true, true, true]); + }); + + it('misses a form no book filed holds', () => { + const index = new FormIndex(); + + index.setBook('GEN', new Set(['in', 'the'])); + + expect(index.has('earth')).toBe(false); }); it("drops forms only a refiled book's earlier forms held", () => { From fee4a107d64d50786395e378a45d66e781a2807c Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 21 Oct 2026 09:13:43 -0400 Subject: [PATCH 18/24] Keep the word stand-in's offsets true and say FormIndex.has takes a normalized form Co-Authored-By: Claude Opus 5.5 --- src/__tests__/services/bookService.test.ts | 4 ++-- src/utils/form-index.ts | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index 05fd42689..b16eb929a 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -158,9 +158,9 @@ function tokenizeToWords() { textVersion: `${raw.contentHash}@${raw.writingSystem}`, duplicateVerseIds: [], segments: [ - makeSegment(`${raw.bookCode} 1:1`, 'word', [ + makeSegment(`${raw.bookCode} 1:1`, `shared ${raw.bookCode}-${raw.contentHash}`, [ makeWordToken(`${raw.bookCode} 1:1:0`, 'shared'), - makeWordToken(`${raw.bookCode} 1:1:7`, `${raw.bookCode}-${raw.contentHash}`), + makeWordToken(`${raw.bookCode} 1:1:7`, `${raw.bookCode}-${raw.contentHash}`, 7), ]), ], })); diff --git a/src/utils/form-index.ts b/src/utils/form-index.ts index 99c6ded6c..faf3a85c4 100644 --- a/src/utils/form-index.ts +++ b/src/utils/form-index.ts @@ -32,7 +32,10 @@ export class FormIndex { forms.forEach((form) => this.bookCounts.set(form, (this.bookCounts.get(form) ?? 0) + 1)); } - /** Whether a filed book holds `form`, normalized as {@link formsOfBook} normalizes forms. */ + /** + * Whether a filed book holds `form`, which must already be normalized as {@link formsOfBook} + * normalizes forms. + */ has(form: string): boolean { return this.bookCounts.has(form); } From 9a54a8e21272b0eb94d453a53f39bda4cc05aa32 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 21 Oct 2026 13:47:43 -0400 Subject: [PATCH 19/24] Read a book the whole-text read could not subscribe to again on the next status request A rejected subscription left the book failed with no way to hear an edit, so isFormInText answered nothing until the view closed. Co-Authored-By: Claude Opus 5.5 --- src/__tests__/services/bookService.test.ts | 44 ++++++++++++++++++++++ src/services/bookService.ts | 22 +++++++++-- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index b16eb929a..7a3c004a1 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -1258,6 +1258,50 @@ describe('getTextReadStatus', () => { expect(readStatus().failedBookIds).toEqual([]); }); + it('reads a book it could not watch again on the next request', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + project.subscribeError = new Error('no provider'); + readStatus(); + await settle(); + project.subscribeError = undefined; + + expect(readStatus().failedBookIds).toEqual(['GEN']); + await settle(); + + expect(readStatus().failedBookIds).toEqual([]); + expect(project.subscriptions).toHaveLength(1); + }); + + it('reads a book it still cannot watch again on each request', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + project.subscribeError = new Error('no provider'); + readStatus(); + await settle(); + readStatus(); + await settle(); + project.subscribeError = undefined; + + readStatus(); + await settle(); + + expect(readStatus().failedBookIds).toEqual([]); + }); + + it('does not read a failed book it watches again until an edit is heard', async () => { + const project = serveProject(() => platformError('busy')); + project.booksPresent = '1'; + readStatus(); + await settle(); + + readStatus(); + await settle(); + + expect(readStatus().failedBookIds).toEqual(['GEN']); + expect(project.subscriptions).toHaveLength(1); + }); + it('stops watching the books once the project closes', async () => { const project = serveProject((bookId) => usj(bookId)); project.booksPresent = '11'; diff --git a/src/services/bookService.ts b/src/services/bookService.ts index 8912d403a..4670116f5 100644 --- a/src/services/bookService.ts +++ b/src/services/bookService.ts @@ -17,6 +17,8 @@ import { reanchorSegmentation } from '../utils/segmentation'; interface BookTextWatch { /** The text as first delivered; `undefined` when the project has no text for the book. */ text: Promise; + /** Whether subscribing to the book's USJ succeeded, so that edits to it can be heard. */ + isSubscribed: Promise; stop: () => Promise; } @@ -65,6 +67,8 @@ interface TextRead { readBookIds: Set; /** Books whose latest read failed. */ failedBookIds: Set; + /** Failed books with no subscription to hear an edit through, read again on the next status. */ + unsubscribedBookIds: Set; forms: FormIndex; /** Stops watching each book the read has started, by book id. */ watches: Map Promise>; @@ -126,6 +130,10 @@ function watchBookText( return { text, + isSubscribed: subscription.then( + () => true, + () => false, + ), stop: async () => { // Unsubscribing ahead of the first delivery would leave `text` unsettled. await text.catch(() => undefined); @@ -357,13 +365,13 @@ async function readTextBook( bookId: string, ): Promise { read.watches.get(bookId)?.(); - const { text, stop } = watchBookText(sourceProjectId, bookId, () => { + const { text, isSubscribed, stop } = watchBookText(sourceProjectId, bookId, () => { if (textReads.get(sourceProjectId) !== read || read.watches.get(bookId) !== stop) return; rereadHeldBook(sourceProjectId, bookId); rereadTextBook(sourceProjectId, read, bookId); }); read.watches.set(bookId, stop); - const forms = await perfTrack('readText:usj', text) + const formsRead = perfTrack('readText:usj', text) .then((usj) => usj ? perfTime('readText:tokenize', () => @@ -375,6 +383,7 @@ async function readTextBook( logger.warn(`Could not read ${bookId} of project ${sourceProjectId}: ${e}`); return undefined; }); + const [forms, subscribed] = await Promise.all([formsRead, isSubscribed]); if (textReads.get(sourceProjectId) !== read) return false; // A later read of the book reads newer text, so this one files only a book none has filed. if (read.watches.get(bookId) !== stop && read.readBookIds.has(bookId)) return false; @@ -384,6 +393,8 @@ async function readTextBook( } else { read.failedBookIds.add(bookId); } + if (subscribed) read.unsubscribedBookIds.delete(bookId); + else read.unsubscribedBookIds.add(bookId); read.readBookIds.add(bookId); return true; } @@ -441,7 +452,8 @@ async function readText(sourceProjectId: string, read: TextRead): Promise /** * Reports how far reading the source project's whole text has got, starting the read when the * project has none. A read whose books could not be listed is reported once, then started again on - * the next request. The read keeps a form index that {@link isFormInText} answers from, and watches + * the next request. Each request also reads again any book that failed with no subscription to hear + * an edit through. The read keeps a form index that {@link isFormInText} answers from, and watches * each book it has read, reporting its edits through {@link onBookTextChanged}, until the project * closes. * @@ -456,12 +468,16 @@ export function getTextReadStatus(sourceProjectId: string): string { writingSystem: '', readBookIds: new Set(), failedBookIds: new Set(), + unsubscribedBookIds: new Set(), forms: new FormIndex(), watches: new Map(), }; textReads.set(sourceProjectId, read); perfTrack('readText', readText(sourceProjectId, read)); } + const unsubscribed = [...read.unsubscribedBookIds]; + read.unsubscribedBookIds.clear(); + unsubscribed.forEach((bookId) => readTextBook(sourceProjectId, read, bookId)); const { state, bookIds, readBookIds, failedBookIds } = read; if (state === 'unlisted') textReads.delete(sourceProjectId); const status: TextReadStatus = { From cf3fba97092c3c709590a6d5456260a46eac9249 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 21 Oct 2026 14:03:20 -0400 Subject: [PATCH 20/24] Read books the whole-text read could not subscribe to again a few at a time Co-Authored-By: Claude Opus 5.5 --- src/__tests__/services/bookService.test.ts | 16 ++++++++++++ src/services/bookService.ts | 30 +++++++++++++++------- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index 7a3c004a1..e8ba27952 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -1289,6 +1289,22 @@ describe('getTextReadStatus', () => { expect(readStatus().failedBookIds).toEqual([]); }); + it('reads books it could not watch again a few at a time', async () => { + const project = serveProject(); + project.booksPresent = '111111'; + project.subscribeError = new Error('no provider'); + readStatus(); + await settle(); + project.subscribeError = undefined; + + readStatus(); + await settle(); + + expect(project.subscriptions.length).toBeGreaterThan(0); + expect(project.subscriptions.length).toBeLessThan(6); + await finishReading(project); + }); + it('does not read a failed book it watches again until an edit is heard', async () => { const project = serveProject(() => platformError('busy')); project.booksPresent = '1'; diff --git a/src/services/bookService.ts b/src/services/bookService.ts index 4670116f5..ca4b64100 100644 --- a/src/services/bookService.ts +++ b/src/services/bookService.ts @@ -423,6 +423,25 @@ async function listBooks( return { bookIds: presentBookIds(booksPresent), writingSystem }; } +/** + * Reads `bookIds` of `read`'s text, {@link READ_CONCURRENCY} at a time, until all are read or the + * read ends. + */ +async function readBooks( + sourceProjectId: string, + read: TextRead, + bookIds: string[], +): Promise { + const unread = [...bookIds]; + const readNext = async (): Promise => { + const bookId = unread.shift(); + if (bookId === undefined || textReads.get(sourceProjectId) !== read) return; + await readTextBook(sourceProjectId, read, bookId); + await readNext(); + }; + await Promise.all(Array.from({ length: READ_CONCURRENCY }, readNext)); +} + /** Lists the books of `read`'s text, then reads each of them. Resolves once the read ends. */ async function readText(sourceProjectId: string, read: TextRead): Promise { const isCurrent = () => textReads.get(sourceProjectId) === read; @@ -438,14 +457,7 @@ async function readText(sourceProjectId: string, read: TextRead): Promise read.bookIds = listed.bookIds; read.writingSystem = listed.writingSystem; read.state = 'reading'; - const unread = [...read.bookIds]; - const readNext = async (): Promise => { - const bookId = unread.shift(); - if (bookId === undefined || !isCurrent()) return; - await readTextBook(sourceProjectId, read, bookId); - await readNext(); - }; - await Promise.all(Array.from({ length: READ_CONCURRENCY }, readNext)); + await readBooks(sourceProjectId, read, read.bookIds); if (isCurrent()) read.state = 'read'; } @@ -477,7 +489,7 @@ export function getTextReadStatus(sourceProjectId: string): string { } const unsubscribed = [...read.unsubscribedBookIds]; read.unsubscribedBookIds.clear(); - unsubscribed.forEach((bookId) => readTextBook(sourceProjectId, read, bookId)); + readBooks(sourceProjectId, read, unsubscribed); const { state, bookIds, readBookIds, failedBookIds } = read; if (state === 'unlisted') textReads.delete(sourceProjectId); const status: TextReadStatus = { From cafa970f70c6154e05c15a152eff8b3ebe866e76 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 21 Oct 2026 14:30:45 -0400 Subject: [PATCH 21/24] Say READ_CONCURRENCY bounds each pass of the whole-text read Co-Authored-By: Claude Opus 5.5 --- src/services/bookService.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/services/bookService.ts b/src/services/bookService.ts index ca4b64100..fd1c06fe1 100644 --- a/src/services/bookService.ts +++ b/src/services/bookService.ts @@ -51,8 +51,9 @@ const closeCounts = new Map(); const outlineCounts = new Map(); /** - * How many books a whole-text read reads at once. Enough to keep the provider busy while earlier - * books are tokenized, few enough to bound the fetches in flight. + * How many books each pass of a whole-text read reads at once: the first pass, and each retry of + * books it could not subscribe to, which may overlap it. Enough to keep the provider busy while + * earlier books are tokenized, few enough to bound the fetches in flight. */ const READ_CONCURRENCY = 4; From 4c7a76b0a7509700ba12fc26b4a3564aa5cfd4d5 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 21 Oct 2026 15:22:32 -0400 Subject: [PATCH 22/24] Get the USJ provider with the book list, and reject the status when listing fails A rejected subscription only happens when the project's USJ_Book provider can't be got, which fails every book at once. Getting the provider with the book list turns that into a listing failure, so the per-book retry goes. getTextReadStatus now resolves once the books are listed and rejects when they can't be, which replaces the listing and unlisted states. Co-Authored-By: Claude Opus 5.5 --- e2e-tests/tests/perf/app-perf.spec.ts | 13 +- src/__tests__/main.test.ts | 4 +- src/__tests__/services/bookService.test.ts | 140 ++++++----------- src/main.ts | 3 +- src/services/bookService.ts | 170 +++++++++------------ src/types/interlinearizer.d.ts | 3 +- src/types/text-read-status.ts | 5 - 7 files changed, 138 insertions(+), 200 deletions(-) diff --git a/e2e-tests/tests/perf/app-perf.spec.ts b/e2e-tests/tests/perf/app-perf.spec.ts index db4d9a7e9..a5abf08e0 100644 --- a/e2e-tests/tests/perf/app-perf.spec.ts +++ b/e2e-tests/tests/perf/app-perf.spec.ts @@ -99,11 +99,14 @@ async function textReadStatus(page: Page, projectId: string): Promise { await expect - .poll(async () => (await textReadStatus(page, projectId))?.state, { - timeout: 300_000, - intervals: [250], - }) - .toBe('read'); + .poll( + async () => { + const status = await textReadStatus(page, projectId); + return status !== undefined && status.booksRead === status.bookCount; + }, + { timeout: 300_000, intervals: [250] }, + ) + .toBe(true); expect((await textReadStatus(page, projectId))?.failedBookIds).toEqual([]); } diff --git a/src/__tests__/main.test.ts b/src/__tests__/main.test.ts index 4ea2438d3..7ae3f9bb6 100644 --- a/src/__tests__/main.test.ts +++ b/src/__tests__/main.test.ts @@ -1496,13 +1496,13 @@ describe('main', () => { ); it('returns the status the service reports for the project', async () => { - mockGetTextReadStatus.mockReturnValue('{"state":"reading"}'); + mockGetTextReadStatus.mockResolvedValue('{"booksRead":1}'); const handler = await getHandler(); const result = await handler('src-project'); expect(mockGetTextReadStatus).toHaveBeenCalledWith('src-project'); - expect(result).toBe('{"state":"reading"}'); + expect(result).toBe('{"booksRead":1}'); }); }); diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index e8ba27952..22c8ba264 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -60,18 +60,20 @@ function platformError(message: string) { /** * Wires the providers to serve one source project. Each subscription delivers `text` for its book * at once unless `text` is omitted, which leaves delivering to the test. Listing the project's - * books throws `booksPresent` when it is an error, and subscribing throws `subscribeError` while it - * is set. + * books throws `booksPresent` when it is an error, getting the USJ provider throws + * `usjProviderError` while it is set, and subscribing throws `subscribeError` while it is set. */ function serveProject(text?: (bookId: string) => Delivery) { const project: { languageTag: unknown; booksPresent: string | Error; + usjProviderError: Error | undefined; subscribeError: Error | undefined; subscriptions: Subscription[]; } = { languageTag: 'en', booksPresent: '', + usjProviderError: undefined, subscribeError: undefined, subscriptions: [], }; @@ -98,9 +100,11 @@ function serveProject(text?: (bookId: string) => Delivery) { if (project.booksPresent instanceof Error) throw project.booksPresent; return project.booksPresent; }; - mockPdpGet.mockImplementation(async (type: string) => - type === 'platform.base' ? { getSetting } : { subscribeBookUSJ }, - ); + mockPdpGet.mockImplementation(async (type: string) => { + if (type === 'platform.base') return { getSetting }; + if (project.usjProviderError) throw project.usjProviderError; + return { subscribeBookUSJ }; + }); return project; } @@ -126,8 +130,8 @@ function parsePage(json: string | undefined): SegmentPage { return JSON.parse(json); } -function readStatus(sourceProjectId = 'src'): TextReadStatus { - return JSON.parse(getTextReadStatus(sourceProjectId)); +async function readStatus(sourceProjectId = 'src'): Promise { + return JSON.parse(await getTextReadStatus(sourceProjectId)); } /** @@ -1107,15 +1111,12 @@ describe('onBookTextChanged', () => { }); describe('getTextReadStatus', () => { - it('reads nothing until the books are known', () => { - serveProject((bookId) => usj(bookId)); + it('answers once the books are known, before any is read', async () => { + const project = serveProject(); + project.booksPresent = '11'; - expect(readStatus()).toEqual({ - state: 'listing', - booksRead: 0, - bookCount: 0, - failedBookIds: [], - }); + expect(await readStatus()).toEqual({ booksRead: 0, bookCount: 2, failedBookIds: [] }); + await finishReading(project); }); it("reads every book the project's text holds, under its language tag", async () => { @@ -1125,7 +1126,7 @@ describe('getTextReadStatus', () => { readStatus(); await settle(); - expect(readStatus()).toEqual({ state: 'read', booksRead: 2, bookCount: 2, failedBookIds: [] }); + expect(await readStatus()).toEqual({ booksRead: 2, bookCount: 2, failedBookIds: [] }); expect(project.subscriptions.map((s) => s.bookId)).toEqual(['GEN', 'LEV']); expect(extractBookFromUsj).toHaveBeenCalledWith(usj('LEV'), 'en'); }); @@ -1140,8 +1141,7 @@ describe('getTextReadStatus', () => { project.subscriptions[1].deliver(usj('EXO')); await settle(); - expect(readStatus()).toEqual({ - state: 'reading', + expect(await readStatus()).toEqual({ booksRead: 2, bookCount: 3, failedBookIds: [], @@ -1185,8 +1185,7 @@ describe('getTextReadStatus', () => { readStatus(); await settle(); - expect(readStatus()).toEqual({ - state: 'read', + expect(await readStatus()).toEqual({ booksRead: 3, bookCount: 3, failedBookIds: ['EXO'], @@ -1200,32 +1199,49 @@ describe('getTextReadStatus', () => { readStatus(); await settle(); - expect(readStatus()).toEqual({ state: 'read', booksRead: 1, bookCount: 1, failedBookIds: [] }); + expect(await readStatus()).toEqual({ booksRead: 1, bookCount: 1, failedBookIds: [] }); }); - it('reports that the books could not be listed', async () => { + it('rejects when the books cannot be listed', async () => { const project = serveProject((bookId) => usj(bookId)); project.booksPresent = new Error('offline'); - readStatus(); - await settle(); + await expect(readStatus()).rejects.toThrow('offline'); + expect(project.subscriptions).toHaveLength(0); + }); + + it('rejects when the project has no provider for its text', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + project.usjProviderError = new Error('no provider'); - expect(readStatus().state).toBe('unlisted'); + await expect(readStatus()).rejects.toThrow('no provider'); expect(project.subscriptions).toHaveLength(0); }); - it('lists the books again on the request after reporting they could not be', async () => { + it('gets the provider of the text once for every book', async () => { const project = serveProject((bookId) => usj(bookId)); - project.booksPresent = new Error('offline'); + project.booksPresent = '111'; + readStatus(); await settle(); - readStatus(); + + expect(project.subscriptions).toHaveLength(3); + expect( + mockPdpGet.mock.calls.filter(([type]) => type === 'platformScripture.USJ_Book'), + ).toHaveLength(1); + }); + + it('lists the books again on the request after one that could not', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = new Error('offline'); + await expect(readStatus()).rejects.toThrow('offline'); project.booksPresent = '1'; readStatus(); await settle(); - expect(readStatus()).toEqual({ state: 'read', booksRead: 1, bookCount: 1, failedBookIds: [] }); + expect(await readStatus()).toEqual({ booksRead: 1, bookCount: 1, failedBookIds: [] }); }); it('clears a failed book once an edit lets it read', async () => { @@ -1239,7 +1255,7 @@ describe('getTextReadStatus', () => { project.subscriptions[0].deliver(delivery); await settle(); - expect(readStatus().failedBookIds).toEqual([]); + expect((await readStatus()).failedBookIds).toEqual([]); }); it('reads a book it could not watch again once the outlined book is edited', async () => { @@ -1255,67 +1271,7 @@ describe('getTextReadStatus', () => { project.subscriptions[0].deliver(usj('GEN', 'v2')); await settle(); - expect(readStatus().failedBookIds).toEqual([]); - }); - - it('reads a book it could not watch again on the next request', async () => { - const project = serveProject((bookId) => usj(bookId)); - project.booksPresent = '1'; - project.subscribeError = new Error('no provider'); - readStatus(); - await settle(); - project.subscribeError = undefined; - - expect(readStatus().failedBookIds).toEqual(['GEN']); - await settle(); - - expect(readStatus().failedBookIds).toEqual([]); - expect(project.subscriptions).toHaveLength(1); - }); - - it('reads a book it still cannot watch again on each request', async () => { - const project = serveProject((bookId) => usj(bookId)); - project.booksPresent = '1'; - project.subscribeError = new Error('no provider'); - readStatus(); - await settle(); - readStatus(); - await settle(); - project.subscribeError = undefined; - - readStatus(); - await settle(); - - expect(readStatus().failedBookIds).toEqual([]); - }); - - it('reads books it could not watch again a few at a time', async () => { - const project = serveProject(); - project.booksPresent = '111111'; - project.subscribeError = new Error('no provider'); - readStatus(); - await settle(); - project.subscribeError = undefined; - - readStatus(); - await settle(); - - expect(project.subscriptions.length).toBeGreaterThan(0); - expect(project.subscriptions.length).toBeLessThan(6); - await finishReading(project); - }); - - it('does not read a failed book it watches again until an edit is heard', async () => { - const project = serveProject(() => platformError('busy')); - project.booksPresent = '1'; - readStatus(); - await settle(); - - readStatus(); - await settle(); - - expect(readStatus().failedBookIds).toEqual(['GEN']); - expect(project.subscriptions).toHaveLength(1); + expect((await readStatus()).failedBookIds).toEqual([]); }); it('stops watching the books once the project closes', async () => { @@ -1365,7 +1321,7 @@ describe('getTextReadStatus', () => { readStatus(); await settle(); - expect(readStatus().state).toBe('read'); + expect(await readStatus()).toEqual({ booksRead: 1, bookCount: 1, failedBookIds: [] }); expect(project.subscriptions).toHaveLength(2); }); diff --git a/src/main.ts b/src/main.ts index 5c182d81f..7b7632bbc 100644 --- a/src/main.ts +++ b/src/main.ts @@ -557,7 +557,8 @@ async function getSegments( * Reports how far reading the source project's whole text has got, starting the read when the * project has none. * - * @returns The `TextReadStatus` as JSON. + * @returns Once the text's books are known, the `TextReadStatus` as JSON. + * @throws If the books or the provider of their text cannot be got. */ async function getTextReadStatus(sourceProjectId: string): Promise { return bookService.getTextReadStatus(sourceProjectId); diff --git a/src/services/bookService.ts b/src/services/bookService.ts index fd1c06fe1..6f2002f7e 100644 --- a/src/services/bookService.ts +++ b/src/services/bookService.ts @@ -4,6 +4,7 @@ import { tokenizeBook } from 'parsers/papi/bookTokenizer'; import { resegmentBook } from 'parsers/papi/resegmentBook'; import { extractBookFromUsj, type UsjDocument } from 'parsers/papi/usjBookExtractor'; import { isPlatformError } from 'platform-bible-utils'; +import type { IUSJBookProjectDataProvider } from 'platform-scripture'; import type { SegmentPage } from '../types/segment-pages'; import type { TextReadStatus } from '../types/text-read-status'; import { isSegmentationDelta } from '../types/type-guards'; @@ -17,8 +18,6 @@ import { reanchorSegmentation } from '../utils/segmentation'; interface BookTextWatch { /** The text as first delivered; `undefined` when the project has no text for the book. */ text: Promise; - /** Whether subscribing to the book's USJ succeeded, so that edits to it can be heard. */ - isSubscribed: Promise; stop: () => Promise; } @@ -51,25 +50,28 @@ const closeCounts = new Map(); const outlineCounts = new Map(); /** - * How many books each pass of a whole-text read reads at once: the first pass, and each retry of - * books it could not subscribe to, which may overlap it. Enough to keep the provider busy while - * earlier books are tokenized, few enough to bound the fetches in flight. + * How many books a whole-text read reads at once. Enough to keep the provider busy while earlier + * books are tokenized, few enough to bound the fetches in flight. */ const READ_CONCURRENCY = 4; /** A source project's whole text, read book by book into a form index, each book watched for edits. */ interface TextRead { - state: TextReadStatus['state']; - /** The text's books in canonical order; empty until they are known. */ - bookIds: string[]; + /** + * The text's books in canonical order, with the tag their text is written in; rejects when they, + * or the provider of their text, could not be got. + */ + listed: Promise<{ bookIds: string[]; writingSystem: string }>; + /** The provider each book is read from, got as the books are listed. */ + usjPdp: Promise; /** The tag the books are tokenized under; `''` until the books are known. */ writingSystem: string; + /** Whether every book has been read once. */ + isRead: boolean; /** Books read at least once, whether or not they failed. */ readBookIds: Set; /** Books whose latest read failed. */ failedBookIds: Set; - /** Failed books with no subscription to hear an edit through, read again on the next status. */ - unsubscribedBookIds: Set; forms: FormIndex; /** Stops watching each book the read has started, by book id. */ watches: Map Promise>; @@ -77,7 +79,7 @@ interface TextRead { /** * Each source project's whole-text read, from the first status asked for until the project closes, - * or until the read reports its books unlisted. + * or until its books could not be listed. */ const textReads = new Map(); @@ -105,6 +107,7 @@ function watchBookText( sourceProjectId: string, bookId: string, onChange: () => void, + usjPdp = papi.projectDataProviders.get('platformScripture.USJ_Book', sourceProjectId), ): BookTextWatch { const { promise: text, @@ -112,29 +115,23 @@ function watchBookText( reject: rejectText, } = Promise.withResolvers(); let isFirstDelivery = true; - const subscription = papi.projectDataProviders - .get('platformScripture.USJ_Book', sourceProjectId) - .then((pdp) => - // PAPI calls back after the first delivery only for text that differs from the last, or for - // an error reading it again. - pdp.subscribeBookUSJ({ book: bookId, chapterNum: 1, verseNum: 1 }, (usj) => { - if (!isFirstDelivery) { - onChange(); - return; - } - isFirstDelivery = false; - if (isPlatformError(usj)) rejectText(new Error(usj.message)); - else resolveText(usj); - }), - ); + const subscription = usjPdp.then((pdp) => + // PAPI calls back after the first delivery only for text that differs from the last, or for + // an error reading it again. + pdp.subscribeBookUSJ({ book: bookId, chapterNum: 1, verseNum: 1 }, (usj) => { + if (!isFirstDelivery) { + onChange(); + return; + } + isFirstDelivery = false; + if (isPlatformError(usj)) rejectText(new Error(usj.message)); + else resolveText(usj); + }), + ); subscription.catch(rejectText); return { text, - isSubscribed: subscription.then( - () => true, - () => false, - ), stop: async () => { // Unsubscribing ahead of the first delivery would leave `text` unsettled. await text.catch(() => undefined); @@ -366,13 +363,18 @@ async function readTextBook( bookId: string, ): Promise { read.watches.get(bookId)?.(); - const { text, isSubscribed, stop } = watchBookText(sourceProjectId, bookId, () => { - if (textReads.get(sourceProjectId) !== read || read.watches.get(bookId) !== stop) return; - rereadHeldBook(sourceProjectId, bookId); - rereadTextBook(sourceProjectId, read, bookId); - }); + const { text, stop } = watchBookText( + sourceProjectId, + bookId, + () => { + if (textReads.get(sourceProjectId) !== read || read.watches.get(bookId) !== stop) return; + rereadHeldBook(sourceProjectId, bookId); + rereadTextBook(sourceProjectId, read, bookId); + }, + read.usjPdp, + ); read.watches.set(bookId, stop); - const formsRead = perfTrack('readText:usj', text) + const forms = await perfTrack('readText:usj', text) .then((usj) => usj ? perfTime('readText:tokenize', () => @@ -384,7 +386,6 @@ async function readTextBook( logger.warn(`Could not read ${bookId} of project ${sourceProjectId}: ${e}`); return undefined; }); - const [forms, subscribed] = await Promise.all([formsRead, isSubscribed]); if (textReads.get(sourceProjectId) !== read) return false; // A later read of the book reads newer text, so this one files only a book none has filed. if (read.watches.get(bookId) !== stop && read.readBookIds.has(bookId)) return false; @@ -394,8 +395,6 @@ async function readTextBook( } else { read.failedBookIds.add(bookId); } - if (subscribed) read.unsubscribedBookIds.delete(bookId); - else read.unsubscribedBookIds.add(bookId); read.readBookIds.add(bookId); return true; } @@ -424,77 +423,60 @@ async function listBooks( return { bookIds: presentBookIds(booksPresent), writingSystem }; } -/** - * Reads `bookIds` of `read`'s text, {@link READ_CONCURRENCY} at a time, until all are read or the - * read ends. - */ -async function readBooks( - sourceProjectId: string, - read: TextRead, - bookIds: string[], -): Promise { - const unread = [...bookIds]; +/** Reads each book of `read`'s text once the books are listed. Resolves once the read ends. */ +async function readText(sourceProjectId: string, read: TextRead): Promise { + const isCurrent = () => textReads.get(sourceProjectId) === read; + const listed = await read.listed.catch(() => { + // Dropped before the failure reaches any request, so the next request starts the read again. + if (isCurrent()) textReads.delete(sourceProjectId); + return undefined; + }); + if (!listed || !isCurrent()) return; + read.writingSystem = listed.writingSystem; + const unread = [...listed.bookIds]; const readNext = async (): Promise => { const bookId = unread.shift(); - if (bookId === undefined || textReads.get(sourceProjectId) !== read) return; + if (bookId === undefined || !isCurrent()) return; await readTextBook(sourceProjectId, read, bookId); await readNext(); }; await Promise.all(Array.from({ length: READ_CONCURRENCY }, readNext)); + if (isCurrent()) read.isRead = true; } -/** Lists the books of `read`'s text, then reads each of them. Resolves once the read ends. */ -async function readText(sourceProjectId: string, read: TextRead): Promise { - const isCurrent = () => textReads.get(sourceProjectId) === read; - const listed = await listBooks(sourceProjectId).catch((e) => { - logger.warn(`Could not list the books of project ${sourceProjectId}: ${e}`); - return undefined; - }); - if (!isCurrent()) return; - if (!listed) { - read.state = 'unlisted'; - return; - } - read.bookIds = listed.bookIds; - read.writingSystem = listed.writingSystem; - read.state = 'reading'; - await readBooks(sourceProjectId, read, read.bookIds); - if (isCurrent()) read.state = 'read'; +/** Starts reading the source project's whole text. */ +function startTextRead(sourceProjectId: string): TextRead { + const usjPdp = papi.projectDataProviders.get('platformScripture.USJ_Book', sourceProjectId); + const read: TextRead = { + listed: Promise.all([listBooks(sourceProjectId), usjPdp]).then(([listed]) => listed), + usjPdp, + writingSystem: '', + isRead: false, + readBookIds: new Set(), + failedBookIds: new Set(), + forms: new FormIndex(), + watches: new Map(), + }; + textReads.set(sourceProjectId, read); + perfTrack('readText', readText(sourceProjectId, read)); + return read; } /** * Reports how far reading the source project's whole text has got, starting the read when the - * project has none. A read whose books could not be listed is reported once, then started again on - * the next request. Each request also reads again any book that failed with no subscription to hear - * an edit through. The read keeps a form index that {@link isFormInText} answers from, and watches + * project has none. The read keeps a form index that {@link isFormInText} answers from, and watches * each book it has read, reporting its edits through {@link onBookTextChanged}, until the project * closes. * - * @returns The `TextReadStatus` as JSON. + * @returns Once the text's books are known, the `TextReadStatus` as JSON. + * @throws If the books or the provider of their text cannot be got. The read is dropped, so the + * next request starts it again. */ -export function getTextReadStatus(sourceProjectId: string): string { - let read = textReads.get(sourceProjectId); - if (!read) { - read = { - state: 'listing', - bookIds: [], - writingSystem: '', - readBookIds: new Set(), - failedBookIds: new Set(), - unsubscribedBookIds: new Set(), - forms: new FormIndex(), - watches: new Map(), - }; - textReads.set(sourceProjectId, read); - perfTrack('readText', readText(sourceProjectId, read)); - } - const unsubscribed = [...read.unsubscribedBookIds]; - read.unsubscribedBookIds.clear(); - readBooks(sourceProjectId, read, unsubscribed); - const { state, bookIds, readBookIds, failedBookIds } = read; - if (state === 'unlisted') textReads.delete(sourceProjectId); +export async function getTextReadStatus(sourceProjectId: string): Promise { + const read = textReads.get(sourceProjectId) ?? startTextRead(sourceProjectId); + const { bookIds } = await read.listed; + const { readBookIds, failedBookIds } = read; const status: TextReadStatus = { - state, booksRead: readBookIds.size, bookCount: bookIds.length, failedBookIds: bookIds.filter((bookId) => failedBookIds.has(bookId)), @@ -509,7 +491,7 @@ export function getTextReadStatus(sourceProjectId: string): string { */ export function isFormInText(sourceProjectId: string, form: string): boolean | undefined { const read = textReads.get(sourceProjectId); - if (read?.state !== 'read' || read.failedBookIds.size > 0) return undefined; + if (!read?.isRead || read.failedBookIds.size > 0) return undefined; return read.forms.has(normalizeSurfaceForm(form)); } diff --git a/src/types/interlinearizer.d.ts b/src/types/interlinearizer.d.ts index 1461c5e03..368a9abf8 100644 --- a/src/types/interlinearizer.d.ts +++ b/src/types/interlinearizer.d.ts @@ -283,7 +283,8 @@ declare module 'papi-shared-types' { * Reports how far reading the source project's whole text has got, starting the read when the * project has none. * - * @returns The `TextReadStatus` as a JSON string. + * @returns Once the text's books are known, the `TextReadStatus` as a JSON string. + * @throws If the books or the provider of their text cannot be got. The next call tries again. */ 'interlinearizer.getTextReadStatus': (sourceProjectId: string) => Promise; diff --git a/src/types/text-read-status.ts b/src/types/text-read-status.ts index 3ac463919..21666ccf9 100644 --- a/src/types/text-read-status.ts +++ b/src/types/text-read-status.ts @@ -1,10 +1,5 @@ /** How far reading a source project's whole text has got. */ export type TextReadStatus = Readonly<{ - /** - * `listing` until the text's books are known, `unlisted` when they could not be, `reading` until - * every book has been read once, then `read`. - */ - state: 'listing' | 'unlisted' | 'reading' | 'read'; /** How many of the text's books have been read at least once, those that failed included. */ booksRead: number; /** How many books the text holds; `0` until they are known. */ From da7bb2b4dcfc844a03c95fc95db16eef787bb6d8 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 21 Oct 2026 16:49:05 -0400 Subject: [PATCH 23/24] Log a failed text read status, and drop the bookCount 0 note Co-Authored-By: Claude Opus 5.5 --- src/__tests__/main.test.ts | 11 +++++++++++ src/main.ts | 10 ++++++++-- src/types/text-read-status.ts | 2 +- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/__tests__/main.test.ts b/src/__tests__/main.test.ts index 7ae3f9bb6..e8df15af8 100644 --- a/src/__tests__/main.test.ts +++ b/src/__tests__/main.test.ts @@ -1504,6 +1504,17 @@ describe('main', () => { expect(mockGetTextReadStatus).toHaveBeenCalledWith('src-project'); expect(result).toBe('{"booksRead":1}'); }); + + it('logs the error and rethrows when the books cannot be listed', async () => { + mockGetTextReadStatus.mockRejectedValue(new Error('offline')); + const handler = await getHandler(); + + await expect(handler('src-project')).rejects.toThrow('offline'); + expect(__mockLogger.error).toHaveBeenCalledWith( + 'Interlinearizer: failed to read the text of project src-project', + expect.any(Error), + ); + }); }); describe('interlinearizer.setPerfMarks command', () => { diff --git a/src/main.ts b/src/main.ts index 7b7632bbc..d854622f7 100644 --- a/src/main.ts +++ b/src/main.ts @@ -558,10 +558,16 @@ async function getSegments( * project has none. * * @returns Once the text's books are known, the `TextReadStatus` as JSON. - * @throws If the books or the provider of their text cannot be got. + * @throws If the books or the provider of their text cannot be got. The error is logged before + * rethrowing. */ async function getTextReadStatus(sourceProjectId: string): Promise { - return bookService.getTextReadStatus(sourceProjectId); + try { + return await bookService.getTextReadStatus(sourceProjectId); + } catch (e) { + logger.error(`Interlinearizer: failed to read the text of project ${sourceProjectId}`, e); + throw e; + } } /** diff --git a/src/types/text-read-status.ts b/src/types/text-read-status.ts index 21666ccf9..63e56e467 100644 --- a/src/types/text-read-status.ts +++ b/src/types/text-read-status.ts @@ -2,7 +2,7 @@ export type TextReadStatus = Readonly<{ /** How many of the text's books have been read at least once, those that failed included. */ booksRead: number; - /** How many books the text holds; `0` until they are known. */ + /** How many books the text holds. */ bookCount: number; /** The books whose latest read failed, in canonical order. */ failedBookIds: readonly string[]; From e6c4f75428e3d7ef2d41060a52eb252d72bf6eb8 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 21 Oct 2026 17:17:42 -0400 Subject: [PATCH 24/24] Report no edit for a book that fails to read again, and tag has's form constraint Core calls back with an error after every save while a book can't be read, so each save reported an edit to every book that kept failing. Co-Authored-By: Claude Opus 5.5 --- src/__tests__/services/bookService.test.ts | 45 ++++++++++++++++++++++ src/services/bookService.ts | 9 +++-- src/utils/form-index.ts | 5 ++- 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index 22c8ba264..169eca697 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -897,6 +897,51 @@ describe('onBookTextChanged', () => { expect(listener).toHaveBeenCalledWith('src', 'EXO'); }); + it('reports nothing for an error on a book that still cannot be read', async () => { + const project = serveProject(() => platformError('busy')); + project.booksPresent = '1'; + const listener = listen(); + readStatus(); + await settle(); + + subscriptionsTo(project, 'GEN').at(-1)?.deliver(platformError('busy')); + await settle(); + + expect(listener).not.toHaveBeenCalled(); + expect((await readStatus()).failedBookIds).toEqual(['GEN']); + }); + + it('reports an edit to a book that failed to read once it reads', async () => { + let delivery: Delivery = platformError('busy'); + const project = serveProject(() => delivery); + project.booksPresent = '1'; + const listener = listen(); + readStatus(); + await settle(); + + delivery = usj('GEN', 'v2'); + subscriptionsTo(project, 'GEN').at(-1)?.deliver(delivery); + await settle(); + + expect(listener).toHaveBeenCalledTimes(1); + }); + + it('reports a book the whole-text read has read once it fails to read', async () => { + let delivery: Delivery = usj('GEN'); + const project = serveProject(() => delivery); + project.booksPresent = '1'; + const listener = listen(); + readStatus(); + await settle(); + + delivery = platformError('busy'); + subscriptionsTo(project, 'GEN').at(-1)?.deliver(delivery); + await settle(); + + expect(listener).toHaveBeenCalledTimes(1); + expect((await readStatus()).failedBookIds).toEqual(['GEN']); + }); + it("reports an edit once the whole-text read's answers include it", async () => { tokenizeToWords(); let version = 'v1'; diff --git a/src/services/bookService.ts b/src/services/bookService.ts index 6f2002f7e..13a0b8b2c 100644 --- a/src/services/bookService.ts +++ b/src/services/bookService.ts @@ -400,15 +400,18 @@ async function readTextBook( } /** - * Reads `bookId` of `read`'s text again, then reports its edit unless the read has ended or a later - * read of the book will report it. + * Reads `bookId` of `read`'s text again, then reports its edit unless the read has ended, a later + * read of the book will report it, or the book failed to read both before and again. */ async function rereadTextBook( sourceProjectId: string, read: TextRead, bookId: string, ): Promise { - if (await readTextBook(sourceProjectId, read, bookId)) reportEdit(sourceProjectId, bookId); + const wasFailed = read.failedBookIds.has(bookId); + if (!(await readTextBook(sourceProjectId, read, bookId))) return; + // Core calls back with an error after every save while a book can't be read. + if (!wasFailed || !read.failedBookIds.has(bookId)) reportEdit(sourceProjectId, bookId); } /** Lists the source project's books in canonical order, with the tag their text is written in. */ diff --git a/src/utils/form-index.ts b/src/utils/form-index.ts index faf3a85c4..1098dfdaf 100644 --- a/src/utils/form-index.ts +++ b/src/utils/form-index.ts @@ -33,8 +33,9 @@ export class FormIndex { } /** - * Whether a filed book holds `form`, which must already be normalized as {@link formsOfBook} - * normalizes forms. + * Whether a filed book holds `form`. + * + * @param form - Normalized as {@link formsOfBook} normalizes forms. */ has(form: string): boolean { return this.bookCounts.has(form);