diff --git a/e2e-tests/tests/perf/app-perf.spec.ts b/e2e-tests/tests/perf/app-perf.spec.ts index 38fbbb102..a5abf08e0 100644 --- a/e2e-tests/tests/perf/app-perf.spec.ts +++ b/e2e-tests/tests/perf/app-perf.spec.ts @@ -6,6 +6,7 @@ import { TIERS, type TierSpec } from '../../../perf/dataset/tiers'; import { provenance, writeResults } from '../../../perf/results'; import { addSavedProject, readDraft, writeDraft } from '../../../perf/storage'; import { readCaptureMeta } from '../../../perf/usj-cache'; +import type { TextReadStatus } from '../../../src/types/text-read-status'; import { expect, test } from '../../fixtures/cdp.fixture'; import { waitForAppAndInterlinearizerReady } from '../../fixtures/helpers'; import { @@ -81,6 +82,34 @@ async function waitForAutosave(frame: Frame): Promise { return measures; } +/** + * Reads the host's whole-text read status for the source, starting the read when it has none. + * + * @returns The status, or `undefined` when the command answers with something other than a string. + */ +async function textReadStatus(page: Page, projectId: string): Promise { + const json = await sendCommand(page, 'interlinearizer.getTextReadStatus', projectId); + return typeof json === 'string' ? JSON.parse(json) : undefined; +} + +/** + * Starts the host's read of the source's whole text, and waits until it has read every book. + * + * @throws If any book failed to read, since the read's timing would then leave it out. + */ +async function readWholeText(page: Page, projectId: string): Promise { + await expect + .poll( + async () => { + const status = await textReadStatus(page, projectId); + return status !== undefined && status.booksRead === status.bookCount; + }, + { timeout: 300_000, intervals: [250] }, + ) + .toBe(true); + expect((await textReadStatus(page, projectId))?.failedBookIds).toEqual([]); +} + /** Waits for any autosave an edit or Save started to land. */ async function settleAutosave(): Promise { if (unsavedFrame) await waitForAutosave(unsavedFrame); @@ -433,6 +462,39 @@ async function measureTier( ); } result.concordance = concordance.summary(); + + // Whole-text read in the host, in a fresh view each run, since the host keeps the read until + // the view closes. + const textRead = new Samples(); + for (let run = 0; run < RUNS; run += 1) { + const isLast = run === RUNS - 1; + // eslint-disable-next-line no-await-in-loop + await closeSettledView(page); + // eslint-disable-next-line no-await-in-loop + const { frame: view } = await openView(page, projectId, tier.viewBook); + // eslint-disable-next-line no-await-in-loop + await waitForMeasure(view, 'reanchor', tier.viewBook); + // eslint-disable-next-line no-await-in-loop + if (isLast) await recordMemory('beforeTextRead'); + // eslint-disable-next-line no-await-in-loop + await takeHostEntries(page); + // eslint-disable-next-line no-await-in-loop + await readWholeText(page, projectId); + // eslint-disable-next-line no-await-in-loop + const entries = await takeHostEntries(page); + entries + .filter(({ name }) => name === 'readText') + .forEach(({ duration }) => textRead.addValue('text-read', duration)); + textRead.addValue( + 'text-read-tokenize', + entries + .filter(({ name }) => name === 'readText:tokenize') + .reduce((total, { duration }) => total + duration, 0), + ); + // eslint-disable-next-line no-await-in-loop + if (isLast) await recordMemory('afterTextRead'); + } + result.textRead = textRead.summary(); } test('measure the app', async ({ mainPage }) => { diff --git a/perf/README.md b/perf/README.md index b8f55a337..82c681835 100644 --- a/perf/README.md +++ b/perf/README.md @@ -13,7 +13,7 @@ Measures the extension rather than tests it: the data layer's pure functions in `PERF_TIERS` (comma-separated tier names) and `PERF_RUNS` (10 by default) narrow a run. The `large` tier's draft exceeds one websocket message, so only the Node benchmarks can measure it. -Results land in `perf/results/` as JSON, each recording the machine, its load average, and both commits. +Results land in `perf/results/` as JSON, each recording the machine, its load average, and both commits. Before committing one, suffix its name with the issue it was measured for and why, such as `-407-baseline`; `perf:summary` reads the time from the file, not its name. ## Dataset shape diff --git a/perf/bench/cli.ts b/perf/bench/cli.ts index 6768b089d..ff22d26a8 100644 --- a/perf/bench/cli.ts +++ b/perf/bench/cli.ts @@ -20,6 +20,7 @@ import { deriveConcordanceRows, indexBook, } from '../../src/utils/concordance'; +import { FormIndex, formsOfBook } from '../../src/utils/form-index'; import { collatorForTag } from '../../src/utils/language-tags'; import { reanchorAnalysisToBook } from '../../src/utils/reanchor-analysis'; import { MAX_UNDO_STEPS } from '../../src/utils/undo-history'; @@ -143,12 +144,35 @@ function benchTier(tier: TierSpec, allBooks: () => Book[]): Record [ta.id, ta])), + ), viewBookCode, ); }), ); + const indexForms = () => { + const index = new FormIndex(); + books.forEach((book) => index.setBook(book.bookRef, formsOfBook(book))); + return index; + }; + time('text.formIndex', measure(opts, noSetup, indexForms)); + // The host's form index and the WebView's read must agree on which catalog forms are missing. + const index = indexForms(); + const textForms = new Set(books.flatMap((book) => [...indexBook(book).occurrencesByForm.keys()])); + const catalogForms = selectCatalogRows( + { analysis, analysisLanguage: language }, + viewBookCode, + ).map((row) => row.form); + const byWebViewRead = catalogForms.filter((form) => !textForms.has(form)); + const byHostIndex = catalogForms.filter((form) => !index.has(form)); + const disagreeing = catalogForms.filter((form) => textForms.has(form) !== index.has(form)).length; + console.log( + ` forms not in the text: ${byWebViewRead.length} by the WebView's read, ${byHostIndex.length} by the host's index, ${disagreeing} disagreeing`, + ); + // The undo history holds a snapshot of the analysis before each of its steps. const undoBytes = retainedBytes(() => { let state: AnalysisState = { analysis, analysisLanguage: language }; @@ -166,6 +190,11 @@ function benchTier(tier: TierSpec, allBooks: () => Book[]): Record = { "Committing one gloss to the store and rerunning the view's selectors, then serializing the draft to save it.", 'bench.timings.concordance': 'Indexing every book of the source and deriving the concordance rows.', + 'bench.timings.text': + "Filing every book's word forms in the host's form index, the books already tokenized.", + 'bench.formsNotInText': + "How many of the catalog's forms the text lacks, by the WebView's whole-text read and by the host's form index, and how many forms the two disagree on.", + 'app.textRead': + "With a view open, the host reads every book of the source and files its word forms, from the first `getTextReadStatus` until it reports every book read. Tokenizing is summed over the books, which are read a few at a time. The host heap with the whole text read includes each book's USJ, which the platform keeps for every book subscription.", 'bench.memory': 'Heap held by the parsed draft, and by an undo history filled to its limit.', }; @@ -77,10 +83,13 @@ const SECTION_LABELS: Record = { picker: 'List saved projects in the picker', save: 'Save the analysis to its project', concordance: 'Open the concordance', + textRead: 'Read the whole text in the host', + formsNotInText: 'Catalog forms not in the text', 'timings.load': 'Load', 'timings.storage': 'Storage', 'timings.edit': 'Edit one gloss', 'timings.concordance': 'Concordance', + 'timings.text': 'Form-in-text index', }; /** Names for measured values, by their key; an unlisted value shows its key. */ @@ -110,6 +119,12 @@ const METRIC_LABELS: Record = { 'save-bytes': 'Analysis sent', 'concordance-read': 'Read every book', 'concordance-entries': 'Build entries', + 'text-read': 'Read every book', + 'text-read-tokenize': 'Tokenize every book, summed', + formIndex: "Index every book's forms", + byWebViewRead: "By the WebView's read", + byHostIndex: "By the host's index", + disagreeing: 'Disagreeing', parseDraft: 'Parse draft', storeMount: 'Mount analysis store', readShards: 'Parse and merge shards', @@ -139,6 +154,8 @@ const MEMORY_POINTS: Record = { withFullUndoHistory: 'full undo history', afterPicker: 'after picker', afterSave: 'after save', + beforeTextRead: 'before the whole-text read', + afterTextRead: 'with the whole text read', }; const isBytes = (name: string) => /bytes/i.test(name); diff --git a/src/__tests__/main.test.ts b/src/__tests__/main.test.ts index 245906c19..e8df15af8 100644 --- a/src/__tests__/main.test.ts +++ b/src/__tests__/main.test.ts @@ -273,10 +273,10 @@ describe('main', () => { return emitters; } - it('emits the project and book of an edit to the open book', async () => { + it('emits the project and book of an edit to a watched book', async () => { const emitters = emittersByEvent(); await activate(createTestActivationContext()); - const [listener] = jest.mocked(bookService.onOpenBookTextChanged).mock.calls[0]; + const [listener] = jest.mocked(bookService.onBookTextChanged).mock.calls[0]; listener('src-project', 'GEN'); @@ -290,7 +290,7 @@ describe('main', () => { it('gives each emitted change its own id', async () => { const emitters = emittersByEvent(); await activate(createTestActivationContext()); - const [listener] = jest.mocked(bookService.onOpenBookTextChanged).mock.calls[0]; + const [listener] = jest.mocked(bookService.onBookTextChanged).mock.calls[0]; listener('src-project', 'GEN'); listener('src-project', 'GEN'); @@ -302,7 +302,7 @@ describe('main', () => { it('disposes the emitter and stops listening on deactivation', async () => { const emitters = emittersByEvent(); const unsubscribe = jest.fn(() => true); - jest.mocked(bookService.onOpenBookTextChanged).mockReturnValue(unsubscribe); + jest.mocked(bookService.onBookTextChanged).mockReturnValue(unsubscribe); const context = createTestActivationContext(); await activate(context); @@ -374,6 +374,7 @@ describe('main', () => { 'interlinearizer.getTokenizedBook', 'interlinearizer.getBookOutline', 'interlinearizer.getSegments', + 'interlinearizer.getTextReadStatus', 'interlinearizer.saveDraft', 'interlinearizer.openSelectProjectModal', 'interlinearizer.openNewProjectModal', @@ -1487,6 +1488,35 @@ describe('main', () => { }); }); + describe('interlinearizer.getTextReadStatus command', () => { + const mockGetTextReadStatus = jest.mocked(bookService.getTextReadStatus); + const getHandler = () => + activateAndGetHandler<(sourceProjectId: string) => Promise>( + 'interlinearizer.getTextReadStatus', + ); + + it('returns the status the service reports for the project', async () => { + mockGetTextReadStatus.mockResolvedValue('{"booksRead":1}'); + const handler = await getHandler(); + + const result = await handler('src-project'); + + expect(mockGetTextReadStatus).toHaveBeenCalledWith('src-project'); + expect(result).toBe('{"booksRead":1}'); + }); + + it('logs the error and rethrows when the books cannot be listed', async () => { + mockGetTextReadStatus.mockRejectedValue(new Error('offline')); + const handler = await getHandler(); + + await expect(handler('src-project')).rejects.toThrow('offline'); + expect(__mockLogger.error).toHaveBeenCalledWith( + 'Interlinearizer: failed to read the text of project src-project', + expect.any(Error), + ); + }); + }); + describe('interlinearizer.setPerfMarks command', () => { it("turns the host's timing entries on or off as asked", async () => { const handler = await activateAndGetHandler<(enabled: boolean) => Promise>( diff --git a/src/__tests__/services/bookService.test.ts b/src/__tests__/services/bookService.test.ts index 3acdd691d..169eca697 100644 --- a/src/__tests__/services/bookService.test.ts +++ b/src/__tests__/services/bookService.test.ts @@ -10,11 +10,14 @@ import { closeProject, getBookOutline, getSegments, + getTextReadStatus, getTokenizedBook, getWritingSystem, - onOpenBookTextChanged, + isFormInText, + onBookTextChanged, } from '../../services/bookService'; import type { BookOutline, SegmentPage } from '../../types/segment-pages'; +import type { TextReadStatus } from '../../types/text-read-status'; import { buildBookOutline } from '../../utils/book-outline'; import * as perfMarks from '../../utils/perf-marks'; import { reanchorSegmentation } from '../../utils/segmentation'; @@ -41,6 +44,7 @@ type Delivery = UsjDocument | undefined | { message: string; platformErrorVersio interface Subscription { bookId: string; deliver: (usj: Delivery) => void; + isDelivered: boolean; unsubscribe: jest.Mock; } @@ -55,26 +59,52 @@ function platformError(message: string) { /** * Wires the providers to serve one source project. Each subscription delivers `text` for its book - * at once unless `text` is omitted, which leaves delivering to the test. + * at once unless `text` is omitted, which leaves delivering to the test. Listing the project's + * books throws `booksPresent` when it is an error, getting the USJ provider throws + * `usjProviderError` while it is set, and subscribing throws `subscribeError` while it is set. */ function serveProject(text?: (bookId: string) => Delivery) { - const project: { languageTag: unknown; subscriptions: Subscription[] } = { + const project: { + languageTag: unknown; + booksPresent: string | Error; + usjProviderError: Error | undefined; + subscribeError: Error | undefined; + subscriptions: Subscription[]; + } = { languageTag: 'en', + booksPresent: '', + usjProviderError: undefined, + subscribeError: undefined, subscriptions: [], }; const subscribeBookUSJ = jest.fn( async ({ book }: { book: string }, callback: (usj: Delivery) => void) => { + if (project.subscribeError) throw project.subscribeError; const unsubscribe = jest.fn(async () => true); - project.subscriptions.push({ bookId: book, deliver: callback, unsubscribe }); - if (text) queueMicrotask(() => callback(text(book))); + const subscription: Subscription = { + bookId: book, + deliver: (delivery) => { + subscription.isDelivered = true; + callback(delivery); + }, + isDelivered: false, + unsubscribe, + }; + project.subscriptions.push(subscription); + if (text) queueMicrotask(() => subscription.deliver(text(book))); return unsubscribe; }, ); - mockPdpGet.mockImplementation(async (type: string) => - type === 'platform.base' - ? { getSetting: async () => project.languageTag } - : { subscribeBookUSJ }, - ); + const getSetting = async (key: string) => { + if (key !== 'platformScripture.booksPresent') return project.languageTag; + if (project.booksPresent instanceof Error) throw project.booksPresent; + return project.booksPresent; + }; + mockPdpGet.mockImplementation(async (type: string) => { + if (type === 'platform.base') return { getSetting }; + if (project.usjProviderError) throw project.usjProviderError; + return { subscribeBookUSJ }; + }); return project; } @@ -100,6 +130,46 @@ function parsePage(json: string | undefined): SegmentPage { return JSON.parse(json); } +async function readStatus(sourceProjectId = 'src'): Promise { + return JSON.parse(await getTextReadStatus(sourceProjectId)); +} + +/** + * Delivers each book a read awaits, and those it goes on to read, so a project can close without + * waiting on a delivery. + */ +async function finishReading(project: ReturnType) { + const pending = project.subscriptions.filter((s) => !s.isDelivered); + if (pending.length === 0) return; + pending.forEach((s) => s.deliver(usj(s.bookId))); + await settle(); + await finishReading(project); +} + +/** The subscriptions made to `bookId`, oldest first. */ +function subscriptionsTo(project: ReturnType, bookId: string) { + return project.subscriptions.filter((s) => s.bookId === bookId); +} + +/** + * Has each book tokenize to the word `shared` and a word naming its book and text version, such as + * `GEN-v1`. + */ +function tokenizeToWords() { + jest.mocked(tokenizeBook).mockImplementation((raw) => ({ + id: raw.bookCode, + bookRef: raw.bookCode, + textVersion: `${raw.contentHash}@${raw.writingSystem}`, + duplicateVerseIds: [], + segments: [ + makeSegment(`${raw.bookCode} 1:1`, `shared ${raw.bookCode}-${raw.contentHash}`, [ + makeWordToken(`${raw.bookCode} 1:1:0`, 'shared'), + makeWordToken(`${raw.bookCode} 1:1:7`, `${raw.bookCode}-${raw.contentHash}`, 7), + ]), + ], + })); +} + /** A tokenized Genesis 1:1–3, standing in for whatever text the project holds. */ const VERSE_BOOK: Book = { id: 'GEN', @@ -535,13 +605,13 @@ describe('closeAllProjects', () => { }); }); -describe('onOpenBookTextChanged', () => { +describe('onBookTextChanged', () => { const unsubscribers: Array<() => boolean> = []; /** A listener registered for the test, removed once it ends. */ - function listen() { - const listener = jest.fn(); - unsubscribers.push(onOpenBookTextChanged(listener)); + function listen(onCall?: () => void) { + const listener = jest.fn(onCall); + unsubscribers.push(onBookTextChanged(listener)); return listener; } @@ -553,6 +623,57 @@ describe('onOpenBookTextChanged', () => { unsubscribers.splice(0).forEach((unsubscribe) => unsubscribe()); }); + it('subscribes once to a book that is outlined and read whole', async () => { + const project = serveProject((bookId) => usj(bookId)); + + await getBookOutline('src', 'GEN'); + await getTokenizedBook('src', 'GEN'); + await getSegments('src', 'GEN', 0, 1); + + expect(project.subscriptions).toHaveLength(1); + }); + + it('subscribes once to a book however often it is outlined', async () => { + const project = serveProject((bookId) => usj(bookId)); + + await getBookOutline('src', 'GEN'); + await getBookOutline('src', 'GEN'); + + expect(project.subscriptions).toHaveLength(1); + }); + + it('keeps the outlined book while other books are read whole', async () => { + const project = serveProject((bookId) => usj(bookId)); + await getBookOutline('src', 'GEN'); + await getTokenizedBook('src', 'EXO'); + await getTokenizedBook('src', 'LEV'); + await settle(); + jest.mocked(tokenizeBook).mockClear(); + + await getTokenizedBook('src', 'GEN'); + await getBookOutline('src', 'GEN'); + const page = parsePage(await getSegments('src', 'GEN', 0, 2)); + + expect(page.segments).toHaveLength(2); + expect(tokenizeBook).not.toHaveBeenCalled(); + expect(subscriptionsTo(project, 'GEN')).toHaveLength(1); + }); + + it('lets go of a book read whole once another replaces it, but not the outlined book', async () => { + const project = serveProject((bookId) => usj(bookId)); + await getBookOutline('src', 'GEN'); + await getTokenizedBook('src', 'EXO'); + + await getTokenizedBook('src', 'LEV'); + await settle(); + + expect(project.subscriptions.map((s) => [s.bookId, s.unsubscribe.mock.calls.length])).toEqual([ + ['GEN', 0], + ['EXO', 1], + ['LEV', 0], + ]); + }); + it('reports an edit to the outlined book with its project and book', async () => { const project = serveProject((bookId) => usj(bookId)); const listener = listen(); @@ -574,35 +695,31 @@ describe('onOpenBookTextChanged', () => { listeners.forEach((listener) => expect(listener).toHaveBeenCalledTimes(1)); }); - it('reads the edited text when the book is next requested', async () => { - let version = 'v1'; - const project = serveProject((bookId) => usj(bookId, version)); + it('watches the book again once an edit to it is reported', async () => { + const project = serveProject((bookId) => usj(bookId)); await getBookOutline('src', 'GEN'); - version = 'v2'; + const [first] = project.subscriptions; - project.subscriptions[0].deliver(usj('GEN', version)); + first.deliver(usj('GEN', 'v2')); await settle(); - expect(parseBook(await getTokenizedBook('src', 'GEN')).textVersion).toBe('v2@en'); + expect(subscriptionsTo(project, 'GEN')).toHaveLength(2); + expect(first.unsubscribe).toHaveBeenCalled(); }); - it('reads the edited text when the held book heard of the edit first', async () => { + it('reads the edited text when the book is next requested', async () => { let version = 'v1'; const project = serveProject((bookId) => usj(bookId, version)); - const listener = listen(); await getBookOutline('src', 'GEN'); version = 'v2'; - project.subscriptions[1].deliver(usj('GEN', version)); project.subscriptions[0].deliver(usj('GEN', version)); await settle(); - expect(listener).toHaveBeenCalledTimes(1); - expect(listener).toHaveBeenCalledWith('src', 'GEN'); expect(parseBook(await getTokenizedBook('src', 'GEN')).textVersion).toBe('v2@en'); }); - it('still reports an edit after other books have been read in between', async () => { + it('reads the edited text after other books were read in between', async () => { let version = 'v1'; const project = serveProject((bookId) => usj(bookId, version)); const listener = listen(); @@ -614,153 +731,421 @@ describe('onOpenBookTextChanged', () => { project.subscriptions[0].deliver(usj('GEN', version)); await settle(); - expect(listener).toHaveBeenCalledTimes(1); expect(listener).toHaveBeenCalledWith('src', 'GEN'); expect(parseBook(await getTokenizedBook('src', 'GEN')).textVersion).toBe('v2@en'); + expect(parseOutline(await getBookOutline('src', 'GEN')).textVersion).toBe('v2@en'); + }); + + it('reports later edits through the book watched again', async () => { + const project = serveProject((bookId) => usj(bookId)); + const listener = listen(); + await getBookOutline('src', 'GEN'); + project.subscriptions[0].deliver(usj('GEN', 'v2')); + await settle(); + + subscriptionsTo(project, 'GEN')[1].deliver(usj('GEN', 'v3')); + + expect(listener).toHaveBeenCalledTimes(2); }); it('keeps the book held for another when the outlined book is edited', async () => { const project = serveProject((bookId) => usj(bookId)); await getBookOutline('src', 'GEN'); await getTokenizedBook('src', 'EXO'); - const held = project.subscriptions.at(-1); + const [exo] = subscriptionsTo(project, 'EXO'); project.subscriptions[0].deliver(usj('GEN', 'v2')); await settle(); - expect(held?.unsubscribe).not.toHaveBeenCalled(); + expect(exo.unsubscribe).not.toHaveBeenCalled(); }); - it('stops a watch that fails to read its book', async () => { - const project = serveProject(); - const outline = getBookOutline('src', 'GEN'); - await settle(); - const [failed, held] = project.subscriptions; + it('stops watching an outlined book that fails to read', async () => { + const project = serveProject(() => platformError('busy')); - failed.deliver(platformError('busy')); - held.deliver(usj('GEN')); - await outline; + await expect(getBookOutline('src', 'GEN')).rejects.toThrow('busy'); await settle(); - expect(failed.unsubscribe).toHaveBeenCalled(); + expect(project.subscriptions[0].unsubscribe).toHaveBeenCalled(); }); - it('watches the book again on the next outline once its watch fails to read it', async () => { - const project = serveProject(); + it('watches an outlined book again on the next outline once it fails to read', async () => { + let delivery: Delivery = platformError('busy'); + const project = serveProject(() => delivery); const listener = listen(); - const outline = getBookOutline('src', 'GEN'); - await settle(); - const [failed, held] = project.subscriptions; - failed.deliver(platformError('busy')); - held.deliver(usj('GEN')); - await outline; + await expect(getBookOutline('src', 'GEN')).rejects.toThrow('busy'); await settle(); + delivery = usj('GEN'); await getBookOutline('src', 'GEN'); - await settle(); - const watch = project.subscriptions[2]; - watch.deliver(usj('GEN')); - watch.deliver(usj('GEN', 'v2')); + subscriptionsTo(project, 'GEN')[1].deliver(usj('GEN', 'v2')); expect(listener).toHaveBeenCalledTimes(1); expect(listener).toHaveBeenCalledWith('src', 'GEN'); }); - it('keeps the newer watch when a replaced watch fails to read its book', async () => { + it('keeps the newer outlined book when the one it replaced fails to read', async () => { const project = serveProject(); getBookOutline('src', 'GEN').catch(() => undefined); await settle(); const replaced = project.subscriptions[0]; - getBookOutline('src', 'EXO').catch(() => undefined); + const outline = getBookOutline('src', 'EXO'); await settle(); - project.subscriptions.filter((s) => s.bookId === 'EXO').forEach((s) => s.deliver(usj('EXO'))); - const subscriptionCount = project.subscriptions.length; + subscriptionsTo(project, 'EXO')[0].deliver(usj('EXO')); + await outline; replaced.deliver(platformError('busy')); await settle(); await getBookOutline('src', 'EXO'); - expect(project.subscriptions).toHaveLength(subscriptionCount); + expect(subscriptionsTo(project, 'EXO')).toHaveLength(1); }); - it('reports nothing for a book only read whole', async () => { + it('reports nothing for an edit to a book only read whole', async () => { const project = serveProject((bookId) => usj(bookId)); const listener = listen(); await getTokenizedBook('src', 'GEN'); await getSegments('src', 'GEN', 0, 1); - project.subscriptions.forEach((subscription) => subscription.deliver(usj('GEN', 'v2'))); + project.subscriptions[0].deliver(usj('GEN', 'v2')); expect(listener).not.toHaveBeenCalled(); }); - it('watches an outlined book once however often it is outlined', async () => { + it('stops watching the previous outlined book once another is outlined', async () => { const project = serveProject((bookId) => usj(bookId)); - - await getBookOutline('src', 'GEN'); await getBookOutline('src', 'GEN'); + await getTokenizedBook('src', 'EXO'); + const [gen] = project.subscriptions; - expect(project.subscriptions).toHaveLength(2); + await getBookOutline('src', 'LEV'); + await settle(); + + expect(gen.unsubscribe).toHaveBeenCalled(); }); - it('stops watching the previous book once another is outlined', async () => { + it('reports edits to the newly outlined book', async () => { const project = serveProject((bookId) => usj(bookId)); + const listener = listen(); await getBookOutline('src', 'GEN'); - const [first] = project.subscriptions; - await getBookOutline('src', 'EXO'); - await settle(); - expect(first.unsubscribe).toHaveBeenCalled(); + subscriptionsTo(project, 'EXO')[0].deliver(usj('EXO', 'v2')); + + expect(listener).toHaveBeenCalledTimes(1); + expect(listener).toHaveBeenCalledWith('src', 'EXO'); }); - it('reports edits to the newly outlined book', async () => { + it('keeps the later outlined book open when an earlier outline reads its language tag last', async () => { const project = serveProject((bookId) => usj(bookId)); const listener = listen(); - await getBookOutline('src', 'GEN'); + const tag = Promise.withResolvers(); + project.languageTag = tag.promise; + const earlier = getBookOutline('src', 'GEN'); + await settle(); + project.languageTag = 'en'; await getBookOutline('src', 'EXO'); - const exo = project.subscriptions.find((s) => s.bookId === 'EXO'); + tag.resolve('en'); + await earlier; - exo?.deliver(usj('EXO', 'v2')); + subscriptionsTo(project, 'EXO')[0].deliver(usj('EXO', 'v2')); - expect(listener).toHaveBeenCalledTimes(1); expect(listener).toHaveBeenCalledWith('src', 'EXO'); }); - it("stops watching the project's outlined book once it closes", async () => { + it("stops watching the project's books once it closes", async () => { const project = serveProject((bookId) => usj(bookId)); await getBookOutline('src', 'GEN'); + await getTokenizedBook('src', 'EXO'); await closeProject('src'); expect(project.subscriptions.every((s) => s.unsubscribe.mock.calls.length === 1)).toBe(true); }); - it('closes a project that has an outlined book but nothing held', async () => { + it('stops watching an outlined book that is also the latest once the project closes', async () => { const project = serveProject((bookId) => usj(bookId)); await getBookOutline('src', 'GEN'); - project.subscriptions[1].deliver(usj('GEN', 'v2')); - await settle(); await closeProject('src'); - expect(project.subscriptions.every((s) => s.unsubscribe.mock.calls.length === 1)).toBe(true); + expect(project.subscriptions[0].unsubscribe).toHaveBeenCalledTimes(1); }); it('stops watching every outlined book once all projects close', async () => { const project = serveProject((bookId) => usj(bookId)); await getBookOutline('src', 'GEN'); - await getBookOutline('other', 'EXO'); + await getTokenizedBook('src', 'EXO'); + await getBookOutline('other', 'LEV'); await closeAllProjects(); expect(project.subscriptions.every((s) => s.unsubscribe.mock.calls.length === 1)).toBe(true); }); + it('reports an edit to a book the whole-text read has read', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '11'; + const listener = listen(); + readStatus(); + await settle(); + + subscriptionsTo(project, 'EXO')[0].deliver(usj('EXO', 'v2')); + await settle(); + + expect(listener).toHaveBeenCalledTimes(1); + expect(listener).toHaveBeenCalledWith('src', 'EXO'); + }); + + it('reports nothing for an error on a book that still cannot be read', async () => { + const project = serveProject(() => platformError('busy')); + project.booksPresent = '1'; + const listener = listen(); + readStatus(); + await settle(); + + subscriptionsTo(project, 'GEN').at(-1)?.deliver(platformError('busy')); + await settle(); + + expect(listener).not.toHaveBeenCalled(); + expect((await readStatus()).failedBookIds).toEqual(['GEN']); + }); + + it('reports an edit to a book that failed to read once it reads', async () => { + let delivery: Delivery = platformError('busy'); + const project = serveProject(() => delivery); + project.booksPresent = '1'; + const listener = listen(); + readStatus(); + await settle(); + + delivery = usj('GEN', 'v2'); + subscriptionsTo(project, 'GEN').at(-1)?.deliver(delivery); + await settle(); + + expect(listener).toHaveBeenCalledTimes(1); + }); + + it('reports a book the whole-text read has read once it fails to read', async () => { + let delivery: Delivery = usj('GEN'); + const project = serveProject(() => delivery); + project.booksPresent = '1'; + const listener = listen(); + readStatus(); + await settle(); + + delivery = platformError('busy'); + subscriptionsTo(project, 'GEN').at(-1)?.deliver(delivery); + await settle(); + + expect(listener).toHaveBeenCalledTimes(1); + expect((await readStatus()).failedBookIds).toEqual(['GEN']); + }); + + it("reports an edit once the whole-text read's answers include it", async () => { + tokenizeToWords(); + let version = 'v1'; + const project = serveProject((bookId) => usj(bookId, version)); + project.booksPresent = '1'; + readStatus(); + await settle(); + const answers: unknown[] = []; + listen(() => answers.push(isFormInText('src', 'gen-v2'))); + + version = 'v2'; + project.subscriptions[0].deliver(usj('GEN', version)); + await settle(); + + expect(answers).toEqual([true]); + }); + + it("reports an edit once the whole-text read's answers include it when a later edit replaces its read", async () => { + tokenizeToWords(); + const project = serveProject(); + project.booksPresent = '1'; + readStatus(); + await settle(); + subscriptionsTo(project, 'GEN')[0].deliver(usj('GEN')); + await settle(); + const answers: unknown[] = []; + listen(() => answers.push(isFormInText('src', 'gen-v1'))); + + subscriptionsTo(project, 'GEN')[0].deliver(usj('GEN', 'v2')); + await settle(); + const replaced = subscriptionsTo(project, 'GEN')[1]; + replaced.deliver(usj('GEN', 'v2')); + replaced.deliver(usj('GEN', 'v3')); + await settle(); + subscriptionsTo(project, 'GEN')[2].deliver(usj('GEN', 'v3')); + await settle(); + + expect(answers).toEqual([false]); + }); + + it('reports an edit to the outlined book once when the whole-text read has read it too', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + const listener = listen(); + await getBookOutline('src', 'GEN'); + readStatus(); + await settle(); + + subscriptionsTo(project, 'GEN').forEach((s) => s.deliver(usj('GEN', 'v2'))); + await settle(); + + expect(listener).toHaveBeenCalledTimes(1); + }); + + it('reports an edit to the outlined book the whole-text read has yet to read', async () => { + const project = serveProject(); + project.booksPresent = '1'; + const listener = listen(); + const outline = getBookOutline('src', 'GEN'); + await settle(); + project.subscriptions[0].deliver(usj('GEN')); + await outline; + readStatus(); + await settle(); + + project.subscriptions[0].deliver(usj('GEN', 'v2')); + await finishReading(project); + + expect(listener).toHaveBeenCalledWith('src', 'GEN'); + }); + + it('reports an edit to the outlined book once when the whole-text read first reads the text from before it', async () => { + const project = serveProject(); + project.booksPresent = '1'; + const listener = listen(); + const outline = getBookOutline('src', 'GEN'); + await settle(); + const [open] = project.subscriptions; + open.deliver(usj('GEN')); + await outline; + readStatus(); + await settle(); + const [, read] = subscriptionsTo(project, 'GEN'); + + open.deliver(usj('GEN', 'v2')); + read.deliver(usj('GEN')); + await settle(); + read.deliver(usj('GEN', 'v2')); + await finishReading(project); + + expect(listener).toHaveBeenCalledTimes(1); + }); + + it('reports an edit to the outlined book the whole-text read first read with the edit in it', async () => { + const project = serveProject(); + project.booksPresent = '1'; + const listener = listen(); + const outline = getBookOutline('src', 'GEN'); + await settle(); + const [open] = project.subscriptions; + open.deliver(usj('GEN')); + await outline; + readStatus(); + await settle(); + subscriptionsTo(project, 'GEN')[1].deliver(usj('GEN', 'v2')); + await settle(); + + open.deliver(usj('GEN', 'v2')); + await settle(); + await finishReading(project); + + expect(listener).toHaveBeenCalledTimes(1); + }); + + it('reports an edit to the outlined book the whole-text read could not watch', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + const listener = listen(); + await getBookOutline('src', 'GEN'); + project.subscribeError = new Error('no provider'); + readStatus(); + await settle(); + project.subscribeError = undefined; + + project.subscriptions[0].deliver(usj('GEN', 'v2')); + await settle(); + + expect(listener).toHaveBeenCalledTimes(1); + expect(listener).toHaveBeenCalledWith('src', 'GEN'); + }); + + it('serves the edited outlined book when the whole-text read hears of the edit first', async () => { + let version = 'v1'; + const project = serveProject((bookId) => usj(bookId, version)); + project.booksPresent = '1'; + await getBookOutline('src', 'GEN'); + readStatus(); + await settle(); + const [, read] = subscriptionsTo(project, 'GEN'); + const listener = listen(); + + version = 'v2'; + read.deliver(usj('GEN', version)); + await settle(); + + expect(listener).toHaveBeenCalledTimes(1); + expect(parseOutline(await getBookOutline('src', 'GEN')).textVersion).toBe('v2@en'); + }); + + it('reports nothing for an edit that lands while the project closes', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + readStatus(); + await settle(); + const listener = listen(); + + const closing = closeProject('src'); + project.subscriptions[0].deliver(usj('GEN', 'v2')); + await closing; + await settle(); + + expect(listener).not.toHaveBeenCalled(); + expect(project.subscriptions).toHaveLength(1); + }); + + it('reports nothing for an edit read again only after the project closes', async () => { + const project = serveProject(); + project.booksPresent = '1'; + readStatus(); + await settle(); + await finishReading(project); + const listener = listen(); + project.subscriptions[0].deliver(usj('GEN', 'v2')); + await settle(); + + const closing = closeProject('src'); + project.subscriptions[1].deliver(usj('GEN', 'v2')); + await closing; + await settle(); + + expect(listener).not.toHaveBeenCalled(); + }); + + it('reports each edit once, through the watch that heard it', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + readStatus(); + await settle(); + const listener = listen(); + const [replaced] = project.subscriptions; + replaced.deliver(usj('GEN', 'v2')); + await settle(); + + replaced.deliver(usj('GEN', 'v3')); + await settle(); + + expect(listener).toHaveBeenCalledTimes(1); + expect(replaced.unsubscribe).toHaveBeenCalled(); + }); + it('does not call a listener once it is removed', async () => { const project = serveProject((bookId) => usj(bookId)); const listener = jest.fn(); - const unsubscribe = onOpenBookTextChanged(listener); + const unsubscribe = onBookTextChanged(listener); await getBookOutline('src', 'GEN'); unsubscribe(); @@ -769,3 +1154,333 @@ describe('onOpenBookTextChanged', () => { expect(listener).not.toHaveBeenCalled(); }); }); + +describe('getTextReadStatus', () => { + it('answers once the books are known, before any is read', async () => { + const project = serveProject(); + project.booksPresent = '11'; + + expect(await readStatus()).toEqual({ booksRead: 0, bookCount: 2, failedBookIds: [] }); + await finishReading(project); + }); + + it("reads every book the project's text holds, under its language tag", async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '101'; + + readStatus(); + await settle(); + + expect(await readStatus()).toEqual({ booksRead: 2, bookCount: 2, failedBookIds: [] }); + expect(project.subscriptions.map((s) => s.bookId)).toEqual(['GEN', 'LEV']); + expect(extractBookFromUsj).toHaveBeenCalledWith(usj('LEV'), 'en'); + }); + + it('reports the books read so far while others are still being read', async () => { + const project = serveProject(); + project.booksPresent = '111'; + readStatus(); + await settle(); + + project.subscriptions[0].deliver(usj('GEN')); + project.subscriptions[1].deliver(usj('EXO')); + await settle(); + + expect(await readStatus()).toEqual({ + booksRead: 2, + bookCount: 3, + failedBookIds: [], + }); + await finishReading(project); + }); + + it('reads a few books at a time', async () => { + const project = serveProject(); + project.booksPresent = '111111'; + readStatus(); + await settle(); + const started = project.subscriptions.length; + + project.subscriptions[0].deliver(usj('GEN')); + await settle(); + + expect(started).toBeLessThan(6); + expect(project.subscriptions).toHaveLength(started + 1); + await finishReading(project); + }); + + it('reads the text once however often its status is asked for', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + + readStatus(); + readStatus(); + await settle(); + readStatus(); + + expect(project.subscriptions).toHaveLength(1); + }); + + it('reports a book that fails to read, and reads the rest', async () => { + const project = serveProject((bookId) => + bookId === 'EXO' ? platformError('busy') : usj(bookId), + ); + project.booksPresent = '111'; + + readStatus(); + await settle(); + + expect(await readStatus()).toEqual({ + booksRead: 3, + bookCount: 3, + failedBookIds: ['EXO'], + }); + }); + + it('counts a book the project has no text for as read', async () => { + const project = serveProject(() => undefined); + project.booksPresent = '1'; + + readStatus(); + await settle(); + + expect(await readStatus()).toEqual({ booksRead: 1, bookCount: 1, failedBookIds: [] }); + }); + + it('rejects when the books cannot be listed', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = new Error('offline'); + + await expect(readStatus()).rejects.toThrow('offline'); + expect(project.subscriptions).toHaveLength(0); + }); + + it('rejects when the project has no provider for its text', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + project.usjProviderError = new Error('no provider'); + + await expect(readStatus()).rejects.toThrow('no provider'); + expect(project.subscriptions).toHaveLength(0); + }); + + it('gets the provider of the text once for every book', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '111'; + + readStatus(); + await settle(); + + expect(project.subscriptions).toHaveLength(3); + expect( + mockPdpGet.mock.calls.filter(([type]) => type === 'platformScripture.USJ_Book'), + ).toHaveLength(1); + }); + + it('lists the books again on the request after one that could not', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = new Error('offline'); + await expect(readStatus()).rejects.toThrow('offline'); + + project.booksPresent = '1'; + readStatus(); + await settle(); + + expect(await readStatus()).toEqual({ booksRead: 1, bookCount: 1, failedBookIds: [] }); + }); + + it('clears a failed book once an edit lets it read', async () => { + let delivery: Delivery = platformError('busy'); + const project = serveProject(() => delivery); + project.booksPresent = '1'; + readStatus(); + await settle(); + + delivery = usj('GEN', 'v2'); + project.subscriptions[0].deliver(delivery); + await settle(); + + expect((await readStatus()).failedBookIds).toEqual([]); + }); + + it('reads a book it could not watch again once the outlined book is edited', async () => { + segmentStandIns(); + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + await getBookOutline('src', 'GEN'); + project.subscribeError = new Error('no provider'); + readStatus(); + await settle(); + project.subscribeError = undefined; + + project.subscriptions[0].deliver(usj('GEN', 'v2')); + await settle(); + + expect((await readStatus()).failedBookIds).toEqual([]); + }); + + it('stops watching the books once the project closes', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '11'; + readStatus(); + await settle(); + + await closeProject('src'); + + expect(project.subscriptions.every((s) => s.unsubscribe.mock.calls.length === 1)).toBe(true); + }); + + it('reads no further books once the project closes', async () => { + const project = serveProject(); + project.booksPresent = '111111'; + readStatus(); + await settle(); + const started = project.subscriptions.length; + + const closing = closeProject('src'); + project.subscriptions.forEach((s) => s.deliver(usj(s.bookId))); + await closing; + await settle(); + + expect(project.subscriptions).toHaveLength(started); + }); + + it('reads no book once the project closes while its books are listed', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + readStatus(); + + await closeProject('src'); + await settle(); + + expect(project.subscriptions).toHaveLength(0); + }); + + it('reads the text again once the project has closed', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + readStatus(); + await settle(); + await closeProject('src'); + + readStatus(); + await settle(); + + expect(await readStatus()).toEqual({ booksRead: 1, bookCount: 1, failedBookIds: [] }); + expect(project.subscriptions).toHaveLength(2); + }); + + it('stops watching the books of every project once all close', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + readStatus('src'); + readStatus('other'); + await settle(); + + await closeAllProjects(); + + expect(project.subscriptions).toHaveLength(2); + expect(project.subscriptions.every((s) => s.unsubscribe.mock.calls.length === 1)).toBe(true); + }); +}); + +describe('isFormInText', () => { + beforeEach(() => { + tokenizeToWords(); + }); + + it('answers nothing for a project whose text has not been read', () => { + expect(isFormInText('src', 'shared')).toBeUndefined(); + }); + + it('answers nothing until every book has been read', async () => { + const project = serveProject(); + project.booksPresent = '11'; + readStatus(); + await settle(); + + project.subscriptions[0].deliver(usj('GEN')); + await settle(); + + expect(isFormInText('src', 'shared')).toBeUndefined(); + await finishReading(project); + }); + + it('finds the forms any book of the text holds', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '11'; + readStatus(); + await settle(); + + expect(['shared', 'gen-v1', 'exo-v1'].map((f) => isFormInText('src', f))).toEqual([ + true, + true, + true, + ]); + }); + + it('misses a form no book of the text holds', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '11'; + readStatus(); + await settle(); + + expect(isFormInText('src', 'lev-v1')).toBe(false); + }); + + it('matches a form however it is cased', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + readStatus(); + await settle(); + + expect(isFormInText('src', 'Shared')).toBe(true); + }); + + it('answers nothing while a book has failed to read', async () => { + const project = serveProject((bookId) => + bookId === 'EXO' ? platformError('busy') : usj(bookId), + ); + project.booksPresent = '11'; + readStatus(); + await settle(); + + expect(isFormInText('src', 'gen-v1')).toBeUndefined(); + }); + + it('answers from the edited text when a read it replaced delivers older text last', async () => { + segmentStandIns(); + tokenizeToWords(); + const project = serveProject(); + project.booksPresent = '1'; + const outline = getBookOutline('src', 'GEN'); + await settle(); + const [open] = project.subscriptions; + open.deliver(usj('GEN')); + await outline; + readStatus(); + await settle(); + const [, replaced] = subscriptionsTo(project, 'GEN'); + open.deliver(usj('GEN', 'v2')); + await settle(); + + subscriptionsTo(project, 'GEN').at(-1)?.deliver(usj('GEN', 'v2')); + await settle(); + replaced.deliver(usj('GEN')); + await finishReading(project); + + expect(isFormInText('src', 'gen-v1')).toBe(false); + expect(isFormInText('src', 'gen-v2')).toBe(true); + }); + + it('answers nothing once a project closes', async () => { + const project = serveProject((bookId) => usj(bookId)); + project.booksPresent = '1'; + readStatus(); + await settle(); + + await closeProject('src'); + + expect(isFormInText('src', 'shared')).toBeUndefined(); + }); +}); diff --git a/src/__tests__/utils/form-index.test.ts b/src/__tests__/utils/form-index.test.ts new file mode 100644 index 000000000..8f2fdc264 --- /dev/null +++ b/src/__tests__/utils/form-index.test.ts @@ -0,0 +1,58 @@ +/// + +import { indexBook } from '../../utils/concordance'; +import { FormIndex, formsOfBook } from '../../utils/form-index'; +import { makeVerseBook } from '../test-helpers'; + +const BOOK = makeVerseBook([ + { sid: 'GEN 1:1', text: 'In the beginning.' }, + { sid: 'GEN 1:2', text: 'The earth' }, +]); + +describe('formsOfBook', () => { + it("holds each of the book's word forms once, normalized", () => { + expect(formsOfBook(BOOK)).toEqual(new Set(['in', 'the', 'beginning', 'earth'])); + }); + + it('holds the forms the concordance files occurrences under', () => { + expect(formsOfBook(BOOK)).toEqual(new Set(indexBook(BOOK).occurrencesByForm.keys())); + }); +}); + +describe('FormIndex', () => { + it('holds the forms of every book filed', () => { + const index = new FormIndex(); + + index.setBook('GEN', new Set(['in', 'the'])); + index.setBook('EXO', new Set(['now'])); + + expect(['in', 'the', 'now'].map((form) => index.has(form))).toEqual([true, true, true]); + }); + + it('misses a form no book filed holds', () => { + const index = new FormIndex(); + + index.setBook('GEN', new Set(['in', 'the'])); + + expect(index.has('earth')).toBe(false); + }); + + it("drops forms only a refiled book's earlier forms held", () => { + const index = new FormIndex(); + index.setBook('GEN', new Set(['in', 'the'])); + + index.setBook('GEN', new Set(['the', 'earth'])); + + expect(['in', 'the', 'earth'].map((form) => index.has(form))).toEqual([false, true, true]); + }); + + it('keeps a form another book still holds once a book is refiled without it', () => { + const index = new FormIndex(); + index.setBook('GEN', new Set(['the'])); + index.setBook('EXO', new Set(['the'])); + + index.setBook('GEN', new Set()); + + expect(index.has('the')).toBe(true); + }); +}); diff --git a/src/hooks/useSourceTextReader.ts b/src/hooks/useSourceTextReader.ts index 040a7d4a6..4e6da0888 100644 --- a/src/hooks/useSourceTextReader.ts +++ b/src/hooks/useSourceTextReader.ts @@ -1,7 +1,7 @@ import papi, { logger } from '@papi/frontend'; -import { Canon } from '@sillsdev/scripture'; import type { Book } from 'interlinearizer'; import { useCallback, useEffect, useMemo, useState } from 'react'; +import { presentBookIds } from '../utils/books-present'; import { indexBook, type BookConcordance } from '../utils/concordance'; import { perfMark, perfMeasure } from '../utils/perf-marks'; import useLatestRef from './useLatestRef'; @@ -55,20 +55,6 @@ export interface SourceText { request: () => void; } -/** - * Reads the books a `platformScripture.booksPresent` flag string marks present — one `'1'` per - * present book, indexed by canonical book number — in canonical order. Flags past the end of the - * canon mark nothing. - */ -function presentBookIds(booksPresent: string): string[] { - const bookCount = Math.min(booksPresent.length, Canon.allBookIds.length); - const ids: string[] = []; - for (let index = 0; index < bookCount; index += 1) { - if (booksPresent[index] === '1') ids.push(Canon.bookNumberToId(index + 1)); - } - return ids; -} - /** * How many books are read at once. Enough to keep the provider busy while earlier books are * indexed, few enough that the whole canon's USJ is never in memory at the same time. diff --git a/src/main.ts b/src/main.ts index 94c9f0695..d854622f7 100644 --- a/src/main.ts +++ b/src/main.ts @@ -553,6 +553,23 @@ async function getSegments( } } +/** + * Reports how far reading the source project's whole text has got, starting the read when the + * project has none. + * + * @returns Once the text's books are known, the `TextReadStatus` as JSON. + * @throws If the books or the provider of their text cannot be got. The error is logged before + * rethrowing. + */ +async function getTextReadStatus(sourceProjectId: string): Promise { + try { + return await bookService.getTextReadStatus(sourceProjectId); + } catch (e) { + logger.error(`Interlinearizer: failed to read the text of project ${sourceProjectId}`, e); + throw e; + } +} + /** * Persists the draft working buffer for a source project. Called from the WebView after every edit * (auto-save) and whenever the draft is reset (New), opened from a project, or wiped. @@ -974,6 +991,30 @@ export async function activate(context: ExecutionActivationContext): Promise + const bookTextUnsubscriber = bookService.onBookTextChanged((sourceProjectId, bookId) => analysisChangedEmitter.emit({ sourceProjectId, changeId: crypto.randomUUID(), @@ -1366,6 +1407,7 @@ export async function activate(context: ExecutionActivationContext): Promise(); +const latestBooks = new Map(); + +/** + * Each source project's open book, the one whose outline was last asked for. It stays kept while + * other books are read, and its edits are reported to {@link onBookTextChanged}'s listeners. + */ +const openBooks = new Map(); /** Tells a request whether its project closed while it was under way. */ const closeCounts = new Map(); +/** Tells an outline request whether a later one, which picks the open book instead, has started. */ +const outlineCounts = new Map(); + +/** + * How many books a whole-text read reads at once. Enough to keep the provider busy while earlier + * books are tokenized, few enough to bound the fetches in flight. + */ +const READ_CONCURRENCY = 4; + +/** A source project's whole text, read book by book into a form index, each book watched for edits. */ +interface TextRead { + /** + * The text's books in canonical order, with the tag their text is written in; rejects when they, + * or the provider of their text, could not be got. + */ + listed: Promise<{ bookIds: string[]; writingSystem: string }>; + /** The provider each book is read from, got as the books are listed. */ + usjPdp: Promise; + /** The tag the books are tokenized under; `''` until the books are known. */ + writingSystem: string; + /** Whether every book has been read once. */ + isRead: boolean; + /** Books read at least once, whether or not they failed. */ + readBookIds: Set; + /** Books whose latest read failed. */ + failedBookIds: Set; + forms: FormIndex; + /** Stops watching each book the read has started, by book id. */ + watches: Map Promise>; +} + /** - * Each source project's open book, the one whose outline was last asked for, watched apart from - * {@link heldBooks} since reading other books replaces the held one. + * Each source project's whole-text read, from the first status asked for until the project closes, + * or until its books could not be listed. */ -const openBooks = new Map(); +const textReads = new Map(); + +/** Receives the source project and book of each edit to a watched book's text. */ +type BookTextListener = (sourceProjectId: string, bookId: string) => void; -/** Receives the source project and book of each edit to an open book's text. */ -type OpenBookTextListener = (sourceProjectId: string, bookId: string) => void; +const bookTextListeners = new Set(); -const openBookTextListeners = new Set(); +function reportEdit(sourceProjectId: string, bookId: string): void { + bookTextListeners.forEach((listener) => listener(sourceProjectId, bookId)); +} /** * Resolves the writing system tag of the source project's text, `und` when the project declares @@ -61,6 +107,7 @@ function watchBookText( sourceProjectId: string, bookId: string, onChange: () => void, + usjPdp = papi.projectDataProviders.get('platformScripture.USJ_Book', sourceProjectId), ): BookTextWatch { const { promise: text, @@ -68,21 +115,19 @@ function watchBookText( reject: rejectText, } = Promise.withResolvers(); let isFirstDelivery = true; - const subscription = papi.projectDataProviders - .get('platformScripture.USJ_Book', sourceProjectId) - .then((pdp) => - // PAPI calls back after the first delivery only for text that differs from the last, or for - // an error reading it again. - pdp.subscribeBookUSJ({ book: bookId, chapterNum: 1, verseNum: 1 }, (usj) => { - if (!isFirstDelivery) { - onChange(); - return; - } - isFirstDelivery = false; - if (isPlatformError(usj)) rejectText(new Error(usj.message)); - else resolveText(usj); - }), - ); + const subscription = usjPdp.then((pdp) => + // PAPI calls back after the first delivery only for text that differs from the last, or for + // an error reading it again. + pdp.subscribeBookUSJ({ book: bookId, chapterNum: 1, verseNum: 1 }, (usj) => { + if (!isFirstDelivery) { + onChange(); + return; + } + isFirstDelivery = false; + if (isPlatformError(usj)) rejectText(new Error(usj.message)); + else resolveText(usj); + }), + ); subscription.catch(rejectText); return { @@ -99,14 +144,34 @@ function watchBookText( }; } -/** Stops watching `watch`'s book, and forgets it unless another book has replaced it. */ +/** Stops watching `watch`'s book, and forgets it wherever the source project keeps it. */ function releaseBook(sourceProjectId: string, watch: BookTextWatch): Promise { - if (heldBooks.get(sourceProjectId)?.watch === watch) heldBooks.delete(sourceProjectId); + if (latestBooks.get(sourceProjectId)?.watch === watch) latestBooks.delete(sourceProjectId); + if (openBooks.get(sourceProjectId)?.watch === watch) openBooks.delete(sourceProjectId); return watch.stop(); } +/** Stops watching `held`'s book once the source project keeps it neither as latest nor as open. */ +function dropUnlessKept(sourceProjectId: string, held: HeldBook | undefined): void { + if (!held) return; + if (latestBooks.get(sourceProjectId) === held || openBooks.get(sourceProjectId) === held) return; + held.watch.stop(); +} + function holdBook(sourceProjectId: string, bookId: string, writingSystem: string): HeldBook { - const watch = watchBookText(sourceProjectId, bookId, () => releaseBook(sourceProjectId, watch)); + const watch = watchBookText(sourceProjectId, bookId, () => { + const isOpen = openBooks.get(sourceProjectId)?.watch === watch; + releaseBook(sourceProjectId, watch); + if (!isOpen) return; + // Read again at once, so the book stays watched and a refetch started by the listeners sees + // the edit. + openBooks.set(sourceProjectId, holdBook(sourceProjectId, bookId, writingSystem)); + // The whole-text read's watch may have first read the book before the edit or after it, so the + // read itself reads the book again over a new watch and reports the edit. + const read = textReads.get(sourceProjectId); + if (read?.watches.has(bookId)) rereadTextBook(sourceProjectId, read, bookId); + else reportEdit(sourceProjectId, bookId); + }); const book = perfTrack('getTokenizedBook:usj', watch.text).then( (usj) => usj && @@ -121,11 +186,19 @@ function holdBook(sourceProjectId: string, bookId: string, writingSystem: string } /** - * Finds the source project's book tokenized under the project's language tag. Each project's latest - * book is kept, and served again until its text changes or {@link closeProject} lets it go. + * Finds the source project's book tokenized under the project's language tag, and keeps it as the + * project's latest book, and as its open book too when `isOpen` and no later outline request has + * started. A kept book is served again until its text changes, another book takes its place, or + * {@link closeProject} lets it go. */ -async function heldBook(sourceProjectId: string, bookId: string): Promise { +async function heldBook( + sourceProjectId: string, + bookId: string, + isOpen = false, +): Promise { const closeCount = closeCounts.get(sourceProjectId); + const outlineCount = (outlineCounts.get(sourceProjectId) ?? 0) + 1; + if (isOpen) outlineCounts.set(sourceProjectId, outlineCount); const writingSystem = await getWritingSystem(sourceProjectId); if (closeCounts.get(sourceProjectId) !== closeCount) { // The project closed while its tag was read, so the book is served but not kept. @@ -133,12 +206,18 @@ async function heldBook(sourceProjectId: string, bookId: string): Promise kept?.bookId === bookId && kept.writingSystem === writingSystem, + ) ?? holdBook(sourceProjectId, bookId, writingSystem); + latestBooks.set(sourceProjectId, held); + if (isOpen && outlineCounts.get(sourceProjectId) === outlineCount) + openBooks.set(sourceProjectId, held); + dropUnlessKept(sourceProjectId, latest); + dropUnlessKept(sourceProjectId, open); + return held; } /** @@ -196,50 +275,20 @@ function segmentsOf( } /** - * Calls `listener` whenever the Scripture text of a source project's open book changes, until the - * returned function is called. A project's open book is the one whose outline was last asked for, - * and a book requested once the listener is called reads the edit. - */ -export function onOpenBookTextChanged(listener: OpenBookTextListener): () => boolean { - openBookTextListeners.add(listener); - return () => openBookTextListeners.delete(listener); -} - -/** - * Makes `bookId` the source project's open book, watching it in place of the previous open book. A - * no-op when it is already the open book. + * Calls `listener` whenever the Scripture text changes in a book the service watches, until the + * returned function is called. A source project's watched books are its open book, the one whose + * outline was last asked for, and every book its whole-text read has read. Once the listener is + * called, a request for the book reads the edit, and the whole-text read's answers include it. */ -function watchOpenBook(sourceProjectId: string, bookId: string): void { - const open = openBooks.get(sourceProjectId); - if (open?.bookId === bookId) return; - open?.watch.stop(); - const watch = watchBookText(sourceProjectId, bookId, () => { - // Either watch may hear of the edit first, so the held text goes before any listener refetches. - const held = heldBooks.get(sourceProjectId); - if (held?.bookId === bookId) releaseBook(sourceProjectId, held.watch); - openBookTextListeners.forEach((listener) => listener(sourceProjectId, bookId)); - }); - // A read failure reaches whoever requests the text. Forgetting the watch lets the next outline - // watch again; stopping it ends a subscription an error delivery leaves open. - watch.text.catch(() => { - if (openBooks.get(sourceProjectId)?.watch !== watch) return; - openBooks.delete(sourceProjectId); - watch.stop(); - }); - openBooks.set(sourceProjectId, { bookId, watch }); -} - -function closeOpenBook(sourceProjectId: string): Promise { - const open = openBooks.get(sourceProjectId); - if (!open) return Promise.resolve(); - openBooks.delete(sourceProjectId); - return open.watch.stop(); +export function onBookTextChanged(listener: BookTextListener): () => boolean { + bookTextListeners.add(listener); + return () => bookTextListeners.delete(listener); } /** * Outlines a book of the source project: its segments, segmented by the boundaries * `segmentationJson` names, each without its text. The book becomes the project's open book, whose - * edits {@link onOpenBookTextChanged} reports. + * edits {@link onBookTextChanged} reports. * * @returns The `BookOutline` as JSON, or `undefined` when the project has no text for the book. * @throws If the book cannot be read or tokenized, or `segmentationJson` is not a JSON @@ -250,8 +299,7 @@ export async function getBookOutline( bookId: string, segmentationJson?: string, ): Promise { - watchOpenBook(sourceProjectId, bookId); - const held = await heldBook(sourceProjectId, bookId); + const held = await heldBook(sourceProjectId, bookId, true); const book = await held.book; if (!book) return undefined; const segments = segmentsOf(held, book, segmentationJson, 'getBookOutline'); @@ -276,8 +324,10 @@ export async function getSegments( end: number, segmentationJson?: string, ): Promise { - const kept = heldBooks.get(sourceProjectId); - const held = kept?.bookId === bookId ? kept : await heldBook(sourceProjectId, bookId); + const held = + [openBooks.get(sourceProjectId), latestBooks.get(sourceProjectId)].find( + (kept) => kept?.bookId === bookId, + ) ?? (await heldBook(sourceProjectId, bookId)); const book = await held.book; if (!book) return undefined; const segments = segmentsOf(held, book, segmentationJson, 'getSegments'); @@ -290,23 +340,190 @@ export async function getSegments( return perfTime('getSegments:serialize', () => JSON.stringify(page)); } +/** Lets go of the source project's kept copies of `bookId`, holding its open book again at once. */ +function rereadHeldBook(sourceProjectId: string, bookId: string): void { + const open = openBooks.get(sourceProjectId); + new Set([latestBooks.get(sourceProjectId), open]).forEach((held) => { + if (held?.bookId === bookId) releaseBook(sourceProjectId, held.watch); + }); + if (open?.bookId === bookId) + openBooks.set(sourceProjectId, holdBook(sourceProjectId, bookId, open.writingSystem)); +} + /** - * Lets go of the book kept for the source project, and of any a request under way would keep, and - * stops watching its open book. + * Reads `bookId` of `read`'s text into its form index, and watches the book until the read ends or + * reads it again. An edit to the book lets go of its kept copies, and is reported once the book is + * read again. + * + * @returns Once the book's text has arrived, whether this read filed the book as read or failed. */ -export async function closeProject(sourceProjectId: string): Promise { - closeCounts.set(sourceProjectId, (closeCounts.get(sourceProjectId) ?? 0) + 1); - const held = heldBooks.get(sourceProjectId); +async function readTextBook( + sourceProjectId: string, + read: TextRead, + bookId: string, +): Promise { + read.watches.get(bookId)?.(); + const { text, stop } = watchBookText( + sourceProjectId, + bookId, + () => { + if (textReads.get(sourceProjectId) !== read || read.watches.get(bookId) !== stop) return; + rereadHeldBook(sourceProjectId, bookId); + rereadTextBook(sourceProjectId, read, bookId); + }, + read.usjPdp, + ); + read.watches.set(bookId, stop); + const forms = await perfTrack('readText:usj', text) + .then((usj) => + usj + ? perfTime('readText:tokenize', () => + formsOfBook(tokenizeBook(extractBookFromUsj(usj, read.writingSystem))), + ) + : new Set(), + ) + .catch((e) => { + logger.warn(`Could not read ${bookId} of project ${sourceProjectId}: ${e}`); + return undefined; + }); + if (textReads.get(sourceProjectId) !== read) return false; + // A later read of the book reads newer text, so this one files only a book none has filed. + if (read.watches.get(bookId) !== stop && read.readBookIds.has(bookId)) return false; + if (forms) { + read.forms.setBook(bookId, forms); + read.failedBookIds.delete(bookId); + } else { + read.failedBookIds.add(bookId); + } + read.readBookIds.add(bookId); + return true; +} + +/** + * Reads `bookId` of `read`'s text again, then reports its edit unless the read has ended, a later + * read of the book will report it, or the book failed to read both before and again. + */ +async function rereadTextBook( + sourceProjectId: string, + read: TextRead, + bookId: string, +): Promise { + const wasFailed = read.failedBookIds.has(bookId); + if (!(await readTextBook(sourceProjectId, read, bookId))) return; + // Core calls back with an error after every save while a book can't be read. + if (!wasFailed || !read.failedBookIds.has(bookId)) reportEdit(sourceProjectId, bookId); +} + +/** Lists the source project's books in canonical order, with the tag their text is written in. */ +async function listBooks( + sourceProjectId: string, +): Promise<{ bookIds: string[]; writingSystem: string }> { + const basePdp = await papi.projectDataProviders.get('platform.base', sourceProjectId); + const [booksPresent, writingSystem] = await Promise.all([ + basePdp.getSetting('platformScripture.booksPresent'), + getWritingSystem(sourceProjectId), + ]); + return { bookIds: presentBookIds(booksPresent), writingSystem }; +} + +/** Reads each book of `read`'s text once the books are listed. Resolves once the read ends. */ +async function readText(sourceProjectId: string, read: TextRead): Promise { + const isCurrent = () => textReads.get(sourceProjectId) === read; + const listed = await read.listed.catch(() => { + // Dropped before the failure reaches any request, so the next request starts the read again. + if (isCurrent()) textReads.delete(sourceProjectId); + return undefined; + }); + if (!listed || !isCurrent()) return; + read.writingSystem = listed.writingSystem; + const unread = [...listed.bookIds]; + const readNext = async (): Promise => { + const bookId = unread.shift(); + if (bookId === undefined || !isCurrent()) return; + await readTextBook(sourceProjectId, read, bookId); + await readNext(); + }; + await Promise.all(Array.from({ length: READ_CONCURRENCY }, readNext)); + if (isCurrent()) read.isRead = true; +} + +/** Starts reading the source project's whole text. */ +function startTextRead(sourceProjectId: string): TextRead { + const usjPdp = papi.projectDataProviders.get('platformScripture.USJ_Book', sourceProjectId); + const read: TextRead = { + listed: Promise.all([listBooks(sourceProjectId), usjPdp]).then(([listed]) => listed), + usjPdp, + writingSystem: '', + isRead: false, + readBookIds: new Set(), + failedBookIds: new Set(), + forms: new FormIndex(), + watches: new Map(), + }; + textReads.set(sourceProjectId, read); + perfTrack('readText', readText(sourceProjectId, read)); + return read; +} + +/** + * Reports how far reading the source project's whole text has got, starting the read when the + * project has none. The read keeps a form index that {@link isFormInText} answers from, and watches + * each book it has read, reporting its edits through {@link onBookTextChanged}, until the project + * closes. + * + * @returns Once the text's books are known, the `TextReadStatus` as JSON. + * @throws If the books or the provider of their text cannot be got. The read is dropped, so the + * next request starts it again. + */ +export async function getTextReadStatus(sourceProjectId: string): Promise { + const read = textReads.get(sourceProjectId) ?? startTextRead(sourceProjectId); + const { bookIds } = await read.listed; + const { readBookIds, failedBookIds } = read; + const status: TextReadStatus = { + booksRead: readBookIds.size, + bookCount: bookIds.length, + failedBookIds: bookIds.filter((bookId) => failedBookIds.has(bookId)), + }; + return JSON.stringify(status); +} + +/** + * Whether the source project's whole text holds `form`, matched as analyses match forms. + * + * @returns `undefined` until every book has been read, and while any book's latest read has failed. + */ +export function isFormInText(sourceProjectId: string, form: string): boolean | undefined { + const read = textReads.get(sourceProjectId); + if (!read?.isRead || read.failedBookIds.size > 0) return undefined; + return read.forms.has(normalizeSurfaceForm(form)); +} + +/** Lets go of the books kept for the source project, and ends its whole-text read. */ +async function releaseProject(sourceProjectId: string): Promise { + const kept = new Set([latestBooks.get(sourceProjectId), openBooks.get(sourceProjectId)]); + const read = textReads.get(sourceProjectId); + textReads.delete(sourceProjectId); await Promise.all([ - held && releaseBook(sourceProjectId, held.watch), - closeOpenBook(sourceProjectId), + ...[...kept].map((held) => held && releaseBook(sourceProjectId, held.watch)), + ...[...(read?.watches.values() ?? [])].map((stop) => stop()), ]); } -/** Lets go of every book kept, and stops watching every open book. */ +/** + * Lets go of the books kept for the source project, and of any a request under way would keep, and + * ends its whole-text read. + */ +export async function closeProject(sourceProjectId: string): Promise { + closeCounts.set(sourceProjectId, (closeCounts.get(sourceProjectId) ?? 0) + 1); + await releaseProject(sourceProjectId); +} + +/** Lets go of every book kept, and ends every whole-text read. */ export async function closeAllProjects(): Promise { - await Promise.all([ - ...[...heldBooks].map(([sourceProjectId, { watch }]) => releaseBook(sourceProjectId, watch)), - ...[...openBooks.keys()].map(closeOpenBook), + const sourceProjectIds = new Set([ + ...latestBooks.keys(), + ...openBooks.keys(), + ...textReads.keys(), ]); + await Promise.all([...sourceProjectIds].map(releaseProject)); } diff --git a/src/types/interlinearizer.d.ts b/src/types/interlinearizer.d.ts index eea1c47af..368a9abf8 100644 --- a/src/types/interlinearizer.d.ts +++ b/src/types/interlinearizer.d.ts @@ -11,8 +11,9 @@ declare module 'papi-shared-types' { */ 'interlinearizer.onUndoFromNotification': { notificationId: string | number }; /** - * Emitted when the text of a source project's open book (the one last outlined) changes, so - * views showing it refetch it. Edits to other books emit nothing. + * Emitted when the text changes in a book the extension watches, so views showing it refetch + * it: a source project's open book (the one last outlined), and every book its whole-text read + * has read. Edits to other books emit nothing. */ 'interlinearizer.onAnalysisChanged': { sourceProjectId: string; @@ -278,6 +279,15 @@ declare module 'papi-shared-types' { segmentationJson?: string, ) => Promise; + /** + * Reports how far reading the source project's whole text has got, starting the read when the + * project has none. + * + * @returns Once the text's books are known, the `TextReadStatus` as a JSON string. + * @throws If the books or the provider of their text cannot be got. The next call tries again. + */ + 'interlinearizer.getTextReadStatus': (sourceProjectId: string) => Promise; + /** * Persists the draft working buffer for the given source project. Called from the WebView after * every edit (auto-save) and whenever the draft is reset (New), opened from a project, or diff --git a/src/types/text-read-status.ts b/src/types/text-read-status.ts new file mode 100644 index 000000000..63e56e467 --- /dev/null +++ b/src/types/text-read-status.ts @@ -0,0 +1,9 @@ +/** How far reading a source project's whole text has got. */ +export type TextReadStatus = Readonly<{ + /** How many of the text's books have been read at least once, those that failed included. */ + booksRead: number; + /** How many books the text holds. */ + bookCount: number; + /** The books whose latest read failed, in canonical order. */ + failedBookIds: readonly string[]; +}>; diff --git a/src/utils/books-present.ts b/src/utils/books-present.ts new file mode 100644 index 000000000..fb9e35c15 --- /dev/null +++ b/src/utils/books-present.ts @@ -0,0 +1,15 @@ +import { Canon } from '@sillsdev/scripture'; + +/** + * Reads the books that a `platformScripture.booksPresent` flag string marks present (one `'1'` per + * present book, indexed by canonical book number), in canonical order. Flags past the end of the + * canon mark nothing. + */ +export function presentBookIds(booksPresent: string): string[] { + const bookCount = Math.min(booksPresent.length, Canon.allBookIds.length); + const ids: string[] = []; + for (let index = 0; index < bookCount; index += 1) { + if (booksPresent[index] === '1') ids.push(Canon.bookNumberToId(index + 1)); + } + return ids; +} diff --git a/src/utils/form-index.ts b/src/utils/form-index.ts new file mode 100644 index 000000000..1098dfdaf --- /dev/null +++ b/src/utils/form-index.ts @@ -0,0 +1,43 @@ +import type { Book } from 'interlinearizer'; +import { isWordToken } from '../types/type-guards'; +import { normalizeSurfaceForm } from './analysis-identity'; + +/** Every word form a tokenized book holds, normalized as analyses match them. */ +export function formsOfBook(book: Book): Set { + const forms = new Set(); + book.segments.forEach((segment) => + segment.tokens.forEach((token) => { + if (isWordToken(token)) forms.add(normalizeSurfaceForm(token.surfaceText)); + }), + ); + return forms; +} + +/** The word forms a text holds, filed book by book so one book's forms can be replaced alone. */ +export class FormIndex { + private readonly formsByBook = new Map>(); + + /** How many filed books hold each form. */ + private readonly bookCounts = new Map(); + + /** Files `forms` as the forms `bookId` holds, in place of any filed for it before. */ + setBook(bookId: string, forms: ReadonlySet): void { + this.formsByBook.get(bookId)?.forEach((form) => { + /* v8 ignore next -- defensive: a filed form always has a count */ + const count = (this.bookCounts.get(form) ?? 1) - 1; + if (count === 0) this.bookCounts.delete(form); + else this.bookCounts.set(form, count); + }); + this.formsByBook.set(bookId, forms); + forms.forEach((form) => this.bookCounts.set(form, (this.bookCounts.get(form) ?? 0) + 1)); + } + + /** + * Whether a filed book holds `form`. + * + * @param form - Normalized as {@link formsOfBook} normalizes forms. + */ + has(form: string): boolean { + return this.bookCounts.has(form); + } +}