diff --git a/AGENTS.md b/AGENTS.md index 1a3163569..6e34f197c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,7 @@ npm run perf:bench # Time the data layer's pure functions in Node npm run perf:app # Production-build the extension and time it, WebView and extension host, in the app npm run perf:summary # Render the newest results as markdown tables npm run perf:shape # Compare analysis shape across tiers and stored projects +npm run perf:indexes # Check derived indexes and lookups against the records, after random edits ``` Only a template merge moves the dependency baseline `npm run lint:dependencies` checks against; refresh it in that same commit with `npm run template:baseline`, which writes both the copy and the `MERGED_TEMPLATE_COMMIT` id recorded beside it in [scripts/check-dependency-scope.cjs](scripts/check-dependency-scope.cjs). [README.md](README.md) has the full procedure. @@ -77,7 +78,7 @@ The WebView root component is assigned to `globalThis.webViewComponent` (not exp [src/services/projectStorage.ts](src/services/projectStorage.ts) — owns all `papi.storage` reads and writes for interlinearizer projects. Two serialization queues prevent interleaved read-modify-write races. Tests must call `resetQueuesForTesting()` between tests because module state is not cleared by `resetMocks`. -Drafts are loaded and saved through [AnalysisRepository](src/types/analysis-repository.ts), the seam a later storage backend implements. [DraftRepository](src/services/draftRepository.ts) implements it over the draft envelope and its per-book shards, and rewrites only the books that changed. It imports nothing from PAPI and reaches storage only through a `KeyValueStorage`. That lets the perf harness run it in Node, straight over the storage files ([perf/storage.ts](perf/storage.ts)). [Pt9ImportRepository](src/services/pt9ImportRepository.ts) serves a Paratext 9 import through the same seam, read-only. +Drafts are loaded and saved through [AnalysisRepository](src/types/analysis-repository.ts), the seam a later storage backend implements. [DraftRepository](src/services/draftRepository.ts) implements it over the draft envelope and its per-book shards, and rewrites only the books that changed. It imports nothing from PAPI and reaches storage only through a `KeyValueStorage`. That lets the perf harness run it in Node, straight over the storage files ([perf/storage.ts](perf/storage.ts)). [Pt9ImportRepository](src/services/pt9ImportRepository.ts) serves a Paratext 9 import through the same seam, read-only. Both answer the seam's lookups from an [AnalysisIndex](src/services/analysisIndex.ts) built from the records on first use. `DraftRepository` keeps it current through `apply`, and `checkIndexes` compares it with one built afresh. [The conformance suite](src/__tests__/analysis-repository-conformance.ts) defines the seam: every `AnalysisRepository` implementation must pass it, and [draftRepository.conformance.test.ts](src/__tests__/services/draftRepository.conformance.test.ts) runs it against `DraftRepository` and `Pt9ImportRepository`. A behavior every implementation owes goes in the suite; one particular to an implementation, such as which shards a save rewrites, goes in that implementation's own tests. A later backend runs the suite by passing `describeAnalysisRepositoryConformance` a factory that returns a fresh `ConformanceStore` for each test. diff --git a/package.json b/package.json index e58cf4829..81f13c3be 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "perf:app": "cross-env E2E_BUILD_SCRIPT=build:production playwright test --config e2e-tests/playwright-perf.config.ts --project=app", "perf:summary": "node -r ts-node/register perf/summary/cli.ts", "perf:shape": "node --max-old-space-size=8192 -r ts-node/register perf/shape/cli.ts", + "perf:indexes": "node --max-old-space-size=12288 -r ts-node/register perf/indexes/cli.ts", "core:start": "npm --prefix ../paranext-core start", "core:stop": "npm --prefix ../paranext-core stop", "core:reset": "npm run core:stop && node ./scripts/delete-temp-files.cjs --core --ext", diff --git a/perf/README.md b/perf/README.md index 82c681835..a30b61422 100644 --- a/perf/README.md +++ b/perf/README.md @@ -10,6 +10,7 @@ Measures the extension rather than tests it: the data layer's pure functions in - `npm run perf:app` times the extension in the app. - `npm run perf:summary` renders the newest results of each kind as markdown tables. - `npm run perf:shape -- ...` compares the shape of analyses; see [Dataset shape](#dataset-shape). +- `npm run perf:indexes -- [tier]...` applies random edits and undoes to each tier in Node, then checks that the derived indexes match ones built afresh and that lookups match a scan of the stored records. It exits nonzero on any difference. `PERF_TIERS` (comma-separated tier names) and `PERF_RUNS` (10 by default) narrow a run. The `large` tier's draft exceeds one websocket message, so only the Node benchmarks can measure it. diff --git a/perf/indexes/cli.ts b/perf/indexes/cli.ts new file mode 100644 index 000000000..3f39cee51 --- /dev/null +++ b/perf/indexes/cli.ts @@ -0,0 +1,345 @@ +/* eslint-disable no-console -- a command-line tool reports to its terminal */ +import type { AssignmentStatus, DraftProject, TextAnalysis } from 'interlinearizer'; +import { DraftRepository } from '../../src/services/draftRepository'; +import { EditHistory } from '../../src/services/editHistory'; +import type { + AnalysisRepository, + ChangeSet, + KeyValueStorage, + WarningLogger, +} from '../../src/types/analysis-repository'; +import { bookOfRef } from '../../src/utils/analysis-book'; +import { normalizeSurfaceForm } from '../../src/utils/analysis-identity'; +import { chance, createRng, type Rng } from '../dataset/rng'; +import { readDraftJson } from '../dataset/store'; +import { TIERS, type TierSpec } from '../dataset/tiers'; + +const STATUSES: AssignmentStatus[] = ['approved', 'suggested', 'candidate', 'rejected', 'stale']; + +const STAMP = '2026-01-01T00:00:00.000Z'; + +/** How many token refs the random edits draw from, so the final lookups revisit what they changed. */ +const EDITED_TOKENS = 200; + +/** How many keys of each kind the final lookups add from the whole analysis. */ +const SAMPLED_KEYS = 50; + +function memoryStorage(): KeyValueStorage { + const records = new Map(); + return { + async read(key) { + return records.get(key); + }, + async write(key, data) { + records.set(key, data); + }, + async delete(key) { + records.delete(key); + }, + }; +} + +/** Random edits break invariants that a load reports, so its warnings say nothing here. */ +const SILENT: WarningLogger = { warn: () => {} }; + +function pick(rng: Rng, items: readonly T[]): T { + return items[Math.floor(rng() * items.length)]; +} + +function sample(rng: Rng, items: readonly T[], count: number): T[] { + return Array.from({ length: Math.min(count, items.length) }, () => pick(rng, items)); +} + +/** A random edit to the links on one of `tokenRefs`, read through `repository` as it stands. */ +async function randomChanges( + repository: AnalysisRepository, + tokenRefs: readonly string[], + surfaceTexts: readonly string[], + rng: Rng, + step: number, +): Promise { + const tokenRef = pick(rng, tokenRefs); + const { tokenAnalysisLinks, phraseAnalysisLinks } = await repository.findLinksByToken([tokenRef]); + const changes: ChangeSet = { intent: `check-${step}` }; + const link = tokenAnalysisLinks.length > 0 ? pick(rng, tokenAnalysisLinks) : undefined; + const roll = rng(); + if (link && roll < 0.3) { + const status = pick( + rng, + STATUSES.filter((s) => s !== link.status), + ); + changes.tokenAnalysisLinks = { puts: [{ ...link, status }], deletes: [] }; + } else if (link && roll < 0.45) { + changes.tokenAnalysisLinks = { puts: [], deletes: [link.id] }; + } else if (link && roll < 0.7) { + // A respelled payload moves to another form's entry. + const [payload] = await repository.getTokenAnalyses([link.analysisId]); + if (payload) { + const surfaceText = pick(rng, surfaceTexts); + changes.tokenAnalyses = { puts: [{ ...payload, surfaceText }], deletes: [] }; + } + } else { + const surfaceText = pick(rng, surfaceTexts); + const analysisId = `check-ta-${step}`; + changes.tokenAnalyses = { + puts: [{ id: analysisId, createdAt: STAMP, updatedAt: STAMP, surfaceText }], + deletes: [], + }; + changes.tokenAnalysisLinks = { + puts: [ + { + id: `check-tl-${step}`, + analysisId, + createdAt: STAMP, + updatedAt: STAMP, + status: 'suggested', + token: { tokenRef, surfaceText }, + }, + ], + deletes: [], + }; + } + if (phraseAnalysisLinks.length > 0 && chance(rng, 0.2)) { + const phrase = pick(rng, phraseAnalysisLinks); + changes.phraseAnalysisLinks = chance(rng, 0.5) + ? { puts: [], deletes: [phrase.id] } + : { puts: [{ ...phrase, status: pick(rng, STATUSES) }], deletes: [] }; + } + return changes; +} + +/** Records in id order, serialized, so two answers compare without regard to order. */ +function canonical(records: readonly { id: string }[]): string { + return JSON.stringify([...records].sort((a, b) => a.id.localeCompare(b.id))); +} + +/** + * Compares the repository's lookups with a scan of `stored`, over keys revisiting `tokenRefs` and + * drawn afresh from `stored`. + * + * @returns The lookups whose answers differ from the scan's. + */ +async function lookupMismatches( + repository: AnalysisRepository, + stored: TextAnalysis, + tokenRefs: readonly string[], + rng: Rng, +): Promise { + const ids = (records: readonly { id: string }[]) => [ + ...sample( + rng, + records.map(({ id }) => id), + SAMPLED_KEYS, + ), + 'check-missing', + ]; + const refs = new Set([ + ...tokenRefs, + ...sample( + rng, + stored.tokenAnalysisLinks.map((l) => l.token.tokenRef), + SAMPLED_KEYS, + ), + ]); + const segmentIds = new Set( + sample( + rng, + stored.segmentAnalysisLinks.map((l) => l.segmentId), + SAMPLED_KEYS, + ), + ); + const analysisIds = new Set([ + ...ids(stored.tokenAnalyses), + ...ids(stored.phraseAnalyses), + ...ids(stored.segmentAnalyses), + ]); + const books = new Set( + sample( + rng, + stored.tokenAnalysisLinks.map((l) => bookOfRef(l.token.tokenRef)), + 2, + ), + ); + const statuses = new Set(sample(rng, STATUSES, 2)); + const forms = new Set( + sample( + rng, + [...stored.tokenAnalyses, ...stored.phraseAnalyses].map((a) => + normalizeSurfaceForm(a.surfaceText), + ), + SAMPLED_KEYS, + ), + ); + const tokenIds = ids(stored.tokenAnalyses); + const phraseIds = ids(stored.phraseAnalyses); + const segmentPayloadIds = ids(stored.segmentAnalyses); + const phraseBook = (l: { tokens: { tokenRef: string }[] }) => bookOfRef(l.tokens[0].tokenRef); + + const byToken = await repository.findLinksByToken([...refs]); + const toAnalyses = await repository.findLinksToAnalyses([...analysisIds]); + const inBooks = await repository.findLinksInBooks([...books]); + const inBooksByStatus = await repository.findLinksInBooks([...books], [...statuses]); + const checks: [string, readonly { id: string }[], readonly { id: string }[]][] = [ + [ + 'getTokenAnalyses', + await repository.getTokenAnalyses(tokenIds), + stored.tokenAnalyses.filter((a) => tokenIds.includes(a.id)), + ], + [ + 'getPhraseAnalyses', + await repository.getPhraseAnalyses(phraseIds), + stored.phraseAnalyses.filter((a) => phraseIds.includes(a.id)), + ], + [ + 'getSegmentAnalyses', + await repository.getSegmentAnalyses(segmentPayloadIds), + stored.segmentAnalyses.filter((a) => segmentPayloadIds.includes(a.id)), + ], + [ + 'findLinksByToken (token links)', + byToken.tokenAnalysisLinks, + stored.tokenAnalysisLinks.filter((l) => refs.has(l.token.tokenRef)), + ], + [ + 'findLinksByToken (phrase links)', + byToken.phraseAnalysisLinks, + stored.phraseAnalysisLinks.filter((l) => l.tokens.some((t) => refs.has(t.tokenRef))), + ], + [ + 'findLinksBySegment', + await repository.findLinksBySegment([...segmentIds]), + stored.segmentAnalysisLinks.filter((l) => segmentIds.has(l.segmentId)), + ], + [ + 'findLinksToAnalyses', + [ + ...toAnalyses.tokenAnalysisLinks, + ...toAnalyses.phraseAnalysisLinks, + ...toAnalyses.segmentAnalysisLinks, + ], + [ + ...stored.tokenAnalysisLinks, + ...stored.phraseAnalysisLinks, + ...stored.segmentAnalysisLinks, + ].filter((l) => analysisIds.has(l.analysisId)), + ], + [ + 'findLinksInBooks', + [ + ...inBooks.tokenAnalysisLinks, + ...inBooks.phraseAnalysisLinks, + ...inBooks.segmentAnalysisLinks, + ], + [ + ...stored.tokenAnalysisLinks.filter((l) => books.has(bookOfRef(l.token.tokenRef))), + ...stored.phraseAnalysisLinks.filter((l) => books.has(phraseBook(l))), + ...stored.segmentAnalysisLinks.filter((l) => books.has(bookOfRef(l.segmentId))), + ], + ], + [ + 'findLinksInBooks (by status)', + [ + ...inBooksByStatus.tokenAnalysisLinks, + ...inBooksByStatus.phraseAnalysisLinks, + ...inBooksByStatus.segmentAnalysisLinks, + ], + [ + ...stored.tokenAnalysisLinks.filter((l) => books.has(bookOfRef(l.token.tokenRef))), + ...stored.phraseAnalysisLinks.filter((l) => books.has(phraseBook(l))), + ...stored.segmentAnalysisLinks.filter((l) => books.has(bookOfRef(l.segmentId))), + ].filter((l) => statuses.has(l.status)), + ], + [ + 'findTokenAnalysesByForm', + await repository.findTokenAnalysesByForm([...forms]), + stored.tokenAnalyses.filter((a) => forms.has(normalizeSurfaceForm(a.surfaceText))), + ], + [ + 'findPhraseAnalysesByForm', + await repository.findPhraseAnalysesByForm([...forms]), + stored.phraseAnalyses.filter((a) => forms.has(normalizeSurfaceForm(a.surfaceText))), + ], + ]; + return checks + .filter(([, answered, scanned]) => canonical(answered) !== canonical(scanned)) + .map(([name]) => name); +} + +/** + * Loads a tier, applies random edits and undoes some, then compares the kept indexes with ones + * built afresh and the lookups with a scan of what storage holds. + * + * @returns Whether the indexes and lookups agreed throughout. + */ +async function checkTier(tier: TierSpec): Promise { + const draft: DraftProject = JSON.parse(readDraftJson(tier.name)); + const { sourceProjectId, analysis } = draft; + const storage = memoryStorage(); + await new DraftRepository(storage, sourceProjectId, SILENT).replaceAll(draft); + const repository = new DraftRepository(storage, sourceProjectId, SILENT); + const history = new EditHistory(repository); + const rng = createRng('indexes', tier.name); + const links = analysis.tokenAnalysisLinks.length; + // Each apply rewrites the draft, so larger tiers get fewer. + const steps = Math.min(300, Math.max(15, Math.round(5e6 / links))); + const tokenRefs = [ + ...new Set( + sample( + rng, + analysis.tokenAnalysisLinks.map((l) => l.token.tokenRef), + EDITED_TOKENS, + ), + ), + ]; + const surfaceTexts = sample( + rng, + analysis.tokenAnalyses.map((a) => a.surfaceText), + EDITED_TOKENS, + ); + + await repository.getTokenAnalyses([]); + const atLoad = await repository.checkIndexes(); + let undos = 0; + for (let step = 0; step < steps; step += 1) { + if (step > 0 && chance(rng, 0.25)) { + // eslint-disable-next-line no-await-in-loop -- each edit builds on the one before + await history.undo(); + undos += 1; + } else { + // eslint-disable-next-line no-await-in-loop -- each edit builds on the one before + await history.apply(await randomChanges(repository, tokenRefs, surfaceTexts, rng, step)); + } + } + const after = await repository.checkIndexes(); + const reader = new DraftRepository(storage, sourceProjectId, SILENT); + const stored = (await reader.exportAll()).draft.analysis; + const mismatches = await lookupMismatches(repository, stored, tokenRefs, rng); + + console.log( + `${tier.name.padEnd(15)} ${links.toLocaleString('en').padStart(8)} links ${steps} steps (${undos} undos) differences: ${atLoad.length} at load, ${after.length} after lookup mismatches: ${mismatches.length}`, + ); + if (mismatches.length > 0) console.log(` mismatched: ${mismatches.join(', ')}`); + return atLoad.length === 0 && after.length === 0 && mismatches.length === 0; +} + +async function main(): Promise { + const requested = process.argv.slice(2); + const unknown = requested.filter((name) => !TIERS.some((t) => t.name === name)); + if (unknown.length > 0) { + console.error( + `Unknown tiers: ${unknown.join(', ')}. Expected any of ${TIERS.map((t) => t.name).join(', ')}.`, + ); + process.exitCode = 1; + return; + } + const tiers = TIERS.filter((t) => requested.length === 0 || requested.includes(t.name)); + let agreed = true; + // eslint-disable-next-line no-restricted-syntax -- tiers run one at a time to bound memory + for (const tier of tiers) { + // eslint-disable-next-line no-await-in-loop -- tiers run one at a time to bound memory + if (!(await checkTier(tier))) agreed = false; + } + if (!agreed) process.exitCode = 1; +} + +main(); diff --git a/src/__tests__/analysis-repository-conformance.ts b/src/__tests__/analysis-repository-conformance.ts index 5205a8e66..b4a41e3f8 100644 --- a/src/__tests__/analysis-repository-conformance.ts +++ b/src/__tests__/analysis-repository-conformance.ts @@ -1,10 +1,16 @@ /// import type { DraftProject, TextAnalysis } from 'interlinearizer'; -import type { AnalysisRepository, ChangeSet, DraftExport } from '../types/analysis-repository'; +import type { + AnalysisLinks, + AnalysisRepository, + ChangeSet, + DraftExport, +} from '../types/analysis-repository'; import { emptyAnalysis, emptyDraft } from '../types/empty-factories'; import { CURRENT_MODEL_VERSION } from '../types/model-version'; import { ReadOnlyAnalysisError } from '../types/read-only-analysis-error'; +import { normalizeSurfaceForm } from '../utils/analysis-identity'; import { scopeRecordOf } from '../utils/change-set'; import { FIXTURE_STAMPS, makePhraseLink, makeTokenLink } from './test-helpers'; @@ -393,6 +399,22 @@ function changeInPlace(draft: DraftProject): void { }); } +/** The ids of `records`, sorted, since lookups promise no order. */ +function idsOf(records: readonly { id: string }[]): string[] { + return records.map(({ id }) => id).sort(); +} + +/** The ids of each kind of link in `links`, sorted, since lookups promise no order. */ +function linkIdsOf(links: Partial) { + return { + ...(links.tokenAnalysisLinks && { tokenAnalysisLinks: idsOf(links.tokenAnalysisLinks) }), + ...(links.phraseAnalysisLinks && { phraseAnalysisLinks: idsOf(links.phraseAnalysisLinks) }), + ...(links.segmentAnalysisLinks && { + segmentAnalysisLinks: idsOf(links.segmentAnalysisLinks), + }), + }; +} + /** Expects `value` to survive a JSON round trip unchanged and to hold no `null`. */ function expectJsonPlain(value: unknown): void { expect(JSON.parse(JSON.stringify(value))).toStrictEqual(value); @@ -1197,6 +1219,239 @@ export function describeAnalysisRepositoryConformance( }); }); + describe('lookups', () => { + /** + * A repository over {@link fullDraft}, saved by another repository so its first lookup loads + * the draft from storage. + */ + async function overFullDraft(): Promise { + await (await store.open(SOURCE)).replaceAll(fullDraft()); + return store.open(SOURCE); + } + + it('gets each payload kind by id, once per id, skipping ids with nothing stored', async () => { + const repository = await overFullDraft(); + + await expect( + repository.getTokenAnalyses(['ta-and', 'ta-missing', 'ta-and']).then(idsOf), + ).resolves.toStrictEqual(['ta-and']); + await expect(repository.getPhraseAnalyses(['pa-in-the'])).resolves.toStrictEqual( + fullAnalysis().phraseAnalyses, + ); + await expect( + repository.getSegmentAnalyses(['sa-exo', 'sa-missing']), + ).resolves.toStrictEqual(fullAnalysis().segmentAnalyses); + }); + + it('finds the token links on tokens, and the phrase links touching them, each once', async () => { + const repository = await overFullDraft(); + + await expect( + repository.findLinksByToken(['GEN 1:1:0', 'GEN 1:1:2', 'GEN 1:1:4']).then(linkIdsOf), + ).resolves.toStrictEqual({ + tokenAnalysisLinks: ['tl-gen-and'], + phraseAnalysisLinks: ['pa-in-the'], + }); + }); + + it('finds the segment links on segments', async () => { + const repository = await overFullDraft(); + + await expect( + repository.findLinksBySegment(['EXO 1:1', 'GEN 1:1']).then(idsOf), + ).resolves.toStrictEqual(['sl-exo']); + }); + + it('finds the links of every kind to analyses', async () => { + const repository = await overFullDraft(); + + await expect( + repository.findLinksToAnalyses(['ta-and', 'pa-in-the', 'sa-exo']).then(linkIdsOf), + ).resolves.toStrictEqual({ + tokenAnalysisLinks: ['tl-exo-and', 'tl-gen-and'], + phraseAnalysisLinks: ['pa-in-the'], + segmentAnalysisLinks: ['sl-exo'], + }); + }); + + it('finds the links of every kind into books', async () => { + const repository = await overFullDraft(); + + await expect(repository.findLinksInBooks(['GEN']).then(linkIdsOf)).resolves.toStrictEqual({ + tokenAnalysisLinks: ['tl-gen-and', 'tl-gen-beginning'], + phraseAnalysisLinks: ['pa-in-the'], + segmentAnalysisLinks: [], + }); + }); + + it('finds the links into books with the statuses asked for', async () => { + const repository = await overFullDraft(); + + await expect( + repository.findLinksInBooks(['GEN', 'EXO'], ['suggested']).then(linkIdsOf), + ).resolves.toStrictEqual({ + tokenAnalysisLinks: ['tl-gen-beginning'], + phraseAnalysisLinks: [], + segmentAnalysisLinks: [], + }); + }); + + it('finds the payloads whose surface text normalizes to a form', async () => { + const repository = await overFullDraft(); + await repository.apply({ + intent: 'addCapitalized', + tokenAnalyses: { + puts: [{ id: 'ta-And', ...FIXTURE_STAMPS, surfaceText: 'And' }], + deletes: [], + }, + }); + + await expect( + repository.findTokenAnalysesByForm([normalizeSurfaceForm('AND')]).then(idsOf), + ).resolves.toStrictEqual(['ta-And', 'ta-and']); + await expect( + repository.findPhraseAnalysesByForm([normalizeSurfaceForm('In The')]).then(idsOf), + ).resolves.toStrictEqual(['pa-in-the']); + }); + + it('finds nothing in a draft never saved', async () => { + const repository = await store.open(SOURCE); + + await expect(repository.findLinksInBooks(['GEN']).then(linkIdsOf)).resolves.toStrictEqual({ + tokenAnalysisLinks: [], + phraseAnalysisLinks: [], + segmentAnalysisLinks: [], + }); + await expect(repository.getTokenAnalyses(['ta-and'])).resolves.toStrictEqual([]); + }); + + it('answers from the records the applied changes leave', async () => { + const repository = await overFullDraft(); + await repository.findLinksInBooks(['GEN']); + + await repository.apply(mixedChanges()); + + await expect( + repository.findLinksToAnalyses(['ta-beginning', 'ta-god']).then(linkIdsOf), + ).resolves.toStrictEqual({ + tokenAnalysisLinks: ['tl-gen-god'], + phraseAnalysisLinks: [], + segmentAnalysisLinks: [], + }); + await expect( + repository.findLinksBySegment(['GEN 1:1', 'EXO 1:1']).then(idsOf), + ).resolves.toStrictEqual(['sl-gen']); + await expect( + repository.findLinksByToken(['GEN 1:1:2', 'GEN 1:1:13']).then(linkIdsOf), + ).resolves.toStrictEqual({ + tokenAnalysisLinks: [], + phraseAnalysisLinks: ['pa-the-beginning'], + }); + await expect(repository.getTokenAnalyses(['ta-and'])).resolves.toStrictEqual([ + { ...fullAnalysis().tokenAnalyses[0], gloss: { en: 'and then' } }, + ]); + }); + + it('files an edited record under its new keys only', async () => { + const repository = await overFullDraft(); + await repository.findLinksInBooks(['GEN']); + const [, , beginning] = fullAnalysis().tokenAnalysisLinks; + + await repository.apply({ + intent: 'approve', + tokenAnalysisLinks: { puts: [{ ...beginning, status: 'approved' }], deletes: [] }, + }); + + await expect( + repository.findLinksInBooks(['GEN'], ['suggested']).then(linkIdsOf), + ).resolves.toMatchObject({ tokenAnalysisLinks: [] }); + await expect( + repository.findLinksInBooks(['GEN'], ['approved']).then(linkIdsOf), + ).resolves.toMatchObject({ tokenAnalysisLinks: ['tl-gen-and', 'tl-gen-beginning'] }); + }); + + it('answers from a draft another repository replaced, once reloaded', async () => { + const repository = await overFullDraft(); + await repository.findLinksInBooks(['GEN']); + await (await store.open(SOURCE)).replaceAll(exodusOnlyDraft()); + + await repository.exportAll(); + + await expect( + repository.findLinksInBooks(['GEN', 'EXO']).then(linkIdsOf), + ).resolves.toStrictEqual({ + tokenAnalysisLinks: ['tl-exo-and'], + phraseAnalysisLinks: [], + segmentAnalysisLinks: ['sl-exo'], + }); + }); + + it('keeps the records of different source projects apart', async () => { + await overFullDraft(); + + const other = await store.open('other-project'); + + await expect(other.getTokenAnalyses(['ta-and'])).resolves.toStrictEqual([]); + }); + + it('returns records the caller may change without changing what is stored', async () => { + const repository = await overFullDraft(); + + const [and] = await repository.getTokenAnalyses(['ta-and']); + and.surfaceText = 'changed'; + const { tokenAnalysisLinks } = await repository.findLinksToAnalyses(['ta-and']); + tokenAnalysisLinks[0].status = 'rejected'; + + await expect(repository.getTokenAnalyses(['ta-and'])).resolves.toStrictEqual([ + fullAnalysis().tokenAnalyses[0], + ]); + await expect( + repository.findLinksInBooks(['GEN', 'EXO'], ['rejected']).then(linkIdsOf), + ).resolves.toMatchObject({ tokenAnalysisLinks: [] }); + }); + + it('answers and applies unaffected by changes to a draft it exported', async () => { + const repository = await overFullDraft(); + changeInPlace((await repository.exportAll()).draft); + + await expect(repository.getTokenAnalyses(['ta-and'])).resolves.toStrictEqual([ + fullAnalysis().tokenAnalyses[0], + ]); + await repository.apply(puttingNewRecords()); + + await expect( + reload(store).then(({ analysis }) => + analysis.tokenAnalyses.find(({ id }) => id === 'ta-and'), + ), + ).resolves.toStrictEqual(fullAnalysis().tokenAnalyses[0]); + }); + + it('keeps indexes that match its records through applies, undo and redo', async () => { + const repository = await overFullDraft(); + await repository.findLinksInBooks(['GEN']); + + const { inverse } = await repository.apply(mixedChanges()); + const { inverse: redo } = await repository.apply(inverse); + await repository.apply(redo); + await repository.apply(puttingNewRecords()); + + await expect(repository.checkIndexes()).resolves.toStrictEqual([]); + }); + + it('answers from a read-only repository', async () => { + const repository = await store.openReadOnly(fullDraft()); + + await expect( + repository.findLinksToAnalyses(['ta-and']).then(linkIdsOf), + ).resolves.toStrictEqual({ + tokenAnalysisLinks: ['tl-exo-and', 'tl-gen-and'], + phraseAnalysisLinks: [], + segmentAnalysisLinks: [], + }); + await expect(repository.checkIndexes()).resolves.toStrictEqual([]); + }); + }); + describe('capabilities', () => { it("lets a source project's draft be written", async () => { expect((await store.open(SOURCE)).capabilities.readOnly).toBe(false); @@ -1236,6 +1491,16 @@ export function describeAnalysisRepositoryConformance( ); }); + it('answers lookups unaffected by changes to a draft it exported', async () => { + const repository = await store.openReadOnly(fullDraft()); + + changeInPlace((await repository.exportAll()).draft); + + await expect(repository.getTokenAnalyses(['ta-and'])).resolves.toStrictEqual([ + fullAnalysis().tokenAnalyses[0], + ]); + }); + it('refuses a save, leaving what it loads unchanged', async () => { const repository = await store.openReadOnly(fullDraft()); diff --git a/src/__tests__/main.test.ts b/src/__tests__/main.test.ts index 5d00fe95b..b24169bbe 100644 --- a/src/__tests__/main.test.ts +++ b/src/__tests__/main.test.ts @@ -400,6 +400,7 @@ describe('main', () => { 'interlinearizer.setAnalysisLanguage', 'interlinearizer.undoEdit', 'interlinearizer.redoEdit', + 'interlinearizer.checkIndexes', 'interlinearizer.openSelectProjectModal', 'interlinearizer.openNewProjectModal', 'interlinearizer.openProjectInfoModal', @@ -1827,6 +1828,7 @@ describe('main', () => { describe('session commands', () => { /** A repository handing out `draft`, as storage would open one. */ function stubRepository(draft: DraftProject, readOnly = false): AnalysisRepository { + const noLinks = { tokenAnalysisLinks: [], phraseAnalysisLinks: [], segmentAnalysisLinks: [] }; return { capabilities: { readOnly }, exportAll: async () => ({ draft, unreadableBooks: [] }), @@ -1834,6 +1836,18 @@ describe('main', () => { replaceAll: async () => [], markClean: async () => {}, apply: async (changes) => ({ inverse: changes }), + getTokenAnalyses: async () => [], + getPhraseAnalyses: async () => [], + getSegmentAnalyses: async () => [], + findLinksByToken: async () => noLinks, + findLinksBySegment: async () => [], + findLinksToAnalyses: async () => noLinks, + findLinksInBooks: async () => noLinks, + findTokenAnalysesByForm: async () => [], + findPhraseAnalysesByForm: async () => [], + checkIndexes: async () => [ + { index: 'tokenAnalyses', key: 'ta-1', missing: ['ta-1'], extra: [] }, + ], }; } @@ -1860,6 +1874,9 @@ describe('main', () => { redoEdit: handler<(sessionId: string) => Promise>( 'interlinearizer.redoEdit', ), + checkIndexes: handler<(sessionId: string) => Promise>( + 'interlinearizer.checkIndexes', + ), }; } @@ -2022,6 +2039,27 @@ describe('main', () => { ); }); + it("replies with the differences the session's repository finds in its indexes", async () => { + const { openSession, checkIndexes } = await activateSessionCommands(); + const { sessionId } = JSON.parse(await openSession('src-project')); + + await expect(checkIndexes(sessionId)).resolves.toBe( + JSON.stringify([{ index: 'tokenAnalyses', key: 'ta-1', missing: ['ta-1'], extra: [] }]), + ); + }); + + it('logs the error and rethrows when the indexes cannot be checked', async () => { + const { checkIndexes } = await activateSessionCommands(); + + await expect(checkIndexes('not-a-session')).rejects.toThrow( + 'no session not-a-session is open', + ); + expect(__mockLogger.error).toHaveBeenCalledWith( + 'Interlinearizer: failed to check the indexes of session not-a-session', + expect.any(Error), + ); + }); + it('closes every session on deactivation', async () => { const { openSession } = await activateSessionCommands(); await openSession('src-project'); diff --git a/src/__tests__/services/analysisIndex.test.ts b/src/__tests__/services/analysisIndex.test.ts new file mode 100644 index 000000000..61529869a --- /dev/null +++ b/src/__tests__/services/analysisIndex.test.ts @@ -0,0 +1,177 @@ +/// + +import type { TextAnalysis } from 'interlinearizer'; +import { AnalysisIndex } from '../../services/analysisIndex'; +import { emptyAnalysis } from '../../types/empty-factories'; +import { normalizeSurfaceForm } from '../../utils/analysis-identity'; +import { FIXTURE_STAMPS, makePhraseLink } from '../test-helpers'; + +const link = { ...FIXTURE_STAMPS, status: 'approved' as const }; + +/** Records in GEN and EXO, with a heading's segment in GEN and a suggested link in EXO. */ +function analysis(): TextAnalysis { + return { + segmentAnalyses: [{ id: 'sa-1', ...FIXTURE_STAMPS, surfaceText: 'The Creation' }], + segmentAnalysisLinks: [{ ...link, id: 'sl-1', analysisId: 'sa-1', segmentId: 'GEN 1:1/s1' }], + tokenAnalyses: [ + { id: 'ta-and', ...FIXTURE_STAMPS, surfaceText: 'and' }, + { id: 'ta-And', ...FIXTURE_STAMPS, surfaceText: 'And' }, + ], + tokenAnalysisLinks: [ + { + ...link, + id: 'tl-gen', + analysisId: 'ta-and', + token: { tokenRef: 'GEN 1:1:0', surfaceText: 'and' }, + }, + { + ...link, + id: 'tl-exo', + analysisId: 'ta-And', + status: 'suggested', + token: { tokenRef: 'EXO 1:1:0', surfaceText: 'And' }, + }, + ], + phraseAnalyses: [{ id: 'pa-1', ...FIXTURE_STAMPS, surfaceText: 'In the' }], + phraseAnalysisLinks: [makePhraseLink('pa-1', ['GEN 1:1:4', 'GEN 1:1:7'])], + }; +} + +const ids = (records: readonly { id: string }[]) => records.map(({ id }) => id).sort(); + +describe('AnalysisIndex', () => { + it('gets payloads of each kind by id', () => { + const index = new AnalysisIndex(analysis()); + + expect(ids(index.getTokenAnalyses(['ta-and', 'missing']))).toStrictEqual(['ta-and']); + expect(ids(index.getPhraseAnalyses(['pa-1']))).toStrictEqual(['pa-1']); + expect(ids(index.getSegmentAnalyses(['sa-1']))).toStrictEqual(['sa-1']); + }); + + it('finds token links by token ref, and phrase links by any of their tokens', () => { + const { tokenAnalysisLinks, phraseAnalysisLinks } = new AnalysisIndex( + analysis(), + ).findLinksByToken(['GEN 1:1:0', 'GEN 1:1:7']); + + expect(ids(tokenAnalysisLinks)).toStrictEqual(['tl-gen']); + expect(ids(phraseAnalysisLinks)).toStrictEqual(['pa-1']); + }); + + it('finds segment links by segment id', () => { + expect(ids(new AnalysisIndex(analysis()).findLinksBySegment(['GEN 1:1/s1']))).toStrictEqual([ + 'sl-1', + ]); + }); + + it('finds the links of every kind to analyses', () => { + const links = new AnalysisIndex(analysis()).findLinksToAnalyses(['ta-and', 'pa-1', 'sa-1']); + + expect(ids(links.tokenAnalysisLinks)).toStrictEqual(['tl-gen']); + expect(ids(links.phraseAnalysisLinks)).toStrictEqual(['pa-1']); + expect(ids(links.segmentAnalysisLinks)).toStrictEqual(['sl-1']); + }); + + it('files a heading segment link and a phrase link under their books', () => { + const links = new AnalysisIndex(analysis()).findLinksInBooks(['GEN']); + + expect(ids(links.tokenAnalysisLinks)).toStrictEqual(['tl-gen']); + expect(ids(links.phraseAnalysisLinks)).toStrictEqual(['pa-1']); + expect(ids(links.segmentAnalysisLinks)).toStrictEqual(['sl-1']); + }); + + it('finds the links into books with the statuses asked for', () => { + const links = new AnalysisIndex(analysis()).findLinksInBooks(['GEN', 'EXO'], ['suggested']); + + expect(ids(links.tokenAnalysisLinks)).toStrictEqual(['tl-exo']); + expect(links.phraseAnalysisLinks).toStrictEqual([]); + expect(links.segmentAnalysisLinks).toStrictEqual([]); + }); + + it('finds payloads by normalized form', () => { + const index = new AnalysisIndex(analysis()); + + expect(ids(index.findTokenAnalysesByForm([normalizeSurfaceForm('AND')]))).toStrictEqual([ + 'ta-And', + 'ta-and', + ]); + expect(ids(index.findPhraseAnalysesByForm(['in the']))).toStrictEqual(['pa-1']); + }); + + it('keeps every lookup current as changes apply', () => { + const index = new AnalysisIndex(analysis()); + + index.apply({ + intent: 'edit', + tokenAnalyses: { + puts: [{ id: 'ta-and', ...FIXTURE_STAMPS, surfaceText: 'but' }], + deletes: [], + }, + tokenAnalysisLinks: { puts: [], deletes: ['tl-exo'] }, + phraseAnalyses: { puts: [], deletes: ['pa-1'] }, + phraseAnalysisLinks: { puts: [], deletes: ['pa-1'] }, + segmentAnalyses: { puts: [], deletes: ['sa-1'] }, + segmentAnalysisLinks: { puts: [], deletes: ['sl-1'] }, + }); + + expect(ids(index.findTokenAnalysesByForm(['and']))).toStrictEqual(['ta-And']); + expect(ids(index.findTokenAnalysesByForm(['but']))).toStrictEqual(['ta-and']); + expect(index.findLinksInBooks(['GEN', 'EXO'])).toStrictEqual({ + tokenAnalysisLinks: [analysis().tokenAnalysisLinks[0]], + phraseAnalysisLinks: [], + segmentAnalysisLinks: [], + }); + expect(index.findPhraseAnalysesByForm(['in the'])).toStrictEqual([]); + expect(index.getSegmentAnalyses(['sa-1'])).toStrictEqual([]); + }); + + describe('differencesFrom', () => { + it('reports none against the analysis it was built from', () => { + expect(new AnalysisIndex(analysis()).differencesFrom(analysis())).toStrictEqual([]); + }); + + it('reports none once kept current through changes the analysis also took', () => { + const index = new AnalysisIndex(analysis()); + const changed = analysis(); + changed.tokenAnalysisLinks.pop(); + + index.apply({ intent: 'delete', tokenAnalysisLinks: { puts: [], deletes: ['tl-exo'] } }); + + expect(index.differencesFrom(changed)).toStrictEqual([]); + }); + + it('reports the records and entries of every kind that an analysis holds differently', () => { + const differences = new AnalysisIndex(emptyAnalysis()).differencesFrom(analysis()); + + expect(new Set(differences.map(({ index }) => index))).toStrictEqual( + new Set([ + 'tokenAnalyses', + 'tokenAnalyses by form', + 'phraseAnalyses', + 'phraseAnalyses by form', + 'segmentAnalyses', + 'tokenAnalysisLinks', + 'tokenAnalysisLinks by analysisId', + 'tokenAnalysisLinks by book', + 'tokenAnalysisLinks by bookStatus', + 'tokenAnalysisLinks by tokenRef', + 'phraseAnalysisLinks', + 'phraseAnalysisLinks by analysisId', + 'phraseAnalysisLinks by book', + 'phraseAnalysisLinks by bookStatus', + 'phraseAnalysisLinks by tokenRef', + 'segmentAnalysisLinks', + 'segmentAnalysisLinks by analysisId', + 'segmentAnalysisLinks by book', + 'segmentAnalysisLinks by bookStatus', + 'segmentAnalysisLinks by segmentId', + ]), + ); + expect(differences).toContainEqual({ + index: 'tokenAnalysisLinks by bookStatus', + key: 'EXO/suggested', + missing: ['tl-exo'], + extra: [], + }); + }); + }); +}); diff --git a/src/__tests__/services/analysisSessions.test.ts b/src/__tests__/services/analysisSessions.test.ts index 0d5688c03..0ebc62366 100644 --- a/src/__tests__/services/analysisSessions.test.ts +++ b/src/__tests__/services/analysisSessions.test.ts @@ -56,10 +56,14 @@ function makeSessions(projects: InterlinearProject[] = []) { releaseDraftRepository: jest.fn, [string]>(async () => {}), openPt9ImportRepository: jest.fn( (interlinearProjectId: string) => - new Pt9ImportRepository(interlinearProjectId, async () => { - const project = projects.find(({ id }) => id === interlinearProjectId); - return project && JSON.parse(JSON.stringify(project)); - }), + new Pt9ImportRepository( + interlinearProjectId, + async () => { + const project = projects.find(({ id }) => id === interlinearProjectId); + return project && JSON.parse(JSON.stringify(project)); + }, + () => 0, + ), ), getInterfaceLanguages: jest.fn(async (): Promise => []), } satisfies SessionHost; diff --git a/src/__tests__/services/draftRepository.conformance.test.ts b/src/__tests__/services/draftRepository.conformance.test.ts index 4ec8e65db..7251c30fe 100644 --- a/src/__tests__/services/draftRepository.conformance.test.ts +++ b/src/__tests__/services/draftRepository.conformance.test.ts @@ -90,10 +90,14 @@ describeAnalysisRepositoryConformance('DraftRepository with Pt9ImportRepository' pt9Import: { fileHashes: {}, importedAt: '2026-01-01T00:00:00.000Z' }, }; await memory.write(projectKey(IMPORT_ID), JSON.stringify(project)); - return new Pt9ImportRepository(IMPORT_ID, async () => { - const raw = await memory.read(projectKey(IMPORT_ID)); - return raw === undefined ? undefined : JSON.parse(raw); - }); + return new Pt9ImportRepository( + IMPORT_ID, + async () => { + const raw = await memory.read(projectKey(IMPORT_ID)); + return raw === undefined ? undefined : JSON.parse(raw); + }, + () => 0, + ); }, }; }); diff --git a/src/__tests__/services/draftRepository.test.ts b/src/__tests__/services/draftRepository.test.ts index 04c4561ea..00a7628db 100644 --- a/src/__tests__/services/draftRepository.test.ts +++ b/src/__tests__/services/draftRepository.test.ts @@ -360,4 +360,113 @@ describe('DraftRepository', () => { expect(records.get(draftAnalysisKey('src', 'GEN'))).not.toContain('and then'); }); }); + + describe('lookups', () => { + it('load the draft once, answering later lookups from what they kept', async () => { + const { storage } = memoryStorage(); + await new DraftRepository(storage, 'src', silentLogger()).replaceAll(twoBookDraft()); + const repository = new DraftRepository(storage, 'src', silentLogger()); + const read = jest.spyOn(storage, 'read'); + + await repository.findLinksInBooks(['GEN']); + const readsAfterFirst = read.mock.calls.length; + await repository.findLinksToAnalyses(['ta-and']); + + expect(readsAfterFirst).toBeGreaterThan(0); + expect(read).toHaveBeenCalledTimes(readsAfterFirst); + }); + + it.each(['exportForCopy', 'markClean'] as const)( + 'keep what they kept across %s', + async (call) => { + const { storage } = memoryStorage(); + const repository = new DraftRepository(storage, 'src', silentLogger()); + await repository.replaceAll({ ...twoBookDraft(), dirty: true }); + await repository.findLinksInBooks(['GEN']); + await repository[call](); + const read = jest.spyOn(storage, 'read'); + + await repository.findLinksInBooks(['GEN']); + + expect(read).not.toHaveBeenCalled(); + }, + ); + + it('read the draft again after exportAll', async () => { + const { storage } = memoryStorage(); + const repository = new DraftRepository(storage, 'src', silentLogger()); + await repository.replaceAll(twoBookDraft()); + await repository.findLinksInBooks(['GEN']); + await repository.exportAll(); + const read = jest.spyOn(storage, 'read'); + + await repository.findLinksInBooks(['GEN']); + + expect(read).toHaveBeenCalled(); + }); + + it('read the draft again after a whole save', async () => { + const { storage } = memoryStorage(); + const repository = new DraftRepository(storage, 'src', silentLogger()); + await repository.exportAll(); + await repository.replaceAll(twoBookDraft()); + const read = jest.spyOn(storage, 'read'); + + await repository.findLinksInBooks(['GEN']); + + expect(read).toHaveBeenCalled(); + }); + + it("return a book's links in stored order after an edit replaces one", async () => { + const { storage } = memoryStorage(); + const [genesis] = twoBookAnalysis().tokenAnalysisLinks; + const later = { + ...genesis, + id: 'tl-GEN-later', + token: { ...genesis.token, tokenRef: 'GEN 1:1!4' }, + }; + const repository = new DraftRepository(storage, 'src', silentLogger()); + await repository.replaceAll({ + ...twoBookDraft(), + analysis: { ...twoBookAnalysis(), tokenAnalysisLinks: [genesis, later] }, + }); + await repository.findLinksInBooks(['GEN']); + + await repository.apply({ + intent: 'reject', + tokenAnalysisLinks: { puts: [{ ...genesis, status: 'rejected' }], deletes: [] }, + }); + + const { tokenAnalysisLinks } = await repository.findLinksInBooks(['GEN']); + const stored = (await new DraftRepository(storage, 'src', silentLogger()).exportAll()).draft + .analysis.tokenAnalysisLinks; + expect(tokenAnalysisLinks.map(({ id }) => id)).toStrictEqual(['tl-GEN', 'tl-GEN-later']); + expect(stored.map(({ id }) => id)).toStrictEqual(['tl-GEN', 'tl-GEN-later']); + }); + + it('answer from what an apply left without loading the draft again', async () => { + const { storage } = memoryStorage(); + const repository = new DraftRepository(storage, 'src', silentLogger()); + await repository.apply({ + intent: 'add', + tokenAnalyses: { puts: twoBookAnalysis().tokenAnalyses, deletes: [] }, + }); + const read = jest.spyOn(storage, 'read'); + + await expect(repository.getTokenAnalyses(['ta-and'])).resolves.toStrictEqual( + twoBookAnalysis().tokenAnalyses, + ); + expect(read).not.toHaveBeenCalled(); + }); + }); + + describe('checkIndexes', () => { + it('reports nothing while it keeps no index', async () => { + const { storage } = memoryStorage(); + const repository = new DraftRepository(storage, 'src', silentLogger()); + await repository.replaceAll(twoBookDraft()); + + await expect(repository.checkIndexes()).resolves.toStrictEqual([]); + }); + }); }); diff --git a/src/__tests__/services/indexedRepository.test.ts b/src/__tests__/services/indexedRepository.test.ts new file mode 100644 index 000000000..79ca69dbe --- /dev/null +++ b/src/__tests__/services/indexedRepository.test.ts @@ -0,0 +1,70 @@ +/// + +import type { TextAnalysis } from 'interlinearizer'; +import { AnalysisIndex } from '../../services/analysisIndex'; +import { type IndexedAnalysis, IndexedRepository } from '../../services/indexedRepository'; +import { emptyAnalysis } from '../../types/empty-factories'; +import { FIXTURE_STAMPS } from '../test-helpers'; + +/** Answers lookups from an index over `records`, reporting it as kept when `keep` is `'kept'`. */ +class StubRepository extends IndexedRepository { + readonly kept: IndexedAnalysis | undefined; + + private readonly index: AnalysisIndex; + + constructor(records: TextAnalysis, keep: 'kept' | 'none') { + super(); + this.index = new AnalysisIndex(records); + this.kept = keep === 'kept' ? { analysis: records, index: this.index } : undefined; + } + + protected async indexed(): Promise { + return this.index; + } + + protected keptIndex(): IndexedAnalysis | undefined { + return this.kept; + } +} + +function analysis(): TextAnalysis { + return { + ...emptyAnalysis(), + tokenAnalyses: [{ id: 'ta-and', ...FIXTURE_STAMPS, surfaceText: 'and' }], + }; +} + +describe('IndexedRepository', () => { + it('answers lookups from its index', async () => { + await expect( + new StubRepository(analysis(), 'none').getTokenAnalyses(['ta-and']), + ).resolves.toStrictEqual(analysis().tokenAnalyses); + }); + + it('hands out copies of the records its index holds', async () => { + const repository = new StubRepository(analysis(), 'none'); + + const [and] = await repository.getTokenAnalyses(['ta-and']); + and.surfaceText = 'changed'; + + await expect(repository.getTokenAnalyses(['ta-and'])).resolves.toStrictEqual( + analysis().tokenAnalyses, + ); + }); + + it('reports the differences of the index it keeps from its analysis', async () => { + const repository = new StubRepository(analysis(), 'kept'); + repository.kept?.analysis.tokenAnalyses.pop(); + + await expect(repository.checkIndexes()).resolves.toContainEqual({ + index: 'tokenAnalyses', + key: 'ta-and', + missing: [], + extra: ['ta-and'], + }); + }); + + it('reports nothing while it keeps no index', async () => { + await expect(new StubRepository(analysis(), 'none').checkIndexes()).resolves.toStrictEqual([]); + }); +}); diff --git a/src/__tests__/services/projectStorage.test.ts b/src/__tests__/services/projectStorage.test.ts index 24a6a63ab..80094fd0f 100644 --- a/src/__tests__/services/projectStorage.test.ts +++ b/src/__tests__/services/projectStorage.test.ts @@ -24,6 +24,7 @@ import { updateProjectMetadata, } from '../../services/projectStorage'; import { summarizeProject } from '../../services/projectSummaries'; +import type { AnalysisRepository } from '../../types/analysis-repository'; import { ReadOnlyAnalysisError } from '../../types/read-only-analysis-error'; import { emptyAnalysis, emptyDraft } from '../../types/empty-factories'; import { removeBookFromAnalysis } from '../../utils/analysis-book'; @@ -766,6 +767,55 @@ describe('projectStorage', () => { expect(__mockWriteUserData).not.toHaveBeenCalled(); }); + it('makes a repository already opened over a deleted import refuse lookups', async () => { + const imported: InterlinearProject = { + ...makeStubProject('import-id'), + pt9Import: { fileHashes: {}, importedAt: '2026-02-01T00:00:00.000Z' }, + }; + mockListedProject(imported); + const repository = openPt9ImportRepository(token, 'import-id'); + await repository.getTokenAnalyses(['ta-1']); + mockStore({ projectSummaries: storedSummaries(imported) }); + + await deleteProject(token, 'import-id'); + + await expect(repository.getTokenAnalyses(['ta-1'])).rejects.toThrow( + 'no Paratext 9 import is stored as project import-id', + ); + }); + + it('keeps no import a repository read while the import was being deleted', async () => { + const imported: InterlinearProject = { + ...makeStubProject('import-id'), + pt9Import: { fileHashes: {}, importedAt: '2026-02-01T00:00:00.000Z' }, + }; + mockListedProject(imported); + let markDeletionStarted = () => {}; + const deletionStarted = new Promise((resolve) => { + markDeletionStarted = resolve; + }); + let finishDeletion = () => {}; + __mockDeleteUserData.mockImplementation( + () => + new Promise((resolve) => { + markDeletionStarted(); + finishDeletion = resolve; + }), + ); + const repository = openPt9ImportRepository(token, 'import-id'); + + const deletion = deleteProject(token, 'import-id'); + await deletionStarted; + await repository.getTokenAnalyses(['ta-1']); + mockStore({}); + finishDeletion(); + await deletion; + + await expect(repository.getTokenAnalyses(['ta-1'])).rejects.toThrow( + 'no Paratext 9 import is stored as project import-id', + ); + }); + it('unlists the project when its record is already missing', async () => { __mockDeleteUserData.mockRejectedValue(enoentError()); mockStore({ projectSummaries: listed }); @@ -2915,6 +2965,42 @@ describe('projectStorage', () => { expect(analysis.tokenAnalysisLinks).toStrictEqual([]); }); + it.each([ + ['getTokenAnalyses', (r: AnalysisRepository) => r.getTokenAnalyses(['analysis-GEN']), 1], + ['getPhraseAnalyses', (r: AnalysisRepository) => r.getPhraseAnalyses(['p']), 0], + ['getSegmentAnalyses', (r: AnalysisRepository) => r.getSegmentAnalyses(['s']), 0], + ['findLinksBySegment', (r: AnalysisRepository) => r.findLinksBySegment(['GEN 1:1']), 0], + [ + 'findLinksByToken', + (r: AnalysisRepository) => + r.findLinksByToken(['GEN 1:1!0']).then((l) => l.tokenAnalysisLinks), + 1, + ], + [ + 'findLinksToAnalyses', + (r: AnalysisRepository) => + r.findLinksToAnalyses(['analysis-GEN']).then((l) => l.tokenAnalysisLinks), + 1, + ], + [ + 'findLinksInBooks', + (r: AnalysisRepository) => + r.findLinksInBooks(['GEN'], ['approved']).then((l) => l.tokenAnalysisLinks), + 1, + ], + [ + 'findTokenAnalysesByForm', + (r: AnalysisRepository) => r.findTokenAnalysesByForm(['word-gen']), + 1, + ], + ['findPhraseAnalysesByForm', (r: AnalysisRepository) => r.findPhraseAnalysesByForm(['x']), 0], + ['checkIndexes', (r: AnalysisRepository) => r.checkIndexes(), 0], + ])('answers %s from the draft saveDraft saved', async (_name, lookup, count) => { + await saveDraft(token, 'src-proj', makeDraftSpanningBooks('src-proj', 'GEN')); + + await expect(lookup(openDraftRepository(token, 'src-proj'))).resolves.toHaveLength(count); + }); + it('waits for a draft call already made before reading', async () => { let finishWrite: () => void = () => {}; __mockWriteUserData.mockImplementationOnce( @@ -3161,6 +3247,29 @@ describe('projectStorage', () => { ); }); + it('makes a repository already opened over the import answer from the new one', async () => { + writeThroughStore({ + projectSummaries: storedSummaries(importedProject), + 'project:import-id': JSON.stringify(importedProject), + }); + const repository = openPt9ImportRepository(token, 'import-id'); + await repository.getTokenAnalyses(['pt9:ta:GEN 1:1:0:0']); + + await savePt9Import( + token, + 'src-project', + 'Paratext 9 Interlinear', + 'Imported from Paratext 9.', + ['en'], + newAnalysis, + NEW_PROVENANCE, + ); + + await expect(repository.getTokenAnalyses(['pt9:ta:GEN 1:1:0:0'])).resolves.toStrictEqual( + newAnalysis.tokenAnalyses, + ); + }); + it("brings the import's listed summary up to date", async () => { mockStore({ projectSummaries: storedSummaries(importedProject), diff --git a/src/__tests__/services/pt9ImportRepository.test.ts b/src/__tests__/services/pt9ImportRepository.test.ts index 8a8c29420..de2d88628 100644 --- a/src/__tests__/services/pt9ImportRepository.test.ts +++ b/src/__tests__/services/pt9ImportRepository.test.ts @@ -18,10 +18,14 @@ function makeImport(overrides: Partial = {}): InterlinearPro }; } +/** The revision of an import that no sync replaces. */ +const NEVER_SYNCED = () => 0; + function repositoryOver(project: InterlinearProject | undefined) { return new Pt9ImportRepository( 'import-id', async () => project && JSON.parse(JSON.stringify(project)), + NEVER_SYNCED, ); } @@ -77,9 +81,13 @@ describe('Pt9ImportRepository', () => { }); it('rethrows a failed read', async () => { - const repository = new Pt9ImportRepository('import-id', async () => { - throw new Error('disk full'); - }); + const repository = new Pt9ImportRepository( + 'import-id', + async () => { + throw new Error('disk full'); + }, + NEVER_SYNCED, + ); await expect(repository.exportAll()).rejects.toThrow('disk full'); }); @@ -97,9 +105,9 @@ describe('Pt9ImportRepository', () => { it('refuses, reading nothing', async () => { const readProject = jest.fn(async () => makeImport()); - await expect(new Pt9ImportRepository('import-id', readProject).markClean()).rejects.toThrow( - ReadOnlyAnalysisError, - ); + await expect( + new Pt9ImportRepository('import-id', readProject, NEVER_SYNCED).markClean(), + ).rejects.toThrow(ReadOnlyAnalysisError); expect(readProject).not.toHaveBeenCalled(); }); }); @@ -107,7 +115,11 @@ describe('Pt9ImportRepository', () => { describe('replaceAll', () => { it('refuses every save', async () => { const readProject = jest.fn(async () => makeImport()); - const repository: AnalysisRepository = new Pt9ImportRepository('import-id', readProject); + const repository: AnalysisRepository = new Pt9ImportRepository( + 'import-id', + readProject, + NEVER_SYNCED, + ); await expect(repository.replaceAll(emptyDraft('src-project'))).rejects.toThrow( ReadOnlyAnalysisError, @@ -119,7 +131,11 @@ describe('Pt9ImportRepository', () => { describe('apply', () => { it('refuses every change', async () => { const readProject = jest.fn(async () => makeImport()); - const repository: AnalysisRepository = new Pt9ImportRepository('import-id', readProject); + const repository: AnalysisRepository = new Pt9ImportRepository( + 'import-id', + readProject, + NEVER_SYNCED, + ); await expect(repository.apply({ intent: 'writeGloss' })).rejects.toThrow( ReadOnlyAnalysisError, @@ -127,4 +143,104 @@ describe('Pt9ImportRepository', () => { expect(readProject).not.toHaveBeenCalled(); }); }); + + describe('lookups', () => { + const imported = () => + makeImport({ + analysis: { + ...emptyAnalysis(), + tokenAnalyses: [{ id: 'ta-1', ...FIXTURE_STAMPS, surfaceText: 'and' }], + }, + }); + + it("answer from the import's records, reading it once", async () => { + const readProject = jest.fn(async () => imported()); + const repository = new Pt9ImportRepository('import-id', readProject, NEVER_SYNCED); + + await expect(repository.getTokenAnalyses(['ta-1'])).resolves.toStrictEqual( + imported().analysis.tokenAnalyses, + ); + await repository.findTokenAnalysesByForm(['and']); + + expect(readProject).toHaveBeenCalledTimes(1); + }); + + it('keep what they read across a load', async () => { + const readProject = jest.fn(async () => imported()); + const repository = new Pt9ImportRepository('import-id', readProject, NEVER_SYNCED); + await repository.getTokenAnalyses(['ta-1']); + + await repository.exportAll(); + await repository.getTokenAnalyses(['ta-1']); + + expect(readProject).toHaveBeenCalledTimes(2); + }); + + it('read the import again once a sync replaces it', async () => { + let revision = 0; + const synced = makeImport({ + analysis: { + ...emptyAnalysis(), + tokenAnalyses: [{ id: 'ta-1', ...FIXTURE_STAMPS, surfaceText: 'but' }], + }, + }); + const readProject = jest.fn(async () => (revision === 0 ? imported() : synced)); + const repository = new Pt9ImportRepository('import-id', readProject, () => revision); + await repository.getTokenAnalyses(['ta-1']); + + revision = 1; + + await expect(repository.getTokenAnalyses(['ta-1'])).resolves.toStrictEqual( + synced.analysis.tokenAnalyses, + ); + expect(readProject).toHaveBeenCalledTimes(2); + }); + + it('read the import again after a sync that lands during a read', async () => { + let revision = 0; + const readProject = jest.fn(async () => { + revision += 1; + return imported(); + }); + const repository = new Pt9ImportRepository('import-id', readProject, () => revision); + + await repository.getTokenAnalyses(['ta-1']); + await repository.getTokenAnalyses(['ta-1']); + + expect(readProject).toHaveBeenCalledTimes(2); + }); + + it('refuse when no import is stored', async () => { + await expect(repositoryOver(undefined).getTokenAnalyses(['ta-1'])).rejects.toThrow( + 'no Paratext 9 import is stored as project import-id', + ); + }); + }); + + describe('checkIndexes', () => { + it('reports nothing before a lookup', async () => { + await expect(repositoryOver(makeImport()).checkIndexes()).resolves.toStrictEqual([]); + }); + + it('reports nothing after a lookup', async () => { + const repository = repositoryOver(makeImport()); + await repository.getTokenAnalyses(['ta-1']); + + await expect(repository.checkIndexes()).resolves.toStrictEqual([]); + }); + + it('reports nothing once a sync replaces the import', async () => { + let revision = 0; + const repository = new Pt9ImportRepository( + 'import-id', + async () => makeImport(), + () => revision, + ); + await repository.getTokenAnalyses(['ta-1']); + + revision = 1; + + await expect(repository.checkIndexes()).resolves.toStrictEqual([]); + }); + }); }); diff --git a/src/__tests__/services/recordTable.test.ts b/src/__tests__/services/recordTable.test.ts new file mode 100644 index 000000000..23e1cf388 --- /dev/null +++ b/src/__tests__/services/recordTable.test.ts @@ -0,0 +1,107 @@ +/// + +import { RecordTable } from '../../services/recordTable'; + +type Item = { id: string; tags: string[]; note?: string }; + +const keyers = { tag: (item: Item) => item.tags }; + +function tableOf(...items: Item[]) { + return new RecordTable('items', keyers, items); +} + +describe('RecordTable', () => { + it('gets records by id, once per id, skipping ids with nothing stored', () => { + const a = { id: 'a', tags: [] }; + + expect(tableOf(a).get(['a', 'missing', 'a'])).toStrictEqual([a]); + }); + + it('finds the records filed under any of the keys, each once', () => { + const a = { id: 'a', tags: ['x', 'y'] }; + const b = { id: 'b', tags: ['y'] }; + const c = { id: 'c', tags: ['z'] }; + + expect(tableOf(a, b, c).find('tag', ['x', 'y', 'missing'])).toStrictEqual([a, b]); + }); + + it('files a record once under a key its keyer returns twice', () => { + const table = tableOf({ id: 'a', tags: ['x', 'x'] }); + + table.apply({ puts: [], deletes: ['a'] }); + + expect(table.find('tag', ['x'])).toStrictEqual([]); + expect(table.differencesFrom(tableOf())).toStrictEqual([]); + }); + + it('files a put record under its new keys only', () => { + const table = tableOf({ id: 'a', tags: ['x'] }); + const moved = { id: 'a', tags: ['y'] }; + + table.apply({ puts: [moved], deletes: [] }); + + expect(table.find('tag', ['x'])).toStrictEqual([]); + expect(table.find('tag', ['y'])).toStrictEqual([moved]); + }); + + it('keeps a replaced record in its place under the keys it keeps', () => { + const b = { id: 'b', tags: ['x'] }; + const table = tableOf({ id: 'a', tags: ['x'] }, b); + const replaced = { id: 'a', tags: ['x', 'y'], note: 'changed' }; + + table.apply({ puts: [replaced], deletes: [] }); + + expect(table.find('tag', ['x'])).toStrictEqual([replaced, b]); + expect(table.find('tag', ['y'])).toStrictEqual([replaced]); + }); + + it('unfiles a deleted record, skipping an id with nothing stored', () => { + const table = tableOf({ id: 'a', tags: ['x'] }); + + table.apply({ puts: [], deletes: ['a', 'missing'] }); + + expect(table.get(['a'])).toStrictEqual([]); + expect(table.find('tag', ['x'])).toStrictEqual([]); + }); + + it('applies nothing for absent changes', () => { + const table = tableOf({ id: 'a', tags: ['x'] }); + + table.apply(undefined); + + expect(table.differencesFrom(tableOf({ id: 'a', tags: ['x'] }))).toStrictEqual([]); + }); + + describe('differencesFrom', () => { + it('reports none against a table holding the same records', () => { + const items = [ + { id: 'a', tags: ['x'] }, + { id: 'b', tags: ['x', 'y'] }, + ]; + + expect(tableOf(...items).differencesFrom(tableOf(...items))).toStrictEqual([]); + }); + + it('reports a record the kept table lacks, and the entries filing it', () => { + expect(tableOf().differencesFrom(tableOf({ id: 'a', tags: ['x'] }))).toStrictEqual([ + { index: 'items', key: 'a', missing: ['a'], extra: [] }, + { index: 'items by tag', key: 'x', missing: ['a'], extra: [] }, + ]); + }); + + it('reports a record only the kept table holds, and the entries filing it', () => { + expect(tableOf({ id: 'a', tags: ['x'] }).differencesFrom(tableOf())).toStrictEqual([ + { index: 'items', key: 'a', missing: [], extra: ['a'] }, + { index: 'items by tag', key: 'x', missing: [], extra: ['a'] }, + ]); + }); + + it('reports a record whose kept copy differs as both missing and extra', () => { + const kept = tableOf({ id: 'a', tags: [], note: 'old' }); + + expect(kept.differencesFrom(tableOf({ id: 'a', tags: [], note: 'new' }))).toStrictEqual([ + { index: 'items', key: 'a', missing: ['a'], extra: ['a'] }, + ]); + }); + }); +}); diff --git a/src/__tests__/utils/change-set.test.ts b/src/__tests__/utils/change-set.test.ts index 26f0cb087..a8f19f1ec 100644 --- a/src/__tests__/utils/change-set.test.ts +++ b/src/__tests__/utils/change-set.test.ts @@ -113,22 +113,6 @@ describe('applyChangeSet', () => { expect(stored).toStrictEqual(draftOf(payload('a'))); }); - it('holds copies of the records the changes put', () => { - const put = payload('b'); - const scope = scopeRecordOf(draftOf()); - const { draft } = applyChangeSet(draftOf(), { - intent: 'add', - scope, - tokenAnalyses: { puts: [put], deletes: [] }, - }); - - put.surfaceText = 'changed'; - scope.analysisLanguages.push('fr'); - - expect(draft.analysis.tokenAnalyses).toStrictEqual([payload('b')]); - expect(draft.analysisLanguages).toStrictEqual(['en']); - }); - it('refuses an id both put and deleted', () => { expect(() => applyChangeSet(draftOf(payload('a')), { diff --git a/src/__tests__/utils/json-copy.test.ts b/src/__tests__/utils/json-copy.test.ts new file mode 100644 index 000000000..ba872ce68 --- /dev/null +++ b/src/__tests__/utils/json-copy.test.ts @@ -0,0 +1,32 @@ +/// + +import { jsonCopy } from '../../utils/json-copy'; + +function original() { + return { + gloss: { en: 'and' }, + morphemes: [{ form: 'and' }], + confidence: 0.5, + morphemesStale: false, + // eslint-disable-next-line no-null/no-null -- JSON carries null + pos: null, + }; +} + +describe('jsonCopy', () => { + it('copies a JSON-plain value deeply, sharing nothing with it', () => { + const value = original(); + + const copy = jsonCopy(value); + copy.gloss.en = 'changed'; + copy.morphemes[0].form = 'changed'; + + expect(copy).not.toBe(value); + expect(copy).toStrictEqual({ + ...original(), + gloss: { en: 'changed' }, + morphemes: [{ form: 'changed' }], + }); + expect(value).toStrictEqual(original()); + }); +}); diff --git a/src/main.ts b/src/main.ts index 6d5af1dcb..039043714 100644 --- a/src/main.ts +++ b/src/main.ts @@ -807,6 +807,24 @@ async function redoEdit(sessionId: string): Promise { } } +/** + * Reports where a session's derived indexes differ from ones built afresh from its records, for + * debugging. + * + * @returns A JSON string of the differences; `"[]"` when there are none. + * @throws If the session is not open. The error is logged before rethrowing. + */ +async function checkIndexes(sessionId: string): Promise { + try { + return JSON.stringify( + await sessions.run(sessionId, (session) => session.repository.checkIndexes()), + ); + } catch (e) { + logger.error(`Interlinearizer: failed to check the indexes of session ${sessionId}`, e); + throw e; + } +} + /** * Returns a summary of every interlinearizer project for the given source project as a JSON string. * The WebView deserializes this to populate its project picker and to decide whether to prompt @@ -1412,6 +1430,23 @@ export async function activate(context: ExecutionActivationContext): Promise setHostPerfMarks(enabled), @@ -1787,6 +1822,7 @@ export async function activate(context: ExecutionActivationContext): Promise( + bookOf: (link: Link) => string, +) { + return { + analysisId: (link: Link) => [link.analysisId], + book: (link: Link) => [bookOf(link)], + bookStatus: (link: Link) => [bookStatusKey(bookOf(link), link.status)], + }; +} + +const formKeyers = { + form: (payload: { surfaceText: string }) => [normalizeSurfaceForm(payload.surfaceText)], +}; + +/** + * An analysis's records by id, with the derived indexes its lookups answer from. Built from the + * records and kept current as changes are applied. It hands out the records it holds, not copies. + */ +export class AnalysisIndex { + private readonly tokenAnalyses: RecordTable; + + private readonly phraseAnalyses: RecordTable; + + private readonly segmentAnalyses: RecordTable; + + private readonly tokenLinks: RecordTable< + TokenAnalysisLink, + 'analysisId' | 'book' | 'bookStatus' | 'tokenRef' + >; + + private readonly phraseLinks: RecordTable< + PhraseAnalysisLink, + 'analysisId' | 'book' | 'bookStatus' | 'tokenRef' + >; + + private readonly segmentLinks: RecordTable< + SegmentAnalysisLink, + 'analysisId' | 'book' | 'bookStatus' | 'segmentId' + >; + + constructor(analysis: TextAnalysis) { + this.tokenAnalyses = new RecordTable('tokenAnalyses', formKeyers, analysis.tokenAnalyses); + this.phraseAnalyses = new RecordTable('phraseAnalyses', formKeyers, analysis.phraseAnalyses); + this.segmentAnalyses = new RecordTable('segmentAnalyses', {}, analysis.segmentAnalyses); + this.tokenLinks = new RecordTable( + 'tokenAnalysisLinks', + { + ...linkKeyers((link: TokenAnalysisLink) => bookOfRef(link.token.tokenRef)), + tokenRef: (link) => [link.token.tokenRef], + }, + analysis.tokenAnalysisLinks, + ); + this.phraseLinks = new RecordTable( + 'phraseAnalysisLinks', + { + // Every token of a phrase shares a book, so the first one places the whole run. + ...linkKeyers((link: PhraseAnalysisLink) => bookOfRef(link.tokens[0].tokenRef)), + tokenRef: (link) => link.tokens.map(({ tokenRef }) => tokenRef), + }, + analysis.phraseAnalysisLinks, + ); + this.segmentLinks = new RecordTable( + 'segmentAnalysisLinks', + { + ...linkKeyers((link: SegmentAnalysisLink) => bookOfRef(link.segmentId)), + segmentId: (link) => [link.segmentId], + }, + analysis.segmentAnalysisLinks, + ); + } + + /** + * Applies `changes`, which must name each id at most once per record kind. The index holds the + * put records themselves, not copies. + */ + apply(changes: ChangeSet): void { + this.tokenAnalyses.apply(changes.tokenAnalyses); + this.phraseAnalyses.apply(changes.phraseAnalyses); + this.segmentAnalyses.apply(changes.segmentAnalyses); + this.tokenLinks.apply(changes.tokenAnalysisLinks); + this.phraseLinks.apply(changes.phraseAnalysisLinks); + this.segmentLinks.apply(changes.segmentAnalysisLinks); + } + + /** The token analyses stored under `ids`, each once; an id with none stored is skipped. */ + getTokenAnalyses(ids: readonly string[]): TokenAnalysis[] { + return this.tokenAnalyses.get(ids); + } + + /** The phrase analyses stored under `ids`, each once; an id with none stored is skipped. */ + getPhraseAnalyses(ids: readonly string[]): PhraseAnalysis[] { + return this.phraseAnalyses.get(ids); + } + + /** The segment analyses stored under `ids`, each once; an id with none stored is skipped. */ + getSegmentAnalyses(ids: readonly string[]): SegmentAnalysis[] { + return this.segmentAnalyses.get(ids); + } + + /** The token links on any of `tokenRefs`, and the phrase links touching any of them. */ + findLinksByToken( + tokenRefs: readonly string[], + ): Pick { + return { + tokenAnalysisLinks: this.tokenLinks.find('tokenRef', tokenRefs), + phraseAnalysisLinks: this.phraseLinks.find('tokenRef', tokenRefs), + }; + } + + /** The segment links on any of `segmentIds`. */ + findLinksBySegment(segmentIds: readonly string[]): SegmentAnalysisLink[] { + return this.segmentLinks.find('segmentId', segmentIds); + } + + /** The links of every kind to any of `analysisIds`. */ + findLinksToAnalyses(analysisIds: readonly string[]): AnalysisLinks { + return { + tokenAnalysisLinks: this.tokenLinks.find('analysisId', analysisIds), + phraseAnalysisLinks: this.phraseLinks.find('analysisId', analysisIds), + segmentAnalysisLinks: this.segmentLinks.find('analysisId', analysisIds), + }; + } + + /** The links of every kind into any of `bookCodes`, limited to `statuses` when given. */ + findLinksInBooks( + bookCodes: readonly string[], + statuses?: readonly AssignmentStatus[], + ): AnalysisLinks { + if (!statuses) { + return { + tokenAnalysisLinks: this.tokenLinks.find('book', bookCodes), + phraseAnalysisLinks: this.phraseLinks.find('book', bookCodes), + segmentAnalysisLinks: this.segmentLinks.find('book', bookCodes), + }; + } + const keys = bookCodes.flatMap((bookCode) => + statuses.map((status) => bookStatusKey(bookCode, status)), + ); + return { + tokenAnalysisLinks: this.tokenLinks.find('bookStatus', keys), + phraseAnalysisLinks: this.phraseLinks.find('bookStatus', keys), + segmentAnalysisLinks: this.segmentLinks.find('bookStatus', keys), + }; + } + + /** + * The token analyses whose surface text {@link normalizeSurfaceForm} maps to one of + * `normalizedForms`. + */ + findTokenAnalysesByForm(normalizedForms: readonly string[]): TokenAnalysis[] { + return this.tokenAnalyses.find('form', normalizedForms); + } + + /** + * The phrase analyses whose surface text {@link normalizeSurfaceForm} maps to one of + * `normalizedForms`. + */ + findPhraseAnalysesByForm(normalizedForms: readonly string[]): PhraseAnalysis[] { + return this.phraseAnalyses.find('form', normalizedForms); + } + + /** Where this index differs from one built afresh from `analysis`. */ + differencesFrom(analysis: TextAnalysis): IndexDifference[] { + const rebuilt = new AnalysisIndex(analysis); + return [ + ...this.tokenAnalyses.differencesFrom(rebuilt.tokenAnalyses), + ...this.phraseAnalyses.differencesFrom(rebuilt.phraseAnalyses), + ...this.segmentAnalyses.differencesFrom(rebuilt.segmentAnalyses), + ...this.tokenLinks.differencesFrom(rebuilt.tokenLinks), + ...this.phraseLinks.differencesFrom(rebuilt.phraseLinks), + ...this.segmentLinks.differencesFrom(rebuilt.segmentLinks), + ]; + } +} diff --git a/src/services/draftRepository.ts b/src/services/draftRepository.ts index 6f2f2cf9b..d4f0e0787 100644 --- a/src/services/draftRepository.ts +++ b/src/services/draftRepository.ts @@ -20,7 +20,10 @@ import { import { mergeAnalyses, splitAnalysisByBook } from '../utils/analysis-book'; import { reportAnalysisViolations } from '../utils/analysis-violations'; import { applyChangeSet } from '../utils/change-set'; +import { jsonCopy } from '../utils/json-copy'; import { perfTime, perfTimeAsync, perfTimeAsyncExcluding } from '../utils/perf-marks'; +import { AnalysisIndex } from './analysisIndex'; +import { IndexedRepository, type IndexedAnalysis } from './indexedRepository'; import { draftAnalysisKey, draftJournalKey, draftKey } from './storageKeys'; /** @@ -104,7 +107,7 @@ type LoadedShard = { * * Calls on one instance must not overlap. */ -export class DraftRepository implements AnalysisRepository { +export class DraftRepository extends IndexedRepository implements AnalysisRepository { readonly capabilities: RepositoryCapabilities = { readOnly: false }; private readonly storage: KeyValueStorage; @@ -129,12 +132,14 @@ export class DraftRepository implements AnalysisRepository { private loaded = false; /** - * The draft as this instance's last apply left it, which the next apply builds on. Dropped by - * every load and every whole save, since either may come from a caller holding another draft. + * The draft this instance last applied changes to or answered a lookup from, with the index over + * its records. A whole save drops it, since its caller may hold another draft, and so does + * {@link exportAll}, which reads storage afresh. */ - private current: DraftProject | undefined; + private current: { draft: DraftProject; index: AnalysisIndex } | undefined; constructor(storage: KeyValueStorage, sourceProjectId: string, logger: WarningLogger) { + super(); this.storage = storage; this.logger = logger; this.sourceProjectId = sourceProjectId; @@ -163,14 +168,13 @@ export class DraftRepository implements AnalysisRepository { */ async exportAll(): Promise { this.current = undefined; - const exported = await this.readDraft(false); - this.loaded = true; - return exported; + return this.read(false); } /** - * Loads the stored draft as {@link exportAll} does, for copying into a project. A draft never - * saved exports as a fresh empty one. + * Loads the stored draft as {@link exportAll} does, for copying into a project, without refreshing + * the draft that lookups and {@link apply} work from. A draft never saved exports as a fresh empty + * one. * * @throws {SyntaxError} If the draft's stored value contains invalid JSON. * @throws {Error} If the stored draft's `modelVersion` is higher than this build's. @@ -179,10 +183,7 @@ export class DraftRepository implements AnalysisRepository { * @throws If storage rejects a read. */ async exportForCopy(): Promise { - this.current = undefined; - const exported = await this.readDraft(true); - this.loaded = true; - return exported; + return this.read(true); } /** @@ -240,7 +241,7 @@ export class DraftRepository implements AnalysisRepository { this.current = undefined; if (!this.loaded) { try { - await this.exportAll(); + await this.read(false); } catch (e) { // An unparseable draft names no shard to keep, and this save replaces it. if (!(e instanceof SyntaxError)) throw e; @@ -250,9 +251,9 @@ export class DraftRepository implements AnalysisRepository { } /** - * Applies the changes to the draft as this instance's last apply left it, or to the stored draft - * when it has made none since its last whole load or save. Only the books whose content the - * changes alter are rewritten, so an edit to a payload rewrites every book linking it. + * Applies the changes to the draft this instance keeps, loading the stored draft first when it + * keeps none, as after {@link exportAll} or a whole save. Only the books whose content the changes + * alter are rewritten, so an edit to a payload rewrites every book linking it. * * @throws {Error} If an id appears more than once in one record kind's changes, or the scope's * record names another source project; nothing is written. @@ -263,13 +264,48 @@ export class DraftRepository implements AnalysisRepository { * is left as it was. */ async apply(changes: ChangeSet): Promise { - this.current ??= (await this.exportAll()).draft; - const { draft, inverse } = applyChangeSet(this.current, changes); + const { draft: base, index } = await this.keptDraft(); + const copied = jsonCopy(changes); + const { draft, inverse } = applyChangeSet(base, copied); await this.writeDraft(draft, undefined, { refuseUnreadable: true }); - this.current = draft; + index.apply(copied); + this.current = { draft, index }; return { inverse }; } + protected async indexed(): Promise { + return (await this.keptDraft()).index; + } + + protected keptIndex(): IndexedAnalysis | undefined { + return this.current && { analysis: this.current.draft.analysis, index: this.current.index }; + } + + /** + * The draft this instance keeps and the index over its records, loading the draft from storage + * and building the index when none is kept. + * + * @throws Whatever {@link exportAll} throws. + */ + private async keptDraft(): Promise<{ draft: DraftProject; index: AnalysisIndex }> { + if (!this.current) { + const { draft } = await this.read(false); + this.current = { draft, index: new AnalysisIndex(draft.analysis) }; + } + return this.current; + } + + /** + * Loads the stored draft as {@link readDraft} does, noting that this instance has loaded it. + * + * @throws Whatever {@link readDraft} throws. + */ + private async read(strictEnvelope: boolean): Promise { + const exported = await this.readDraft(strictEnvelope); + this.loaded = true; + return exported; + } + /** * Saves `draft` as {@link replaceAll} describes. * diff --git a/src/services/indexedRepository.ts b/src/services/indexedRepository.ts new file mode 100644 index 000000000..bfbcc7ff0 --- /dev/null +++ b/src/services/indexedRepository.ts @@ -0,0 +1,98 @@ +import type { + AssignmentStatus, + PhraseAnalysis, + SegmentAnalysis, + SegmentAnalysisLink, + TextAnalysis, + TokenAnalysis, +} from 'interlinearizer'; +import type { + AnalysisLinks, + AnalysisRepository, + IndexDifference, +} from '../types/analysis-repository'; +import { jsonCopy } from '../utils/json-copy'; +import type { AnalysisIndex } from './analysisIndex'; + +/** An index, with the analysis whose records it is kept over. */ +export type IndexedAnalysis = { + analysis: TextAnalysis; + index: AnalysisIndex; +}; + +/** The lookups of a repository that answers them from an {@link AnalysisIndex} over its records. */ +export abstract class IndexedRepository implements Pick< + AnalysisRepository, + | 'getTokenAnalyses' + | 'getPhraseAnalyses' + | 'getSegmentAnalyses' + | 'findLinksByToken' + | 'findLinksBySegment' + | 'findLinksToAnalyses' + | 'findLinksInBooks' + | 'findTokenAnalysesByForm' + | 'findPhraseAnalysesByForm' + | 'checkIndexes' +> { + getTokenAnalyses(ids: string[]): Promise { + return this.query((index) => index.getTokenAnalyses(ids)); + } + + getPhraseAnalyses(ids: string[]): Promise { + return this.query((index) => index.getPhraseAnalyses(ids)); + } + + getSegmentAnalyses(ids: string[]): Promise { + return this.query((index) => index.getSegmentAnalyses(ids)); + } + + findLinksByToken( + tokenRefs: string[], + ): Promise> { + return this.query((index) => index.findLinksByToken(tokenRefs)); + } + + findLinksBySegment(segmentIds: string[]): Promise { + return this.query((index) => index.findLinksBySegment(segmentIds)); + } + + findLinksToAnalyses(analysisIds: string[]): Promise { + return this.query((index) => index.findLinksToAnalyses(analysisIds)); + } + + findLinksInBooks(bookCodes: string[], statuses?: AssignmentStatus[]): Promise { + return this.query((index) => index.findLinksInBooks(bookCodes, statuses)); + } + + findTokenAnalysesByForm(normalizedForms: string[]): Promise { + return this.query((index) => index.findTokenAnalysesByForm(normalizedForms)); + } + + findPhraseAnalysesByForm(normalizedForms: string[]): Promise { + return this.query((index) => index.findPhraseAnalysesByForm(normalizedForms)); + } + + /** + * Reports where the kept index differs from one built afresh from the records, for debugging; + * nothing while none is built. + */ + async checkIndexes(): Promise { + const kept = this.keptIndex(); + return kept ? kept.index.differencesFrom(kept.analysis) : []; + } + + /** Runs `read` against the index and returns a copy of its result, so callers may change it. */ + private async query(read: (index: AnalysisIndex) => T): Promise { + return jsonCopy(read(await this.indexed())); + } + + /** + * The kept index, built over the stored records first when none is kept. + * + * @throws Whatever loading the stored records throws. + */ + protected abstract indexed(): Promise; + + /** The kept index and the analysis it is kept over, or `undefined` when none is kept. */ + protected abstract keptIndex(): IndexedAnalysis | undefined; +} diff --git a/src/services/projectStorage.ts b/src/services/projectStorage.ts index e6eefe7a3..f3b6e714a 100644 --- a/src/services/projectStorage.ts +++ b/src/services/projectStorage.ts @@ -74,6 +74,12 @@ const draftRepositories = new Map(); */ const draftWriteCounts = new Map(); +/** + * How many times a sync has replaced or a deletion removed each project, so a repository over a + * Paratext 9 import can tell its kept records are out of date. + */ +const importRevisions = new Map(); + /** A project created from a copy of the stored draft, and the books the copy left out. */ export type NewProjectFromDraft = { project: InterlinearProject; @@ -582,7 +588,7 @@ export async function getPt9ImportForSource( * record, or replaces it wholesale when one exists. This is the only write path for * `pt9Import`-carrying projects - the public update functions reject them - and every field except * `id` and `createdAt` is rebuilt from the arguments on replace, so nothing from the previous - * import survives a sync. + * import survives a sync, not even in a repository already opened over it. * * @param token - The execution token for storage access. * @param sourceProjectId - The Platform.Bible project ID the interlinear data was imported from. @@ -640,6 +646,7 @@ export async function savePt9Import( createdAt: current.createdAt, }; await papi.storage.writeUserData(token, projectKey(current.id), JSON.stringify(updated)); + importRevisions.set(current.id, (importRevisions.get(current.id) ?? 0) + 1); return (await refreshSummary(token, summarizeProject(updated))) ? updated : undefined; }); if (replaced) return replaced; @@ -945,7 +952,8 @@ export async function updateProjectMetadata( /** * Unlists the project with the given ID and deletes its record. No-ops silently if the project does * not exist. The deletion is committed once the project is unlisted: a record that then fails to - * delete is logged and left to a later {@link sweepPendingCleanup}, not reported. + * delete is logged and left to a later {@link sweepPendingCleanup}, not reported. A repository + * already opened over a deleted Paratext 9 import stops answering from what it kept. * * @throws {SyntaxError} If the stored project summaries contain invalid JSON. * @throws {Error} If the stored project summaries were written by a newer build, or are corrupt. @@ -970,10 +978,13 @@ export async function deleteProject(token: ExecutionToken, id: string): Promise< try { await papi.storage.deleteUserData(token, projectKey(id)); } catch (e) { - if (isNotFound(e)) return; - logger.error(`Failed to delete the record of unlisted project ${id}:`, e); - await recordForCleanup(); + if (!isNotFound(e)) { + logger.error(`Failed to delete the record of unlisted project ${id}:`, e); + await recordForCleanup(); + } } + // Bumped after the deletion settles, so a repository drops whatever it read meanwhile. + importRevisions.set(id, (importRevisions.get(id) ?? 0) + 1); }); } @@ -1133,6 +1144,21 @@ export function openDraftRepository( replaceAll: (draft, options) => counted((repository) => repository.replaceAll(draft, options)), markClean: () => queued((repository) => repository.markClean()), apply: (changes) => counted((repository) => repository.apply(changes)), + getTokenAnalyses: (ids) => queued((repository) => repository.getTokenAnalyses(ids)), + getPhraseAnalyses: (ids) => queued((repository) => repository.getPhraseAnalyses(ids)), + getSegmentAnalyses: (ids) => queued((repository) => repository.getSegmentAnalyses(ids)), + findLinksByToken: (tokenRefs) => queued((repository) => repository.findLinksByToken(tokenRefs)), + findLinksBySegment: (segmentIds) => + queued((repository) => repository.findLinksBySegment(segmentIds)), + findLinksToAnalyses: (analysisIds) => + queued((repository) => repository.findLinksToAnalyses(analysisIds)), + findLinksInBooks: (bookCodes, statuses) => + queued((repository) => repository.findLinksInBooks(bookCodes, statuses)), + findTokenAnalysesByForm: (normalizedForms) => + queued((repository) => repository.findTokenAnalysesByForm(normalizedForms)), + findPhraseAnalysesByForm: (normalizedForms) => + queued((repository) => repository.findPhraseAnalysesByForm(normalizedForms)), + checkIndexes: () => queued((repository) => repository.checkIndexes()), }; } @@ -1152,8 +1178,10 @@ export function openPt9ImportRepository( token: ExecutionToken, interlinearProjectId: string, ): AnalysisRepository { - return new Pt9ImportRepository(interlinearProjectId, () => - getProject(token, interlinearProjectId), + return new Pt9ImportRepository( + interlinearProjectId, + () => getProject(token, interlinearProjectId), + () => importRevisions.get(interlinearProjectId) ?? 0, ); } @@ -1169,4 +1197,5 @@ export function resetQueuesForTesting(): void { draftQueues.clear(); draftRepositories.clear(); draftWriteCounts.clear(); + importRevisions.clear(); } diff --git a/src/services/pt9ImportRepository.ts b/src/services/pt9ImportRepository.ts index a8468b5a6..4aec82bda 100644 --- a/src/services/pt9ImportRepository.ts +++ b/src/services/pt9ImportRepository.ts @@ -1,4 +1,4 @@ -import type { InterlinearProject } from 'interlinearizer'; +import type { DraftProject, InterlinearProject } from 'interlinearizer'; import type { AnalysisRepository, ApplyResult, @@ -7,29 +7,44 @@ import type { } from '../types/analysis-repository'; import { CURRENT_MODEL_VERSION } from '../types/model-version'; import { ReadOnlyAnalysisError } from '../types/read-only-analysis-error'; +import { AnalysisIndex } from './analysisIndex'; +import { IndexedRepository, type IndexedAnalysis } from './indexedRepository'; /** * A Paratext 9 import's analysis, read in a draft's shape. Only a sync replaces an import's * content, so every write is refused. */ -export class Pt9ImportRepository implements AnalysisRepository { +export class Pt9ImportRepository extends IndexedRepository implements AnalysisRepository { readonly capabilities: RepositoryCapabilities = { readOnly: true }; private readonly interlinearProjectId: string; private readonly readProject: () => Promise; + private readonly revision: () => number; + + /** + * The import as first read for a lookup, with the index over its records; dropped once a sync + * replaces the import or a deletion removes it. + */ + private loaded: { draft: DraftProject; revision: number; index: AnalysisIndex } | undefined; + /** * @param interlinearProjectId - Id of the project to read, which must be a Paratext 9 import. * @param readProject - Reads the project stored under that id afresh, as a new object that the * repository may hand out, resolving `undefined` when none is stored. + * @param revision - The import's revision, which moves whenever a sync replaces the import or a + * deletion removes it. */ constructor( interlinearProjectId: string, readProject: () => Promise, + revision: () => number, ) { + super(); this.interlinearProjectId = interlinearProjectId; this.readProject = readProject; + this.revision = revision; } /** @@ -41,25 +56,7 @@ export class Pt9ImportRepository implements AnalysisRepository { * @throws Whatever reading the project throws. */ async exportAll(): Promise { - const project = await this.readProject(); - if (!project?.pt9Import) { - throw new Error( - `Interlinearizer: no Paratext 9 import is stored as project ${this.interlinearProjectId}`, - ); - } - const { sourceProjectId, analysisLanguages, targetProjectId, analysis, segmentation } = project; - return { - draft: { - sourceProjectId, - modelVersion: CURRENT_MODEL_VERSION, - analysisLanguages, - ...(targetProjectId !== undefined && { targetProjectId }), - analysis, - dirty: false, - ...(segmentation !== undefined && { segmentation }), - }, - unreadableBooks: [], - }; + return { draft: await this.readImport(), unreadableBooks: [] }; } /** @@ -86,6 +83,51 @@ export class Pt9ImportRepository implements AnalysisRepository { throw this.readOnlyError(); } + protected async indexed(): Promise { + const kept = this.kept(); + if (kept) return kept.index; + // The revision is taken before the read, so a sync landing during it forces another. + const revision = this.revision(); + const draft = await this.readImport(); + this.loaded = { draft, revision, index: new AnalysisIndex(draft.analysis) }; + return this.loaded.index; + } + + protected keptIndex(): IndexedAnalysis | undefined { + const kept = this.kept(); + return kept && { analysis: kept.draft.analysis, index: kept.index }; + } + + /** The import as last read, unless a sync or a deletion has changed it since. */ + private kept() { + if (this.loaded?.revision !== this.revision()) this.loaded = undefined; + return this.loaded; + } + + /** + * Reads the import as {@link exportAll} describes. + * + * @throws Whatever {@link exportAll} throws. + */ + private async readImport(): Promise { + const project = await this.readProject(); + if (!project?.pt9Import) { + throw new Error( + `Interlinearizer: no Paratext 9 import is stored as project ${this.interlinearProjectId}`, + ); + } + const { sourceProjectId, analysisLanguages, targetProjectId, analysis, segmentation } = project; + return { + sourceProjectId, + modelVersion: CURRENT_MODEL_VERSION, + analysisLanguages, + ...(targetProjectId !== undefined && { targetProjectId }), + analysis, + dirty: false, + ...(segmentation !== undefined && { segmentation }), + }; + } + private readOnlyError(): ReadOnlyAnalysisError { return new ReadOnlyAnalysisError( `Interlinearizer: Paratext 9 import ${this.interlinearProjectId} is read-only`, diff --git a/src/services/recordTable.ts b/src/services/recordTable.ts new file mode 100644 index 000000000..a54316468 --- /dev/null +++ b/src/services/recordTable.ts @@ -0,0 +1,131 @@ +import type { IndexDifference, RecordChanges } from '../types/analysis-repository'; + +/** The keys a derived index files a record under. */ +export type KeysOf = (record: T) => readonly string[]; + +/** Ids filed under keys. */ +type IdsByKey = Map>; + +function fileId(index: IdsByKey, key: string, id: string): void { + const ids = index.get(key); + if (ids) ids.add(id); + else index.set(key, new Set([id])); +} + +function unfileId(index: IdsByKey, key: string, id: string): void { + const ids = index.get(key); + /* v8 ignore next -- a record's keys were filed when it was put */ + if (!ids) return; + ids.delete(id); + if (ids.size === 0) index.delete(key); +} + +/** The ids in `a` that `b` lacks. */ +function idsLacking(a: ReadonlySet | undefined, b: ReadonlySet | undefined) { + return [...(a ?? [])].filter((id) => !b?.has(id)); +} + +/** One record kind's records by id, with the derived indexes kept over them. */ +export class RecordTable { + private readonly kind: string; + + private readonly keyers: Record>; + + private readonly records = new Map(); + + private readonly indexes = new Map(); + + constructor(kind: string, keyers: Record>, records: readonly T[]) { + this.kind = kind; + this.keyers = keyers; + this.indexNames().forEach((index) => this.indexes.set(index, new Map())); + records.forEach((record) => this.put(record)); + } + + /** The records stored under `ids`, each once. */ + get(ids: readonly string[]): T[] { + return [...new Set(ids)].flatMap((id) => this.records.get(id) ?? []); + } + + /** The records `index` files under any of `keys`, each once. */ + find(index: Index, keys: readonly string[]): T[] { + const filed = this.filed(index); + const ids = new Set(keys.flatMap((key) => [...(filed.get(key) ?? [])])); + return this.get([...ids]); + } + + /** Applies one record kind's puts and deletes; absent changes apply nothing. */ + apply(changes: RecordChanges | undefined): void { + changes?.puts.forEach((record) => this.put(record)); + changes?.deletes.forEach((id) => this.remove(id)); + } + + /** + * Where this table differs from `rebuilt`, a table of the same kind built afresh. A record whose + * kept copy differs from the rebuilt one is both missing and extra. + */ + differencesFrom(rebuilt: RecordTable): IndexDifference[] { + const ids = new Set([...this.records.keys(), ...rebuilt.records.keys()]); + const recordDifferences = [...ids].flatMap((id): IndexDifference[] => { + const kept = this.records.get(id); + const stored = rebuilt.records.get(id); + if (JSON.stringify(kept) === JSON.stringify(stored)) return []; + return [{ index: this.kind, key: id, missing: stored ? [id] : [], extra: kept ? [id] : [] }]; + }); + const indexDifferences = this.indexNames().flatMap((index) => { + const kept = this.filed(index); + const fresh = rebuilt.filed(index); + const keys = new Set([...kept.keys(), ...fresh.keys()]); + return [...keys].flatMap((key): IndexDifference[] => { + const missing = idsLacking(fresh.get(key), kept.get(key)); + const extra = idsLacking(kept.get(key), fresh.get(key)); + if (missing.length === 0 && extra.length === 0) return []; + return [{ index: `${this.kind} by ${index}`, key, missing, extra }]; + }); + }); + return [...recordDifferences, ...indexDifferences]; + } + + private indexNames(): Index[] { + // Object.keys widens to string[]; the keys are exactly the keyers' index names. + return Object.keys(this.keyers).filter((name): name is Index => name in this.keyers); + } + + private filed(index: Index): IdsByKey { + const filed = this.indexes.get(index); + /* v8 ignore next -- the constructor gives every index its map */ + if (!filed) throw new Error(`Interlinearizer: ${this.kind} has no index by ${index}`); + return filed; + } + + /** The keys `index` files `record` under, each once. */ + private keysOf(index: Index, record: T): Set { + return new Set(this.keyers[index](record)); + } + + /** Stores `record`, keeping a replaced record's place under every key it keeps. */ + private put(record: T): void { + const prior = this.records.get(record.id); + this.records.set(record.id, record); + this.indexNames().forEach((index) => { + const filed = this.filed(index); + const before = prior ? this.keysOf(index, prior) : new Set(); + const after = this.keysOf(index, record); + before.forEach((key) => { + if (!after.has(key)) unfileId(filed, key, record.id); + }); + after.forEach((key) => { + if (!before.has(key)) fileId(filed, key, record.id); + }); + }); + } + + private remove(id: string): void { + const record = this.records.get(id); + if (!record) return; + this.records.delete(id); + this.indexNames().forEach((index) => + this.keysOf(index, record).forEach((key) => unfileId(this.filed(index), key, id)), + ); + } +} diff --git a/src/types/analysis-repository.ts b/src/types/analysis-repository.ts index 848f60163..cd5bdda2a 100644 --- a/src/types/analysis-repository.ts +++ b/src/types/analysis-repository.ts @@ -1,4 +1,14 @@ -import type { DraftProject, TextAnalysis } from 'interlinearizer'; +import type { + AssignmentStatus, + DraftProject, + PhraseAnalysis, + SegmentAnalysis, + SegmentAnalysisLink, + TextAnalysis, + TokenAnalysis, +} from 'interlinearizer'; +// eslint-disable-next-line @typescript-eslint/no-unused-vars -- named only for its doc links +import type { normalizeSurfaceForm } from '../utils/analysis-identity'; /** Text records by key. */ export interface KeyValueStorage { @@ -89,16 +99,40 @@ export type ApplyResult = { inverse: ChangeSet; }; +/** Links of every kind, one list per kind. */ +export type AnalysisLinks = Pick< + TextAnalysis, + 'tokenAnalysisLinks' | 'phraseAnalysisLinks' | 'segmentAnalysisLinks' +>; + +/** A difference between a repository's kept indexes and ones built afresh from its records. */ +export type IndexDifference = { + /** The record kind, or ` by ` for a derived index. */ + index: string; + /** The record id, or the key whose entries differ in a derived index. */ + key: string; + /** + * Ids filed under `key` in the fresh index and not the kept one; where `index` names a record + * kind, also the id of a record whose kept copy differs. + */ + missing: string[]; + /** + * Ids filed under `key` in the kept index and not the fresh one; where `index` names a record + * kind, also the id of a record whose kept copy differs. + */ + extra: string[]; +}; + /** * Reads and writes one source project's draft, or, when read-only, reads analysis in a draft's * shape. Every call is async and takes and returns only JSON-plain values, so an implementation may * sit behind a network boundary or another language. Once a call settles, the caller may change a - * draft it passed in or got back without changing what is stored. Lists of records come back in no - * promised order. + * draft or records it passed in or got back without changing what is stored. Lists of records come + * back in no promised order, and a lookup by several keys returns each record once. * * Calls for one draft must not overlap, whether on one repository or several. After one repository - * saves a draft, any other that had already loaded or saved it must load it again before its next - * save. + * saves a draft, any other that had already loaded or saved it must load it again through + * {@link exportAll} before its next save or lookup; {@link exportForCopy} does not count. */ export interface AnalysisRepository { /** Fixed for the repository's lifetime. */ @@ -171,4 +205,78 @@ export interface AnalysisRepository { * unchanged, though storage may hold some of the changes until a later save. */ apply(changes: ChangeSet): Promise; + + /** + * The token analyses stored under `ids`; an id with none stored under it is skipped. + * + * @throws Whatever {@link AnalysisRepository.exportAll} throws. + */ + getTokenAnalyses(ids: string[]): Promise; + + /** + * The phrase analyses stored under `ids`; an id with none stored under it is skipped. + * + * @throws Whatever {@link AnalysisRepository.exportAll} throws. + */ + getPhraseAnalyses(ids: string[]): Promise; + + /** + * The segment analyses stored under `ids`; an id with none stored under it is skipped. + * + * @throws Whatever {@link AnalysisRepository.exportAll} throws. + */ + getSegmentAnalyses(ids: string[]): Promise; + + /** + * The token links on any of `tokenRefs`, and the phrase links touching any of them. + * + * @throws Whatever {@link AnalysisRepository.exportAll} throws. + */ + findLinksByToken( + tokenRefs: string[], + ): Promise>; + + /** + * The segment links on any of `segmentIds`. + * + * @throws Whatever {@link AnalysisRepository.exportAll} throws. + */ + findLinksBySegment(segmentIds: string[]): Promise; + + /** + * The links of every kind to any of `analysisIds`. + * + * @throws Whatever {@link AnalysisRepository.exportAll} throws. + */ + findLinksToAnalyses(analysisIds: string[]): Promise; + + /** + * The links of every kind into any of `bookCodes`, limited to `statuses` when given. + * + * @throws Whatever {@link AnalysisRepository.exportAll} throws. + */ + findLinksInBooks(bookCodes: string[], statuses?: AssignmentStatus[]): Promise; + + /** + * The token analyses whose surface text {@link normalizeSurfaceForm} maps to one of + * `normalizedForms`. + * + * @throws Whatever {@link AnalysisRepository.exportAll} throws. + */ + findTokenAnalysesByForm(normalizedForms: string[]): Promise; + + /** + * The phrase analyses whose surface text {@link normalizeSurfaceForm} maps to one of + * `normalizedForms`. + * + * @throws Whatever {@link AnalysisRepository.exportAll} throws. + */ + findPhraseAnalysesByForm(normalizedForms: string[]): Promise; + + /** + * Reports where the derived indexes the repository keeps differ from ones built afresh from its + * records, for debugging, leaving the kept ones as they are. A repository keeping none reports + * nothing. + */ + checkIndexes(): Promise; } diff --git a/src/types/interlinearizer.d.ts b/src/types/interlinearizer.d.ts index 06ab18d47..c43af3ea6 100644 --- a/src/types/interlinearizer.d.ts +++ b/src/types/interlinearizer.d.ts @@ -381,6 +381,15 @@ declare module 'papi-shared-types' { */ 'interlinearizer.redoEdit': (sessionId: string) => Promise; + /** + * Reports where a session's derived indexes differ from ones built afresh from its records, for + * debugging. + * + * @returns A JSON string of the index differences; `"[]"` when none differ. + * @throws If the session is not open (logged before rethrowing). + */ + 'interlinearizer.checkIndexes': (sessionId: string) => Promise; + /** * Turns the extension host's timing entries on or off, discarding any it holds. Off until * turned on. diff --git a/src/utils/change-set.ts b/src/utils/change-set.ts index e28124d9a..0f48c72e5 100644 --- a/src/utils/change-set.ts +++ b/src/utils/change-set.ts @@ -7,11 +7,6 @@ export type AppliedChanges = { inverse: ChangeSet; }; -/** A copy of a JSON-plain value that shares nothing with it. */ -function copyOf(value: T): T { - return JSON.parse(JSON.stringify(value)); -} - /** The draft's record, without its analysis or its `dirty` flag. */ // eslint-disable-next-line @typescript-eslint/no-unused-vars -- named only to leave them out export function scopeRecordOf({ analysis, dirty, ...scope }: DraftProject): ScopeRecord { @@ -41,7 +36,7 @@ function applyRecordChanges( const stored = new Map(records.map((record) => [record.id, record])); const deleted = new Set(changes.deletes); - const puts = new Map(changes.puts.map((record) => [record.id, copyOf(record)])); + const puts = new Map(changes.puts.map((record) => [record.id, record])); const inverse: RecordChanges = { puts: [], deletes: [] }; changes.puts.forEach(({ id }) => { const prior = stored.get(id); @@ -68,9 +63,9 @@ function applyRecordChanges( } /** - * Applies `changes` to `draft`, leaving `draft` as it was. The result is marked unsaved. It holds - * copies of the records the changes put, and shares everything else with `draft`, the records the - * inverse restores included. + * Applies `changes` to `draft`, leaving `draft` as it was. The result is marked unsaved. It shares + * the put records and any scope record with `changes`, and everything else with `draft`, including + * the records the inverse restores. * * @throws {Error} If an id appears more than once in one record kind's changes, or the scope's * record names another source project. @@ -125,7 +120,7 @@ export function applyChangeSet(draft: DraftProject, changes: ChangeSet): Applied }; return { draft: { - ...(changes.scope ? copyOf(changes.scope) : scopeRecordOf(draft)), + ...(changes.scope ?? scopeRecordOf(draft)), dirty: true, analysis: { segmentAnalyses: segmentAnalyses.records, diff --git a/src/utils/json-copy.ts b/src/utils/json-copy.ts new file mode 100644 index 000000000..eb6d43879 --- /dev/null +++ b/src/utils/json-copy.ts @@ -0,0 +1,4 @@ +/** A deep copy of a JSON-plain value, sharing nothing with it. */ +export function jsonCopy(value: T): T { + return JSON.parse(JSON.stringify(value)); +}