From 655a1b9bb73e95f2aa222746ca3c6183a2c2559f Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 14:37:18 +1100 Subject: [PATCH 01/20] fix: treat a REMS network failure as holding no licences (#57) --- src/auth.ts | 26 +++++++++++++---------- src/licence.test.ts | 51 ++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/src/auth.ts b/src/auth.ts index 04d9fe7..60bb0d4 100644 --- a/src/auth.ts +++ b/src/auth.ts @@ -44,19 +44,23 @@ async function fetchUserLicenses(request: FastifyRequest): Promise item.resource); + } catch (err) { + request.log.error({ err }, 'Failed to fetch user entitlements'); return; } - const data = await res.json(); - return data.map((item: RemsUserEntitlement) => item.resource); } const userIdsByRequest = new WeakMap>(); diff --git a/src/licence.test.ts b/src/licence.test.ts index 43be883..4e15738 100644 --- a/src/licence.test.ts +++ b/src/licence.test.ts @@ -1,7 +1,8 @@ import { createServer, type IncomingMessage, type Server } from 'node:http'; import type { AddressInfo } from 'node:net'; +import { Writable } from 'node:stream'; import { generateKeyPair, importJWK, type JWTPayload, SignJWT } from 'jose'; -import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { buildTestApp, repositoryWithContent, type TestApp } from '../test/app.ts'; import { generateSigningKey, type Jwk } from './oidc/keys.ts'; @@ -12,11 +13,26 @@ const RESTRICTED = 'https://example.org/licenses/restricted'; const RESTRICTED_METADATA = 'https://example.org/licenses/restricted-metadata'; const ENTITLED = 'upstream-user-entitled'; const UNENTITLED = 'upstream-user-unentitled'; +const UNREACHABLE = 'upstream-user-rems-unreachable'; +const GARBLED = 'upstream-user-rems-garbled'; const OPEN_FILE = 'http://test/licence/open.txt'; const RESTRICTED_FILE = 'http://test/licence/restricted.txt'; const RESTRICTED_ENTITY = 'http://test/licence/restricted'; +const logs = vi.hoisted(() => [] as Record[]); + +vi.mock('./logger.ts', async () => { + const { pino } = await import('pino'); + const stream = new Writable({ + write(chunk, _encoding, done) { + logs.push(JSON.parse(chunk.toString())); + done(); + }, + }); + return { logger: pino({ level: 'error' }, stream) }; +}); + let app: TestApp; let prisma: Awaited>['prisma']; let prefix: string; @@ -59,6 +75,14 @@ beforeAll(async () => { server = createServer((request, response) => { received.push(request); const user = new URL(request.url ?? '', 'http://rems').searchParams.get('user'); + if (user === UNREACHABLE) { + request.socket.destroy(); + return; + } + if (user === GARBLED) { + response.writeHead(200, { 'content-type': 'application/json' }).end('{"entitlements": ['); + return; + } const entitlements = user === ENTITLED ? [{ resource: RESTRICTED }, { resource: RESTRICTED_METADATA }] : []; response.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify(entitlements)); }); @@ -74,6 +98,7 @@ beforeAll(async () => { OIDC_UPSTREAM_CLIENT_ID: 'ldacapi', OIDC_UPSTREAM_CLIENT_SECRET: 'secret', REMS_ENDPOINT: fake, + ENROLLMENT_URL: 'https://rems.example.org/apply-for?resource={licenseId}', OIDC_SIGNING_KEYS: JSON.stringify({ keys: SIGNING_KEYS }), }, { repository: repositoryWithContent }, @@ -89,6 +114,7 @@ beforeAll(async () => { beforeEach(() => { received = []; + logs.length = 0; }); afterAll(async () => { @@ -127,6 +153,29 @@ describe('licence checks with our access tokens', () => { expect(remsRequests()).toHaveLength(1); }); + describe.each([ + ['is unreachable', UNREACHABLE], + ['answers with a garbled body', GARBLED], + ])('when REMS %s', (_, sub) => { + it('denies restricted content with its enrolment link, and logs the failure', async () => { + const response = await get(`/entity/${encodeURIComponent(RESTRICTED_FILE)}`, await token({ sub })); + + expect(response.statusCode).toBe(200); + expect(response.json().access).toEqual({ + metadata: true, + content: false, + contentAuthorizationUrl: `https://rems.example.org/apply-for?resource=${encodeURIComponent(RESTRICTED)}`, + }); + expect(logs).toContainEqual(expect.objectContaining({ level: 50, msg: 'Failed to fetch user entitlements' })); + }); + + it('still serves content under an open licence', async () => { + const response = await file(OPEN_FILE, await token({ sub })); + + expect(response.statusCode).toBe(200); + }); + }); + it('does not call the upstream provider', async () => { await file(RESTRICTED_FILE, await token()); From d327e69bc667ee481a521a4068bea812ecb092cd Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 14:38:34 +1100 Subject: [PATCH 02/20] feat: rely on arocapi's sandbox headers instead of Caddy's CSP (#58) --- README.md | 8 ++--- deploy/Caddyfile | 9 +---- docs/adr/0002-server-rendered-admin-pages.md | 4 +-- src/app.test.ts | 37 +++++++++++++++++++- src/app.ts | 1 + test/app.ts | 9 +++-- 6 files changed, 51 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 2a000ef..2d26709 100644 --- a/README.md +++ b/README.md @@ -320,10 +320,10 @@ from Let's Encrypt on its own. It serves two hostnames: | `ADMIN_DOMAIN` | `/admin/*` | The API's admin pages | The admin pages need their own hostname, so that script in a crate file can -never run on their origin (ADR 0002). Caddy also serves everything under -`/api/` with `Content-Security-Policy: sandbox`, which gives any HTML or SVG in -a crate file an opaque origin, so its scripts can't reach Oni's sign-in on -`DOMAIN`. +never run on their origin (ADR 0002). arocapi serves crate files, other than +audio and video, with a `sandbox` Content-Security-Policy, which gives any HTML +or SVG in a crate file an opaque origin, so its scripts can't reach Oni's +sign-in on `DOMAIN`. No other container publishes a port, on the host or on localhost, so Postgres, OpenSearch, the API and Oni are reachable only from within the stack. diff --git a/deploy/Caddyfile b/deploy/Caddyfile index e64641c..3168413 100644 --- a/deploy/Caddyfile +++ b/deploy/Caddyfile @@ -25,14 +25,7 @@ file_server } - # Crate files are served inline with their own content type. The sandbox gives - # any HTML or SVG among them an opaque origin, so its scripts can't reach Oni's sign-in. - handle /api/* { - header Content-Security-Policy "sandbox allow-scripts allow-popups allow-downloads" - reverse_proxy api:8080 - } - - @provider path /api /oauth/* /.well-known/openid-configuration + @provider path /api /api/* /oauth/* /.well-known/openid-configuration handle @provider { reverse_proxy api:8080 } diff --git a/docs/adr/0002-server-rendered-admin-pages.md b/docs/adr/0002-server-rendered-admin-pages.md index 2e55d01..16af409 100644 --- a/docs/adr/0002-server-rendered-admin-pages.md +++ b/docs/adr/0002-server-rendered-admin-pages.md @@ -2,11 +2,11 @@ ## Context -Admins manage the API's indexes. A separate browser app calling an admin API with one shared key exposes the key to anyone who loads the app, records nobody's identity, and can only take access away from everyone at once. The API is already an OIDC provider that signs users in at the upstream provider (ADR 0001), and it can reach the repository and indexer directly. It also serves crate files inline with their own content type and no sandbox, so HTML or SVG in a crate runs script on the API's origin. +Admins manage the API's indexes. A separate browser app calling an admin API with one shared key exposes the key to anyone who loads the app, records nobody's identity, and can only take access away from everyone at once. The API is already an OIDC provider that signs users in at the upstream provider (ADR 0001), and it can reach the repository and indexer directly. It also serves crate files inline, on the API's origin, with their own content type. arocapi sends them with `nosniff` and a `sandbox` Content-Security-Policy, which gives HTML or SVG in a crate an opaque origin, but leaves the CSP off audio and video so they play when opened directly. That protection rests on each response's headers and content type, and doesn't cover media. ## Decision -The API serves its own admin pages at `/admin/`, rendered on the server, in the API's process but on their own origin, `ADMIN_ORIGIN`, which must differ from the issuer's. Every admin route is constrained to the admin origin's host, and nothing else is served there, so script in a crate file runs on another origin than admin, with the browser's boundary between them. They are registered only when sign-in and `ADMIN_ORIGIN` are configured. They are the only way into admin: there is no admin API and no shared admin key, and the pages call the repository directly. +The API serves its own admin pages at `/admin/`, rendered on the server, in the API's process but on their own origin, `ADMIN_ORIGIN`, which must differ from the issuer's. Every admin route is constrained to the admin origin's host, and nothing else is served there, so any script a crate file does run is on another origin than admin, with the browser's boundary between them. They are registered only when sign-in and `ADMIN_ORIGIN` are configured. They are the only way into admin: there is no admin API and no shared admin key, and the pages call the repository directly. - The admin pages are a built-in confidential client of our provider, not listed in `OIDC_CLIENTS`. Its client secret is derived from `OIDC_COOKIE_KEYS` with HKDF, and it uses only the authorisation code grant, asking for `openid profile`. - Sign-in is a server-side authorisation code flow with PKCE, state and nonce. The browser goes through the provider as usual, but the token request reaches the provider in-process, with its metadata built from configuration rather than discovery, so the API never calls its own public URL. Only the ID token is kept. diff --git a/src/app.test.ts b/src/app.test.ts index 2e11c28..f19cb80 100644 --- a/src/app.test.ts +++ b/src/app.test.ts @@ -1,7 +1,7 @@ import { Client } from '@opensearch-project/opensearch'; import { describe, expect, it } from 'vitest'; import pkg from '../package.json' with { type: 'json' }; -import { repository } from '../test/app.ts'; +import { buildTestApp, repository, repositoryWithContent } from '../test/app.ts'; import { buildApp } from './app.ts'; import { config } from './configuration.ts'; import { createSearchBackend } from './indexer/search.ts'; @@ -81,3 +81,38 @@ describe('buildApp', () => { await app.close(); }); }); + +describe('crate file delivery', () => { + const OPEN = 'https://creativecommons.org/licenses/by/4.0/'; + const files = { 'http://test/delivery/page.html': 'text/html', 'http://test/delivery/song.mp3': 'audio/mpeg' }; + + it('sandboxes crate files but not audio or video, so media plays when opened directly', async () => { + const { app, prisma, config } = await buildTestApp({}, { repository: repositoryWithContent }); + for (const [id, mediaType] of Object.entries(files)) { + await prisma.entity.create({ + data: { id, name: id, description: '', entityType: 'http://schema.org/MediaObject', metadataLicenseId: OPEN, contentLicenseId: OPEN }, + }); + await prisma.file.create({ data: { id, filename: id.split('/').at(-1) ?? id, mediaType, size: 7 } }); + } + + try { + const deliver = (id: string) => app.inject({ method: 'GET', url: `${config.prefix}/file/${encodeURIComponent(id)}` }); + + const html = await deliver('http://test/delivery/page.html'); + expect(html.statusCode).toBe(200); + expect(html.headers['x-content-type-options']).toBe('nosniff'); + expect(html.headers['content-security-policy']).toContain('sandbox'); + + const audio = await deliver('http://test/delivery/song.mp3'); + expect(audio.statusCode).toBe(200); + expect(audio.headers['x-content-type-options']).toBe('nosniff'); + expect(audio.headers['content-security-policy']).toBeUndefined(); + } finally { + const ids = Object.keys(files); + await prisma.file.deleteMany({ where: { id: { in: ids } } }); + await prisma.entity.deleteMany({ where: { id: { in: ids } } }); + await app.close(); + await prisma.$disconnect(); + } + }); +}); diff --git a/src/app.ts b/src/app.ts index 21a9c80..4a927a1 100644 --- a/src/app.ts +++ b/src/app.ts @@ -39,6 +39,7 @@ export function buildApp({ repository, search }: AppDeps) { fastify.register(fastifySensible); fastify.register( async (api) => { + // Here rather than at the root, to keep CORS headers off /oauth and /admin api.register(cors, { origin: '*', methods: ['GET', 'HEAD', 'POST'], maxAge: 2 * 60 * 60 }); api.register(arocapi, { prisma, diff --git a/test/app.ts b/test/app.ts index 762b168..3552b16 100644 --- a/test/app.ts +++ b/test/app.ts @@ -12,10 +12,15 @@ export type TestRepository = Repository | ((deps: RepositoryDeps) => Repository /** A repository with no content */ export const repository = { storage: () => ({ content: {} }), async *objects() {} } as unknown as Repository; -/** A repository whose every file holds `content` */ +/** A repository whose every file holds `content`, served as the file's media type */ export const repositoryWithContent = { storage: () => ({ - content: () => ({ type: 'stream', open: () => Readable.from(['content']), contentLength: 7, contentType: 'text/plain' }), + content: (file: { mediaType?: string }) => ({ + type: 'stream', + open: () => Readable.from(['content']), + contentLength: 7, + contentType: file.mediaType ?? 'text/plain', + }), }), } as unknown as Repository; From 7b1b1fa4fa8ef6dbd7e4bbf56e6269bce312c0af Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 14:39:21 +1100 Subject: [PATCH 03/20] refactor: rename the structural index to the database index (#55) --- README.md | 4 ++-- src/admin/actions.test.ts | 8 ++++---- src/admin/crates.test.ts | 4 ++-- src/admin/crates.ts | 2 +- src/entity-meta.ts | 2 +- .../{structural.test.ts => database.test.ts} | 14 +++++++------- src/indexer/{structural.ts => database.ts} | 8 ++++---- src/indexer/search.ts | 2 +- src/ocfl.ts | 4 ++-- test/admin.ts | 2 +- 10 files changed, 25 insertions(+), 25 deletions(-) rename src/indexer/{structural.test.ts => database.test.ts} (92%) rename src/indexer/{structural.ts => database.ts} (96%) diff --git a/README.md b/README.md index 2d26709..ba69012 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ app mounts as a Fastify plugin. ldacapi adds what LDaCA needs on top: - **OCFL storage**: crates live in an [OCFL](https://ocfl.io/) repository on disk (`src/ocfl.ts`). -- **Indexing**: crates are read from OCFL into the database (the structural +- **Indexing**: crates are read from OCFL into the database (the database index, used by arocapi's entity and file routes) and, as one search document per entity, into arocapi's OpenSearch search backend (`src/indexer/`). The search fields are declared in `src/indexer/search.ts`. @@ -26,7 +26,7 @@ app mounts as a Fastify plugin. ldacapi adds what LDaCA needs on top: decided by licence, checked against REMS (`src/auth.ts`). - **Files and metadata**: arocapi serves crate files from the OCFL repository through its OCFL storage backend, and each entity's RO-Crate metadata from - the structural index (`src/app.ts`). + the database index (`src/app.ts`). ## Getting started diff --git a/src/admin/actions.test.ts b/src/admin/actions.test.ts index ad88490..6edab98 100644 --- a/src/admin/actions.test.ts +++ b/src/admin/actions.test.ts @@ -77,7 +77,7 @@ describe('index and delete actions', () => { const waitUntilIndexed = async (...crateIds: string[]) => { await waitUntilSettled(); for (const crateId of crateIds) { - expect(await repository.getState(crateId)).toEqual({ structural: 'indexed', search: 'indexed' }); + expect(await repository.getState(crateId)).toEqual({ database: 'indexed', search: 'indexed' }); } }; @@ -178,7 +178,7 @@ describe('index and delete actions', () => { const page = await act(browser, 'index'); expect(page.body).toMatch(/ { await indexing; - expect(read).toEqual({ structural: 'queued', search: 'queued' }); + expect(read).toEqual({ database: 'queued', search: 'queued' }); expect(state).toEqual(read); }); @@ -307,7 +307,7 @@ describe('protecting the actions', () => { ])('refuses to delete %s while a crate it covers is %s', async (_, state, form, asked) => { const repository = repositoryOf([ { id: PEOPLE, state: {} }, - { id: EXTRA, state: { structural: state } }, + { id: EXTRA, state: { database: state } }, ]); const browser = await signIn((await start(repository)).app); diff --git a/src/admin/crates.test.ts b/src/admin/crates.test.ts index bdf8366..68464d4 100644 --- a/src/admin/crates.test.ts +++ b/src/admin/crates.test.ts @@ -112,8 +112,8 @@ describe('the crate tree', () => { await vi.waitFor(async () => { const page = await browser.get('/admin/'); const tree = crateTree(page.body); - expect(find(tree, BUNDLED)?.row).toMatch(/Structural: indexed .*Search: indexed/); - expect(find(tree, PEOPLE)?.row).toMatch(/Structural: not indexed .*Search: not indexed/); + expect(find(tree, BUNDLED)?.row).toMatch(/Database: indexed .*Search: indexed/); + expect(find(tree, PEOPLE)?.row).toMatch(/Database: not indexed .*Search: not indexed/); expect(page.body).not.toContain('http-equiv="refresh"'); }); }); diff --git a/src/admin/crates.ts b/src/admin/crates.ts index 091de16..0f9ebb8 100644 --- a/src/admin/crates.ts +++ b/src/admin/crates.ts @@ -2,7 +2,7 @@ import { hasState, type RepositoryObject, State } from '../repository.ts'; type Crate = RepositoryObject & { label: string; children: Crate[] }; -const INDEXES = { structural: 'Structural', search: 'Search' } as const; +const INDEXES = { database: 'Database', search: 'Search' } as const; /** How often the page reloads while an index is changing */ const REFRESH_SECONDS = 5; diff --git a/src/entity-meta.ts b/src/entity-meta.ts index 13fbde9..36946ce 100644 --- a/src/entity-meta.ts +++ b/src/entity-meta.ts @@ -5,7 +5,7 @@ const ACCESS_CONTROLS = ['Public', 'AuthorizationByApplication'] as const; /** How Oni shows access is granted: at once, or by applying in REMS */ export type AccessControl = (typeof ACCESS_CONTROLS)[number]; -/** What the structural index keeps in `Entity.meta` */ +/** What the database index keeps in `Entity.meta` */ export type EntityMeta = { rocrate: unknown; accessControl: AccessControl }; export const accessControlOf = (meta: unknown): AccessControl | undefined => { diff --git a/src/indexer/structural.test.ts b/src/indexer/database.test.ts similarity index 92% rename from src/indexer/structural.test.ts rename to src/indexer/database.test.ts index b8dbcfa..a0c0c1d 100644 --- a/src/indexer/structural.test.ts +++ b/src/indexer/database.test.ts @@ -3,8 +3,8 @@ import { ROCrate } from 'ro-crate'; import { afterEach, beforeAll, describe, expect, it } from 'vitest'; import { accessControlOf } from '../entity-meta.ts'; import { prisma } from '../prisma.ts'; +import { DatabaseIndexer, entityAsCrate } from './database.ts'; import type { CrateObject } from './indexer.ts'; -import { entityAsCrate, StructuralIndexer } from './structural.ts'; describe('entityAsCrate', () => { it('preserves the original RO-Crate context when serializing an entity', async () => { @@ -16,15 +16,15 @@ describe('entityAsCrate', () => { }); }); -describe('StructuralIndexer', () => { - const ROOT = 'arcp://name,structural-test'; +describe('DatabaseIndexer', () => { + const ROOT = 'arcp://name,database-test'; const OPEN = 'https://creativecommons.org/licenses/by/4.0/'; const RESTRICTED = 'https://www.ldaca.edu.au/licenses/restricted/v1/'; - let indexer: StructuralIndexer; + let indexer: DatabaseIndexer; beforeAll(async () => { - indexer = await StructuralIndexer.create({}); + indexer = await DatabaseIndexer.create({}); }); afterEach(async () => { @@ -40,7 +40,7 @@ describe('StructuralIndexer', () => { '@id': ROOT, '@type': ['Dataset', 'RepositoryCollection'], conformsTo: { '@id': 'https://w3id.org/ldac/profile#Collection' }, - name: 'Structural test', + name: 'Database test', license: OPEN, hasPart: files.map((f) => ({ '@id': f['@id'] })), }, @@ -48,7 +48,7 @@ describe('StructuralIndexer', () => { ], }); const crateObject: CrateObject = { - id: 'structural-test', + id: 'database-test', root: '/nowhere', text: async () => '', file: async (path) => { diff --git a/src/indexer/structural.ts b/src/indexer/database.ts similarity index 96% rename from src/indexer/structural.ts rename to src/indexer/database.ts index 94b27a4..ae75db4 100644 --- a/src/indexer/structural.ts +++ b/src/indexer/database.ts @@ -7,18 +7,18 @@ import { prisma } from '../prisma.ts'; import { firstStringOrId } from '../utils.ts'; import { type BaseOptions, type CrateFile, Indexer, RecordType, recordType } from './indexer.ts'; -const log = logger.child({ module: 'indexer/structural' }); +const log = logger.child({ module: 'indexer/database' }); type IndexRecord = { entity: Prisma.EntityUncheckedCreateInput; file?: Prisma.FileUncheckedCreateInput }; -type StructuralOptions = BaseOptions & { +type DatabaseOptions = BaseOptions & { memberOfField?: string; }; -export class StructuralIndexer extends Indexer { +export class DatabaseIndexer extends Indexer { memberOfField: string; - constructor(opt: StructuralOptions) { + constructor(opt: DatabaseOptions) { super(opt); this.memberOfField = opt.memberOfField || 'pcdm:memberOf'; } diff --git a/src/indexer/search.ts b/src/indexer/search.ts index fe23353..d7d77ad 100644 --- a/src/indexer/search.ts +++ b/src/indexer/search.ts @@ -5,9 +5,9 @@ import type { Entity, ROCrate } from 'ro-crate'; import { config } from '../configuration.ts'; import { logger } from '../logger.ts'; import { firstStringOrId } from '../utils.ts'; +import { pickSingleMemberOf } from './database.ts'; import type { BaseOptions, CrateObject } from './indexer.ts'; import { Indexer, recordType } from './indexer.ts'; -import { pickSingleMemberOf } from './structural.ts'; const log = logger.child({ module: 'indexer/search' }); diff --git a/src/ocfl.ts b/src/ocfl.ts index 909f593..3557b51 100644 --- a/src/ocfl.ts +++ b/src/ocfl.ts @@ -6,9 +6,9 @@ import type { StorageBackend } from 'arocapi'; import { OcflStorage } from 'arocapi/storage/ocfl'; import { type RawEntity, ROCrate } from 'ro-crate'; import { config } from './configuration.ts'; +import { DatabaseIndexer } from './indexer/database.ts'; import type { CrateObject, Indexer } from './indexer/indexer.ts'; import { createSearchBackend, SearchIndexer } from './indexer/search.ts'; -import { StructuralIndexer } from './indexer/structural.ts'; import { logger } from './logger.ts'; import { type IndexState, State } from './repository.ts'; @@ -24,7 +24,7 @@ let backend: StorageBackend; export async function init(opts: { opensearchClient: Client; search?: ReturnType }) { log.info('Initializing OCFL repository and indexers'); INDEXER = { - structural: await StructuralIndexer.create({ + database: await DatabaseIndexer.create({ defaultLicense, defaultMetadataLicense, }), diff --git a/test/admin.ts b/test/admin.ts index c7951e5..00b41b3 100644 --- a/test/admin.ts +++ b/test/admin.ts @@ -39,7 +39,7 @@ export const repositoryOf = (crates: { id: string; name?: string; state: Record< }; /** A repository with one crate, `PEOPLE`, whose search index is in `state` */ -export const repositoryIn = (state: string) => repositoryOf([{ id: PEOPLE, name: 'People', state: { structural: 'indexed', search: state } }]); +export const repositoryIn = (state: string) => repositoryOf([{ id: PEOPLE, name: 'People', state: { database: 'indexed', search: state } }]); /** Runs a fake upstream provider for a file's tests, then closes it and every app started with `start` */ export const adminTestbed = () => { From de1010e87bdce63084b69a1b6f20c0098d35432f Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 14:40:19 +1100 Subject: [PATCH 04/20] feat!: facet search on entityType, the entity's class URI (#56) --- deploy/oni/configuration.json | 2 +- docker/oni/configuration.json | 2 +- src/indexer/database.ts | 4 +-- src/indexer/indexer.ts | 5 ++- src/indexer/search.test.ts | 8 ++++- src/indexer/search.ts | 7 ++-- src/search.test.ts | 66 +++++++++++++++++++++++++++++++++++ 7 files changed, 85 insertions(+), 9 deletions(-) create mode 100644 src/search.test.ts diff --git a/deploy/oni/configuration.json b/deploy/oni/configuration.json index 51fac8f..be73144 100644 --- a/deploy/oni/configuration.json +++ b/deploy/oni/configuration.json @@ -325,7 +325,7 @@ }, { "display": "Record Type", - "name": "type" + "name": "entityType" }, { "display": "Language", diff --git a/docker/oni/configuration.json b/docker/oni/configuration.json index 85dea36..4ec48e1 100644 --- a/docker/oni/configuration.json +++ b/docker/oni/configuration.json @@ -325,7 +325,7 @@ }, { "display": "Record Type", - "name": "type" + "name": "entityType" }, { "display": "Language", diff --git a/src/indexer/database.ts b/src/indexer/database.ts index ae75db4..dede8e8 100644 --- a/src/indexer/database.ts +++ b/src/indexer/database.ts @@ -5,7 +5,7 @@ import type { Prisma } from '../generated/prisma/client.ts'; import { logger } from '../logger.ts'; import { prisma } from '../prisma.ts'; import { firstStringOrId } from '../utils.ts'; -import { type BaseOptions, type CrateFile, Indexer, RecordType, recordType } from './indexer.ts'; +import { type BaseOptions, type CrateFile, entityClass, Indexer, recordType } from './indexer.ts'; const log = logger.child({ module: 'indexer/database' }); @@ -41,7 +41,7 @@ export class DatabaseIndexer extends Indexer { id: entityId, name: entity.name?.join('; ') || entityId, description: entity.description?.join('; ') || '', - entityType: crate.getContextDefinition(entityType) || RecordType[entityType], + entityType: entityClass(crate, entityType), memberOf: pickSingleMemberOf(entity), rootCollection: crate.rootId, metadataLicenseId: metadataLicense, diff --git a/src/indexer/indexer.ts b/src/indexer/indexer.ts index 9bef0b2..a2fc973 100644 --- a/src/indexer/indexer.ts +++ b/src/indexer/indexer.ts @@ -145,7 +145,7 @@ export class Indexer { } } -export const RecordType: Record = config.indexType; +const RecordType: Record = config.indexType; /** The entity's first type that we index, if it conforms to that type's profile */ export function recordType(entity: Entity): string | undefined { @@ -156,3 +156,6 @@ export function recordType(entity: Entity): string | undefined { } return type; } + +/** The class URI of a record type, as the entity's row holds it */ +export const entityClass = (crate: ROCrate, type: string): string => crate.getContextDefinition(type) || RecordType[type]; diff --git a/src/indexer/search.test.ts b/src/indexer/search.test.ts index 1ab291e..7161b17 100644 --- a/src/indexer/search.test.ts +++ b/src/indexer/search.test.ts @@ -37,7 +37,7 @@ describe('searchDocument', () => { expect(await searchDocument(crate, crateObject, entity, LICENSE, path)).toMatchObject({ id: `${ROOT}/data.txt`, name: 'Data', - type: ['File'], + entityType: 'http://schema.org/MediaObject', inLanguage: ['English'], communicationMode: ['https://w3id.org/ldac/terms#SpokenLanguage'], encodingFormat: ['text/plain'], @@ -49,6 +49,12 @@ describe('searchDocument', () => { }); }); + it('has no type field', async () => { + const { crate, entity } = await crateWith({}); + + expect(await searchDocument(crate, crateObject, entity, LICENSE)).not.toHaveProperty('type'); + }); + it('leaves out dates it cannot store', async () => { const { crate, entity } = await crateWith({ datePublished: 'circa 1900', dateCreated: ['1901-02-03', 'unknown'] }); diff --git a/src/indexer/search.ts b/src/indexer/search.ts index d7d77ad..394d3db 100644 --- a/src/indexer/search.ts +++ b/src/indexer/search.ts @@ -7,7 +7,7 @@ import { logger } from '../logger.ts'; import { firstStringOrId } from '../utils.ts'; import { pickSingleMemberOf } from './database.ts'; import type { BaseOptions, CrateObject } from './indexer.ts'; -import { Indexer, recordType } from './indexer.ts'; +import { entityClass, Indexer, recordType } from './indexer.ts'; const log = logger.child({ module: 'indexer/search' }); @@ -15,7 +15,7 @@ const fields = { name: { type: 'text', label: 'Name', weight: 3, advanced: true, sort: true }, description: { type: 'text', label: 'Description', advanced: true }, text: { type: 'text', label: 'Text', advanced: true }, - type: { type: 'keyword', label: 'Record Type', facet: true }, + entityType: { type: 'keyword', label: 'Record Type', facet: true }, inLanguage: { type: 'keyword', label: 'Language', advanced: true, facet: true }, communicationMode: { type: 'keyword', label: 'Communication Mode', facet: true }, linguisticGenre: { type: 'keyword', label: 'Linguistic Genre', facet: true }, @@ -103,6 +103,7 @@ function isText(entity: Entity) { /** One entity's search document; File entities with text content include it */ export async function searchDocument(crate: ROCrate, crateObject: CrateObject, entity: Entity, license: string, path?: string): Promise { const id: string = entity['@id']; + const type = recordType(entity); const [datePublished] = dates(entity, 'datePublished'); const memberOf = pickSingleMemberOf(entity); const point = location(entity); @@ -110,7 +111,7 @@ export async function searchDocument(crate: ROCrate, crateObject: CrateObject, e id, name: entity.name?.join('; ') || id, description: labels(entity, 'description'), - type: entity['@type'], + ...(type && { entityType: entityClass(crate, type) }), inLanguage: labels(entity, 'inLanguage'), communicationMode: labels(entity, 'communicationMode'), linguisticGenre: labels(entity, 'linguisticGenre'), diff --git a/src/search.test.ts b/src/search.test.ts new file mode 100644 index 0000000..3ee8189 --- /dev/null +++ b/src/search.test.ts @@ -0,0 +1,66 @@ +import { execFileSync } from 'node:child_process'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { ocflRepository } from '../test/admin.ts'; +import { buildTestApp } from '../test/app.ts'; + +const COLLECTION = 'http://pcdm.org/models#Collection'; +const OBJECT = 'http://pcdm.org/models#Object'; +const MEDIA = 'http://schema.org/MediaObject'; +const BUNDLED = 'arcp://name,bundled'; + +let storage: string; +let started: Awaited>; + +const search = async (body: Record) => { + const response = await started.app.inject({ method: 'POST', url: `${started.config.prefix}/search`, payload: body }); + expect(response.statusCode).toBe(200); + return response.json(); +}; + +beforeAll(async () => { + storage = await mkdtemp(join(tmpdir(), 'ldacapi-search-')); + const ocflRoot = join(storage, 'ocfl'); + execFileSync('node', ['scripts/seed.ts'], { env: { ...process.env, OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, stdio: 'ignore' }); + started = await buildTestApp({ OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, { repository: ocflRepository }); + await started.repository.deleteIndex(); + await started.repository.createIndex(BUNDLED); +}); + +afterAll(async () => { + await started?.repository.deleteIndex(); + await started?.app.close(); + await started?.prisma.$disconnect(); + await rm(storage, { recursive: true, force: true }); +}); + +describe('search by entityType', () => { + it('declares an entityType facet and no type facet', async () => { + const response = await started.app.inject({ method: 'GET', url: `${started.config.prefix}/capabilities` }); + + const { facets } = response.json().search; + expect(facets).toHaveProperty('entityType'); + expect(facets).not.toHaveProperty('type'); + }); + + it("facets on each entity's class URI", async () => { + const { facets } = await search({ query: '' }); + + const names = facets.entityType.map(({ name }: { name: string }) => name); + expect(names).toEqual(expect.arrayContaining([COLLECTION, OBJECT, MEDIA])); + expect(names.every((name: string) => /^https?:\/\//.test(name))).toBe(true); + }); + + it('filters on entityType, agreeing with listing entities by entityType', async () => { + const { total, entities } = await search({ query: '', filters: { entityType: [OBJECT] }, limit: 100 }); + const listed = ( + await started.app.inject({ method: 'GET', url: `${started.config.prefix}/entities?entityType=${encodeURIComponent(OBJECT)}&limit=100` }) + ).json(); + + expect(total).toBeGreaterThan(0); + expect(entities.every(({ entityType }: { entityType: string }) => entityType === OBJECT)).toBe(true); + expect(entities.map(({ id }: { id: string }) => id).sort()).toEqual(listed.entities.map(({ id }: { id: string }) => id).sort()); + }); +}); From 1e01032094ffe87725e4e336e0f0a150db5f0748 Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 14:47:46 +1100 Subject: [PATCH 05/20] fix: find a crate's rows by meta.crateId, and remove its search documents through arocapi (#59) --- src/admin/actions.test.ts | 130 +++++++++++++++++++++++++++++++------- src/entity-meta.ts | 4 +- src/indexer/database.ts | 16 +++-- src/indexer/search.ts | 19 ++---- src/ocfl.ts | 38 ++++++++--- test/app.ts | 4 +- test/bootstrap.test.ts | 30 ++++++--- 7 files changed, 181 insertions(+), 60 deletions(-) diff --git a/src/admin/actions.test.ts b/src/admin/actions.test.ts index 6edab98..0b5ac78 100644 --- a/src/admin/actions.test.ts +++ b/src/admin/actions.test.ts @@ -12,6 +12,9 @@ import { hasState, type Repository, State } from '../repository.ts'; const BUNDLED = 'arcp://name,bundled'; const EXTRA = `${PEOPLE}/extra`; const PEOPLE_TOO = `${PEOPLE}2`; +const DISTRIBUTED_ROOT = 'arcp://name,distributed_root'; +const CONTENT_RESTRICTED = 'arcp://name,distributed_content_restricted'; +const METADATA_RESTRICTED = 'arcp://name,distributed_metadata_restricted'; const testbed = adminTestbed(); let storage: string; @@ -55,13 +58,30 @@ describe('index and delete actions', () => { let repository: Repository; let prisma: Awaited>['prisma']; let entityIndex: string; + let search: Awaited>['search']; - const entityCount = (crateId: string) => prisma.entity.count({ where: { rootCollection: crateId } }); + /** The ids of the entity rows indexed from `crateId` */ + const rowIds = async (crateId: string) => + (await prisma.entity.findMany({ where: { meta: { path: ['crateId'], equals: crateId } }, select: { id: true } })).map(({ id }) => id).sort(); - const searchCount = async (crateId: string) => { - await opensearch.indices.refresh({ index: entityIndex, ignore_unavailable: true }); - const { body } = await opensearch.count({ index: entityIndex, ignore_unavailable: true, body: { query: { term: { rootCollection: crateId } } } }); - return body.count; + const entityCount = async (crateId: string) => (await rowIds(crateId)).length; + + /** How many of `ids` have a search document */ + const searchCount = async (ids: string[]) => { + if (!ids.length || !(await opensearch.indices.exists({ index: entityIndex })).body) { + return 0; + } + const { total } = await search.search({ + searchType: 'basic', + query: '', + filters: { terms: {}, ranges: {} }, + limit: 1, + offset: 0, + sort: 'relevance', + order: 'asc', + visibility: { id: { in: ids } }, + }); + return total; }; /** Waits until no crate is queued, indexing or deleting, so an earlier action can't leak into the next */ @@ -83,7 +103,7 @@ describe('index and delete actions', () => { beforeAll(async () => { let config: Awaited>['config']; - ({ app, repository, prisma, config } = await start(ocflRepository)); + ({ app, repository, prisma, config, search } = await start(ocflRepository)); entityIndex = config.search.entityIndex; opensearch = new Client({ node: config.opensearchUrl }); }); @@ -101,10 +121,21 @@ describe('index and delete actions', () => { expect(flash(page.body)).toBe(`Indexing ${BUNDLED} has been queued.`); await waitUntilIndexed(BUNDLED); - expect(await entityCount(BUNDLED)).toBeGreaterThan(0); - expect(await searchCount(BUNDLED)).toBeGreaterThan(0); + const ids = await rowIds(BUNDLED); + expect(ids).toContain(BUNDLED); + expect(await searchCount(ids)).toBe(ids.length); expect(await entityCount(PEOPLE)).toBe(0); - expect(await searchCount(PEOPLE)).toBe(0); + }); + + it('records the crate each entity row came from', async () => { + const browser = await signIn(app); + + await act(browser, 'index'); + + await waitUntilIndexed(BUNDLED, PEOPLE, DISTRIBUTED_ROOT, CONTENT_RESTRICTED, METADATA_RESTRICTED); + const counts = await Promise.all([BUNDLED, PEOPLE, DISTRIBUTED_ROOT, CONTENT_RESTRICTED, METADATA_RESTRICTED].map(entityCount)); + expect(counts.every((count) => count > 0)).toBe(true); + expect(counts.reduce((sum, count) => sum + count)).toBe(await prisma.entity.count()); }); it('refuses to index an indexed crate, asking for a reindex', async () => { @@ -121,15 +152,30 @@ describe('index and delete actions', () => { const browser = await signIn(app); await act(browser, 'index', { crateId: BUNDLED }); await waitUntilIndexed(BUNDLED); - const entities = await entityCount(BUNDLED); - const documents = await searchCount(BUNDLED); + const ids = await rowIds(BUNDLED); const page = await act(browser, 'index', { crateId: BUNDLED, force: 'true' }); expect(flash(page.body)).toBe(`Reindexing ${BUNDLED} has been queued.`); await waitUntilIndexed(BUNDLED); - expect(await entityCount(BUNDLED)).toBe(entities); - expect(await searchCount(BUNDLED)).toBe(documents); + expect(await rowIds(BUNDLED)).toEqual(ids); + expect(await searchCount(ids)).toBe(ids.length); + }); + + it('reindexes one crate of a distributed collection, leaving the rest of it alone', async () => { + const browser = await signIn(app); + await act(browser, 'index'); + await waitUntilIndexed(DISTRIBUTED_ROOT, CONTENT_RESTRICTED, METADATA_RESTRICTED); + const ids = await rowIds(CONTENT_RESTRICTED); + const siblings = [...(await rowIds(DISTRIBUTED_ROOT)), ...(await rowIds(METADATA_RESTRICTED))]; + + await act(browser, 'index', { crateId: CONTENT_RESTRICTED, force: 'true' }); + + await waitUntilIndexed(CONTENT_RESTRICTED); + expect(await rowIds(CONTENT_RESTRICTED)).toEqual(ids); + expect(await searchCount(ids)).toBe(ids.length); + expect([...(await rowIds(DISTRIBUTED_ROOT)), ...(await rowIds(METADATA_RESTRICTED))]).toEqual(siblings); + expect(await searchCount(siblings)).toBe(siblings.length); }); it('indexes every crate', async () => { @@ -182,34 +228,74 @@ describe('index and delete actions', () => { await waitUntilIndexed(BUNDLED, PEOPLE); }); - it('deletes one crate’s indexes', async () => { + it('deletes one crate’s indexes, including its people and organisations', async () => { const browser = await signIn(app); await act(browser, 'index'); await waitUntilIndexed(BUNDLED, PEOPLE); + const ids = await rowIds(PEOPLE); + expect(ids).toEqual(expect.arrayContaining([`${PEOPLE}/JohnSmith`, 'arcp://name,organisation/ldaca'])); + const others = await rowIds(BUNDLED); - const page = await act(browser, 'delete', { crateId: BUNDLED }); + const page = await act(browser, 'delete', { crateId: PEOPLE }); - expect(flash(page.body)).toBe(`Deleting the indexes of ${BUNDLED} has started.`); + expect(flash(page.body)).toBe(`Deleting the indexes of ${PEOPLE} has started.`); await vi.waitFor(async () => { - expect(await entityCount(BUNDLED)).toBe(0); - expect(await searchCount(BUNDLED)).toBe(0); + expect(await prisma.entity.count({ where: { id: { in: ids } } })).toBe(0); + expect(await searchCount(ids)).toBe(0); }); - expect(await entityCount(PEOPLE)).toBeGreaterThan(0); - expect(await searchCount(PEOPLE)).toBeGreaterThan(0); + expect(await rowIds(BUNDLED)).toEqual(others); + expect(await searchCount(others)).toBe(others.length); + }); + + it('deletes one crate of a distributed collection, leaving the rest of it alone', async () => { + const browser = await signIn(app); + await act(browser, 'index'); + await waitUntilIndexed(DISTRIBUTED_ROOT, CONTENT_RESTRICTED, METADATA_RESTRICTED); + const ids = await rowIds(CONTENT_RESTRICTED); + const siblings = [...(await rowIds(DISTRIBUTED_ROOT)), ...(await rowIds(METADATA_RESTRICTED))]; + + await act(browser, 'delete', { crateId: CONTENT_RESTRICTED }); + + await vi.waitFor(async () => { + expect(await prisma.entity.count({ where: { id: { in: ids } } })).toBe(0); + expect(await searchCount(ids)).toBe(0); + }); + expect([...(await rowIds(DISTRIBUTED_ROOT)), ...(await rowIds(METADATA_RESTRICTED))]).toEqual(siblings); + expect(await searchCount(siblings)).toBe(siblings.length); + }); + + it('keeps a crate’s database rows when deleting its search documents fails', async () => { + const browser = await signIn(app); + await act(browser, 'index'); + await waitUntilIndexed(BUNDLED, PEOPLE); + const ids = await rowIds(PEOPLE); + const remove = vi.spyOn(search, 'remove').mockRejectedValue(new Error('Search is down')); + + try { + await act(browser, 'delete', { crateId: PEOPLE }); + await waitUntilSettled(); + expect(remove).toHaveBeenCalled(); + } finally { + remove.mockRestore(); + } + + expect(await rowIds(PEOPLE)).toEqual(ids); + expect(await searchCount(ids)).toBe(ids.length); + expect(await repository.getState(PEOPLE)).toEqual({ database: 'indexed', search: 'indexed' }); }); it('deletes every index', async () => { const browser = await signIn(app); await act(browser, 'index'); await waitUntilIndexed(BUNDLED, PEOPLE); + const ids = [...(await rowIds(BUNDLED)), ...(await rowIds(PEOPLE))]; const page = await act(browser, 'delete'); expect(flash(page.body)).toBe('Deleting every index has started.'); await vi.waitFor(async () => { expect(await prisma.entity.count()).toBe(0); - expect(await searchCount(BUNDLED)).toBe(0); - expect(await searchCount(PEOPLE)).toBe(0); + expect(await searchCount(ids)).toBe(0); }); }); diff --git a/src/entity-meta.ts b/src/entity-meta.ts index 36946ce..9c446c4 100644 --- a/src/entity-meta.ts +++ b/src/entity-meta.ts @@ -5,8 +5,8 @@ const ACCESS_CONTROLS = ['Public', 'AuthorizationByApplication'] as const; /** How Oni shows access is granted: at once, or by applying in REMS */ export type AccessControl = (typeof ACCESS_CONTROLS)[number]; -/** What the database index keeps in `Entity.meta` */ -export type EntityMeta = { rocrate: unknown; accessControl: AccessControl }; +/** What the database index keeps in `Entity.meta`; `crateId` is the crate the entity was indexed from */ +export type EntityMeta = { crateId: string; rocrate: unknown; accessControl: AccessControl }; export const accessControlOf = (meta: unknown): AccessControl | undefined => { const value = (meta as Partial | null)?.accessControl; diff --git a/src/indexer/database.ts b/src/indexer/database.ts index dede8e8..f48c78a 100644 --- a/src/indexer/database.ts +++ b/src/indexer/database.ts @@ -46,7 +46,7 @@ export class DatabaseIndexer extends Indexer { rootCollection: crate.rootId, metadataLicenseId: metadataLicense, contentLicenseId, - meta: { rocrate, accessControl: accessControlFor(contentLicenseId) } satisfies EntityMeta, + meta: { crateId, rocrate, accessControl: accessControlFor(contentLicenseId) } satisfies EntityMeta, }, }; if (entityType.endsWith('://schema.org/MediaObject') || entityType === 'File') { @@ -81,16 +81,24 @@ export class DatabaseIndexer extends Indexer { } async delete(crateId?: string) { - await prisma.file.deleteMany({ where: crateId ? { entity: { rootCollection: crateId } } : {} }); - await prisma.entity.deleteMany({ where: crateId ? { rootCollection: crateId } : {} }); + await prisma.file.deleteMany({ where: { entity: rowsOf(crateId) } }); + await prisma.entity.deleteMany({ where: rowsOf(crateId) }); log.debug({ crateId }, 'Index deleted'); } async count(crateId?: string) { - return await prisma.entity.count(crateId ? { where: { id: crateId } } : undefined); + return await countRows(crateId); } } +/** The entity rows indexed from `crateId`, or every row */ +const rowsOf = (crateId?: string): Prisma.EntityWhereInput => (crateId ? { meta: { path: ['crateId'], equals: crateId } } : {}); + +export const countRows = (crateId?: string) => prisma.entity.count({ where: rowsOf(crateId) }); + +/** The ids of the entity rows indexed from `crateId` */ +export const rowIds = async (crateId: string) => (await prisma.entity.findMany({ where: rowsOf(crateId), select: { id: true } })).map(({ id }) => id); + export function entityAsCrate(crate: ROCrate, entity: Entity, license: string) { const newCrate = new ROCrate({ '@context': crate['@context'] }, { array: true, link: true }); for (const key in entity) { diff --git a/src/indexer/search.ts b/src/indexer/search.ts index 394d3db..c9ed4a3 100644 --- a/src/indexer/search.ts +++ b/src/indexer/search.ts @@ -5,7 +5,7 @@ import type { Entity, ROCrate } from 'ro-crate'; import { config } from '../configuration.ts'; import { logger } from '../logger.ts'; import { firstStringOrId } from '../utils.ts'; -import { pickSingleMemberOf } from './database.ts'; +import { countRows, pickSingleMemberOf, rowIds } from './database.ts'; import type { BaseOptions, CrateObject } from './indexer.ts'; import { entityClass, Indexer, recordType } from './indexer.ts'; @@ -32,7 +32,8 @@ const fields = { type Document = SearchDocument; -export const createSearchBackend = (client: Client) => OpenSearchBackend({ client, fields, indexName: config.search.entityIndex }); +export const createSearchBackend = (client: Client, options: { refresh?: boolean | 'wait_for' } = {}) => + OpenSearchBackend({ client, fields, indexName: config.search.entityIndex, ...options }); type SearchBackend = ReturnType; @@ -160,9 +161,10 @@ export class SearchIndexer extends Indexer { await this.search.createIndex(); } + /** A crate's documents are found by its database rows, so delete them before the rows */ async delete(crateId?: string) { if (crateId) { - await this.client.deleteByQuery({ index: this.indexName, refresh: true, body: { query: { term: { rootCollection: crateId } } } }); + await this.search.remove(await rowIds(crateId)); } else { await this.client.indices.delete({ index: this.indexName, ignore_unavailable: true }); } @@ -170,15 +172,7 @@ export class SearchIndexer extends Indexer { } async count(crateId?: string) { - try { - const res = await this.client.count({ - index: this.indexName, - ...(crateId && { body: { query: { term: { rootCollection: crateId } } } }), - }); - return res.body.count; - } catch { - return 0; - } + return await countRows(crateId); } async _index({ crateObject, crate, paths, license }: Parameters[0]) { @@ -197,7 +191,6 @@ export class SearchIndexer extends Indexer { } } await this.indexDocuments(docs); - await this.client.indices.refresh({ index: this.indexName }); } /** Index a batch, falling back to one at a time so one bad document doesn't lose the rest */ diff --git a/src/ocfl.ts b/src/ocfl.ts index 3557b51..d3b1a1d 100644 --- a/src/ocfl.ts +++ b/src/ocfl.ts @@ -142,6 +142,21 @@ export async function getState(crateId: string, type?: string) { return type ? { [type]: stateCache[crateId]?.[type] } : { ...stateCache[crateId] }; } +/** The search index finds a crate's documents by its database rows, so is deleted from first */ +const DELETE_ORDER = ['search', 'database']; + +/** Stops at the first failure, so the rows that find a crate's search documents outlive them */ +async function deleteFrom(types: string[], crateId?: string) { + for (const t of types.toSorted((a, b) => DELETE_ORDER.indexOf(a) - DELETE_ORDER.indexOf(b))) { + try { + await INDEXER[t]?.delete(crateId); + } catch (error) { + log.error({ err: error, crateId, indexer: t }, 'Error deleting index'); + throw error; + } + } +} + async function indexObject(crateId: string, types: string[], force?: boolean) { try { const ocflObject = repository.object(crateId); @@ -150,16 +165,21 @@ async function indexObject(crateId: string, types: string[], force?: boolean) { const jsonContent = await ocflObject.getFile({ logicalPath: 'ro-crate-metadata.json' }).text(); const rawCrate = JSON.parse(jsonContent); const crate = await ROCrate.create(rawCrate); + if (force) { + setState(ocflObject.id, types, State.INDEXING); + try { + await deleteFrom(types, crate.rootId); + } catch { + setState(ocflObject.id, types); + return; + } + } for (const t of types) { const indexer = INDEXER[t]; if (indexer) { setState(ocflObject.id, t, State.INDEXING); try { - if (force) { - await indexer.delete(crate.rootId); - } - const crateObject = wrap(ocflObject); - await indexer.index({ crateObject, crate }); + await indexer.index({ crateObject: wrap(ocflObject), crate }); } catch (error) { log.error({ err: error, crateId: ocflObject.id, indexer: t }, 'Error indexing object'); } finally { @@ -201,10 +221,12 @@ export async function createIndex(crateIds?: string | string[] | RegExp, type?: export async function deleteIndex(crateId?: string, type?: string | string[]) { const types = type ? ([] as string[]).concat(type) : Object.keys(INDEXER); - const indexers = types.map((t) => INDEXER[t]).filter((i) => !!i); setState(crateId, types, State.DELETING); - await Promise.allSettled(indexers.map((indexer) => indexer.delete(crateId))); - setState(crateId, types); + try { + await deleteFrom(types, crateId); + } finally { + setState(crateId, types); + } } export async function* objects(prefix?: string, refresh?: boolean) { diff --git a/test/app.ts b/test/app.ts index 3552b16..636456b 100644 --- a/test/app.ts +++ b/test/app.ts @@ -36,11 +36,11 @@ export const buildTestApp = async (env: Record, options: { repos const { Client } = await import('@opensearch-project/opensearch'); const { createSearchBackend } = await import('../src/indexer/search.ts'); const opensearchClient = new Client({ node: config.opensearchUrl }); - const search = createSearchBackend(opensearchClient); + const search = createSearchBackend(opensearchClient, { refresh: true }); const repo = typeof options.repository === 'function' ? await options.repository({ opensearchClient, search }) : (options.repository ?? repository); /** Builds another app, as another server process would */ const build = () => buildApp({ repository: repo, search }); - return { app: build(), build, config, prisma, repository: repo }; + return { app: build(), build, config, prisma, repository: repo, search }; } finally { vi.unstubAllEnvs(); } diff --git a/test/bootstrap.test.ts b/test/bootstrap.test.ts index d5eef81..19026e1 100644 --- a/test/bootstrap.test.ts +++ b/test/bootstrap.test.ts @@ -3,8 +3,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { Client } from '@opensearch-project/opensearch'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; import { config } from '../src/configuration.ts'; +import { createSearchBackend } from '../src/indexer/search.ts'; import { prisma } from '../src/prisma.ts'; const crateIds = [ @@ -37,14 +38,25 @@ describe('seed and index', () => { it('indexes every crate into OpenSearch', async () => { const opensearch = new Client({ node: config.opensearchUrl }); - const indexed = await Promise.all( - crateIds.map(async (id) => { - const { body } = await opensearch.count({ index: config.search.entityIndex, body: { query: { term: { rootCollection: id } } } }); - return body.count > 0; - }), - ); + const search = createSearchBackend(opensearch); + const rows = await prisma.entity.findMany({ select: { id: true } }); + const ids = rows.map(({ id }) => id); + expect(ids).toEqual(expect.arrayContaining(crateIds)); + + // Documents are searchable after OpenSearch's next refresh + await vi.waitFor(async () => { + const { total } = await search.search({ + searchType: 'basic', + query: '', + filters: { terms: {}, ranges: {} }, + limit: 1, + offset: 0, + sort: 'relevance', + order: 'asc', + visibility: { id: { in: ids } }, + }); + expect(total).toBe(ids.length); + }); await opensearch.close(); - - expect(indexed).toEqual(crateIds.map(() => true)); }); }); From 44d71844e3fef2e4d3eee5b6bea299be73714d9a Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 14:51:06 +1100 Subject: [PATCH 06/20] feat!: rootCollection is the top-level collection, across distributed crates (#60) --- src/indexer/database.ts | 6 +-- src/indexer/indexer.ts | 17 +++++++-- src/indexer/search.test.ts | 6 +-- src/indexer/search.ts | 18 ++++++--- src/ocfl.ts | 57 +++++++++++++++++++++++----- src/root-collection.test.ts | 74 +++++++++++++++++++++++++++++++++++++ 6 files changed, 155 insertions(+), 23 deletions(-) create mode 100644 src/root-collection.test.ts diff --git a/src/indexer/database.ts b/src/indexer/database.ts index f48c78a..f8fa57f 100644 --- a/src/indexer/database.ts +++ b/src/indexer/database.ts @@ -5,7 +5,7 @@ import type { Prisma } from '../generated/prisma/client.ts'; import { logger } from '../logger.ts'; import { prisma } from '../prisma.ts'; import { firstStringOrId } from '../utils.ts'; -import { type BaseOptions, type CrateFile, entityClass, Indexer, recordType } from './indexer.ts'; +import { type BaseOptions, type CrateFile, entityClass, Indexer, recordType, rootCollection } from './indexer.ts'; const log = logger.child({ module: 'indexer/database' }); @@ -23,7 +23,7 @@ export class DatabaseIndexer extends Indexer { this.memberOfField = opt.memberOfField || 'pcdm:memberOf'; } - override async _index({ crateObject, crate, paths, license, metadataLicense }: Parameters[0]) { + override async _index({ crateObject, crate, paths, license, metadataLicense, topCollection }: Parameters[0]) { const crateId = crate.rootId; let count = 0; for (const entity of crate.entities()) { @@ -43,7 +43,7 @@ export class DatabaseIndexer extends Indexer { description: entity.description?.join('; ') || '', entityType: entityClass(crate, entityType), memberOf: pickSingleMemberOf(entity), - rootCollection: crate.rootId, + rootCollection: rootCollection(topCollection, entityId), metadataLicenseId: metadataLicense, contentLicenseId, meta: { crateId, rocrate, accessControl: accessControlFor(contentLicenseId) } satisfies EntityMeta, diff --git a/src/indexer/indexer.ts b/src/indexer/indexer.ts index a2fc973..8d77a88 100644 --- a/src/indexer/indexer.ts +++ b/src/indexer/indexer.ts @@ -110,7 +110,8 @@ export class Indexer { throw new Error('Not Implemented'); } - async index({ crateObject, crate }: { crateObject: CrateObject; crate: ROCrate }) { + /** `topCollection` is the top-level collection the crate sits under, the crate's own root if it has no parent */ + async index({ crateObject, crate, topCollection = crate.rootId }: { crateObject: CrateObject; crate: ROCrate; topCollection?: string }) { const rootDataset = crate.root; const crateId = crate.rootId; const metadataLicense = firstStringOrId(crate.descriptor.license) || this.defaultMetadataLicense; @@ -127,7 +128,7 @@ export class Indexer { log.info({ crateId }, 'Indexing crate'); const paths = this.deriveUniqueEntityIds(crate); try { - await this._index({ crateObject, crate, paths, license, metadataLicense }); + await this._index({ crateObject, crate, paths, license, metadataLicense, topCollection }); } catch (error) { log.error({ err: error, crateId, path: crateObject.root }, 'Error indexing crate'); } @@ -135,7 +136,7 @@ export class Indexer { } } - async _index(_param: { crateObject: CrateObject; crate: ROCrate; paths: LogicalPaths; license: string; metadataLicense: string }) { + async _index(_param: { crateObject: CrateObject; crate: ROCrate; paths: LogicalPaths; license: string; metadataLicense: string; topCollection: string }) { throw new Error('Not Implemented'); } @@ -145,6 +146,16 @@ export class Indexer { } } +/** An entity's rootCollection: empty on the top-level collection itself */ +export const rootCollection = (topCollection: string, entityId: string) => (entityId === topCollection ? null : topCollection); + +/** The id of the collection a crate's root is a member of, if any */ +export function parentCollection(crate: ROCrate): string | undefined { + const root = crate.root; + const parent: string | undefined = (root['pcdm:memberOf'] ?? root.memberOf)?.[0]?.['@id']; + return parent && parent !== crate.rootId ? parent : undefined; +} + const RecordType: Record = config.indexType; /** The entity's first type that we index, if it conforms to that type's profile */ diff --git a/src/indexer/search.test.ts b/src/indexer/search.test.ts index 7161b17..19d3397 100644 --- a/src/indexer/search.test.ts +++ b/src/indexer/search.test.ts @@ -34,7 +34,7 @@ describe('searchDocument', () => { contentLocation: { '@id': '#place' }, }); - expect(await searchDocument(crate, crateObject, entity, LICENSE, path)).toMatchObject({ + expect(await searchDocument(crate, crateObject, entity, LICENSE, ROOT, path)).toMatchObject({ id: `${ROOT}/data.txt`, name: 'Data', entityType: 'http://schema.org/MediaObject', @@ -52,13 +52,13 @@ describe('searchDocument', () => { it('has no type field', async () => { const { crate, entity } = await crateWith({}); - expect(await searchDocument(crate, crateObject, entity, LICENSE)).not.toHaveProperty('type'); + expect(await searchDocument(crate, crateObject, entity, LICENSE, ROOT)).not.toHaveProperty('type'); }); it('leaves out dates it cannot store', async () => { const { crate, entity } = await crateWith({ datePublished: 'circa 1900', dateCreated: ['1901-02-03', 'unknown'] }); - const doc = await searchDocument(crate, crateObject, entity, LICENSE); + const doc = await searchDocument(crate, crateObject, entity, LICENSE, ROOT); expect(doc).not.toHaveProperty('datePublished'); expect(doc.dateCreated).toEqual(['1901-02-03']); diff --git a/src/indexer/search.ts b/src/indexer/search.ts index c9ed4a3..ef36808 100644 --- a/src/indexer/search.ts +++ b/src/indexer/search.ts @@ -7,7 +7,7 @@ import { logger } from '../logger.ts'; import { firstStringOrId } from '../utils.ts'; import { countRows, pickSingleMemberOf, rowIds } from './database.ts'; import type { BaseOptions, CrateObject } from './indexer.ts'; -import { entityClass, Indexer, recordType } from './indexer.ts'; +import { entityClass, Indexer, recordType, rootCollection } from './indexer.ts'; const log = logger.child({ module: 'indexer/search' }); @@ -102,12 +102,20 @@ function isText(entity: Entity) { } /** One entity's search document; File entities with text content include it */ -export async function searchDocument(crate: ROCrate, crateObject: CrateObject, entity: Entity, license: string, path?: string): Promise { +export async function searchDocument( + crate: ROCrate, + crateObject: CrateObject, + entity: Entity, + license: string, + topCollection: string, + path?: string, +): Promise { const id: string = entity['@id']; const type = recordType(entity); const [datePublished] = dates(entity, 'datePublished'); const memberOf = pickSingleMemberOf(entity); const point = location(entity); + const root = rootCollection(topCollection, id); const doc: Document = { id, name: entity.name?.join('; ') || id, @@ -118,7 +126,7 @@ export async function searchDocument(crate: ROCrate, crateObject: CrateObject, e linguisticGenre: labels(entity, 'linguisticGenre'), annotationType: labels(entity, 'annotationType'), encodingFormat: labels(entity, 'encodingFormat'), - rootCollection: crate.rootId, + ...(root && { rootCollection: root }), ...(memberOf && { memberOf }), license: firstStringOrId(entity.license) || license, ...(datePublished && { datePublished }), @@ -175,14 +183,14 @@ export class SearchIndexer extends Indexer { return await countRows(crateId); } - async _index({ crateObject, crate, paths, license }: Parameters[0]) { + async _index({ crateObject, crate, paths, license, topCollection }: Parameters[0]) { await this.search.createIndex(); const docs: Document[] = []; for (const entity of crate.entities()) { if (!recordType(entity)) { continue; } - const doc = await searchDocument(crate, crateObject, entity, license, paths.get(entity['@id'])); + const doc = await searchDocument(crate, crateObject, entity, license, topCollection, paths.get(entity['@id'])); if (doc.text) { // Indexed alone to keep each request small await this.indexDocuments([doc]); diff --git a/src/ocfl.ts b/src/ocfl.ts index d3b1a1d..4cc2819 100644 --- a/src/ocfl.ts +++ b/src/ocfl.ts @@ -7,7 +7,7 @@ import { OcflStorage } from 'arocapi/storage/ocfl'; import { type RawEntity, ROCrate } from 'ro-crate'; import { config } from './configuration.ts'; import { DatabaseIndexer } from './indexer/database.ts'; -import type { CrateObject, Indexer } from './indexer/indexer.ts'; +import { type CrateObject, type Indexer, parentCollection } from './indexer/indexer.ts'; import { createSearchBackend, SearchIndexer } from './indexer/search.ts'; import { logger } from './logger.ts'; import { type IndexState, State } from './repository.ts'; @@ -157,14 +157,51 @@ async function deleteFrom(types: string[], crateId?: string) { } } -async function indexObject(crateId: string, types: string[], force?: boolean) { +/** A crate's OCFL object, whose id is the crate's root id, and its metadata */ +async function readCrate(crateId: string) { + const ocflObject = repository.object(crateId); + await ocflObject.load(); + const jsonContent = await ocflObject.getFile({ logicalPath: 'ro-crate-metadata.json' }).text(); + return { ocflObject, crate: await ROCrate.create(JSON.parse(jsonContent)) }; +} + +/** + * Finds the top-level collection a crate sits under by following memberOf up through the parent crates in the repository, + * so it doesn't depend on which crates are indexed yet. Each crate's parent is cached for the life of the finder. + */ +function topCollectionFinder() { + const parents = new Map>(); + const parentOf = (crateId: string) => { + let parent = parents.get(crateId); + if (!parent) { + parent = readCrate(crateId).then(({ crate }) => parentCollection(crate)); + parents.set(crateId, parent); + } + return parent; + }; + return async (crate: ROCrate) => { + let top = crate.rootId; + let parent = parentCollection(crate); + const seen = new Set([top]); + while (parent && !seen.has(parent)) { + seen.add(parent); + try { + const next = await parentOf(parent); + top = parent; + parent = next; + } catch (error) { + log.warn({ err: error, crateId: crate.rootId, parent, topCollection: top }, 'Cannot read parent collection; using the highest found'); + break; + } + } + return top; + }; +} + +async function indexObject(crateId: string, types: string[], findTopCollection: ReturnType, force?: boolean) { try { - const ocflObject = repository.object(crateId); - await ocflObject.load(); + const { ocflObject, crate } = await readCrate(crateId); log.debug({ crateId: ocflObject.id }, 'Found OCFL object'); - const jsonContent = await ocflObject.getFile({ logicalPath: 'ro-crate-metadata.json' }).text(); - const rawCrate = JSON.parse(jsonContent); - const crate = await ROCrate.create(rawCrate); if (force) { setState(ocflObject.id, types, State.INDEXING); try { @@ -174,12 +211,13 @@ async function indexObject(crateId: string, types: string[], force?: boolean) { return; } } + const topCollection = await findTopCollection(crate); for (const t of types) { const indexer = INDEXER[t]; if (indexer) { setState(ocflObject.id, t, State.INDEXING); try { - await indexer.index({ crateObject: wrap(ocflObject), crate }); + await indexer.index({ crateObject: wrap(ocflObject), crate, topCollection }); } catch (error) { log.error({ err: error, crateId: ocflObject.id, indexer: t }, 'Error indexing object'); } finally { @@ -213,8 +251,9 @@ export async function createIndex(crateIds?: string | string[] | RegExp, type?: for (const id of ids) { setState(id, types, State.QUEUED); } + const findTopCollection = topCollectionFinder(); for (const id of ids) { - await indexObject(id, types, force); + await indexObject(id, types, findTopCollection, force); } log.debug('Indexing finished'); } diff --git a/src/root-collection.test.ts b/src/root-collection.test.ts new file mode 100644 index 0000000..72e4e63 --- /dev/null +++ b/src/root-collection.test.ts @@ -0,0 +1,74 @@ +import { execFileSync } from 'node:child_process'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { ocflRepository } from '../test/admin.ts'; +import { buildTestApp } from '../test/app.ts'; + +const BUNDLED = 'arcp://name,bundled'; +const ROOT = 'arcp://name,distributed_root'; +const CONTENT = 'arcp://name,distributed_content_restricted'; +const METADATA = 'arcp://name,distributed_metadata_restricted'; + +let storage: string; +let started: Awaited>; + +const rootCollectionOf = async (id: string) => { + const response = await started.app.inject({ method: 'GET', url: `${started.config.prefix}/entity/${encodeURIComponent(id)}` }); + expect(response.statusCode).toBe(200); + return response.json().rootCollection?.id ?? null; +}; + +const collectionFacet = async () => { + const response = await started.app.inject({ method: 'POST', url: `${started.config.prefix}/search`, payload: { query: '' } }); + expect(response.statusCode).toBe(200); + return response.json().facets.rootCollection.map(({ name }: { name: string }) => name); +}; + +beforeAll(async () => { + storage = await mkdtemp(join(tmpdir(), 'ldacapi-root-collection-')); + const ocflRoot = join(storage, 'ocfl'); + execFileSync('node', ['scripts/seed.ts'], { env: { ...process.env, OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, stdio: 'ignore' }); + started = await buildTestApp({ OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, { repository: ocflRepository }); +}); + +afterAll(async () => { + await started?.repository.deleteIndex(); + await started?.app.close(); + await started?.prisma.$disconnect(); + await rm(storage, { recursive: true, force: true }); +}); + +describe.each([ + ['before', [ROOT, CONTENT, METADATA, BUNDLED]], + ['after', [CONTENT, METADATA, ROOT, BUNDLED]], +])('rootCollection, with the distributed parent crate indexed %s its children', (_order, crates) => { + beforeAll(async () => { + await started.repository.deleteIndex(); + await started.repository.createIndex(crates); + }); + + it('is empty on a top-level collection', async () => { + expect(await rootCollectionOf(ROOT)).toBeNull(); + expect(await rootCollectionOf(BUNDLED)).toBeNull(); + }); + + it("is the distributed collection's root for the entities of its nested crates", async () => { + for (const id of [CONTENT, `${CONTENT}/main`, METADATA, `${METADATA}/main`]) { + expect(await rootCollectionOf(id)).toBe(ROOT); + } + }); + + it("is a stand-alone crate's root for its other entities", async () => { + expect(await rootCollectionOf(`${BUNDLED}/ch1`)).toBe(BUNDLED); + }); + + it('lists the distributed collection once in the Collection facet', async () => { + const names = await collectionFacet(); + + expect(names.filter((name: string) => name === ROOT)).toHaveLength(1); + expect(names).not.toContain(CONTENT); + expect(names).not.toContain(METADATA); + }); +}); From b6474659fe5a9c4d3871a704ec1da1226fff0dc3 Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 14:53:38 +1100 Subject: [PATCH 07/20] fix: index no search document for an entity without a database row (#61) --- src/indexer/database.ts | 3 ++ src/indexer/indexer.ts | 35 ++++++++++++--- src/indexer/search.ts | 12 ++++- src/ocfl.ts | 5 ++- src/search-total.test.ts | 95 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 141 insertions(+), 9 deletions(-) create mode 100644 src/search-total.test.ts diff --git a/src/indexer/database.ts b/src/indexer/database.ts index f8fa57f..96bf14d 100644 --- a/src/indexer/database.ts +++ b/src/indexer/database.ts @@ -25,6 +25,7 @@ export class DatabaseIndexer extends Indexer { override async _index({ crateObject, crate, paths, license, metadataLicense, topCollection }: Parameters[0]) { const crateId = crate.rootId; + const written = new Set(); let count = 0; for (const entity of crate.entities()) { const entityType = recordType(entity); @@ -69,6 +70,7 @@ export class DatabaseIndexer extends Indexer { } try { await prisma.entity.create({ data: param.entity }); + written.add(entityId); if (param.file) { await prisma.file.create({ data: param.file }); } @@ -78,6 +80,7 @@ export class DatabaseIndexer extends Indexer { } log.info({ crateId, entities: count }, 'Indexed crate'); + return written; } async delete(crateId?: string) { diff --git a/src/indexer/indexer.ts b/src/indexer/indexer.ts index 8d77a88..3eecf23 100644 --- a/src/indexer/indexer.ts +++ b/src/indexer/indexer.ts @@ -24,6 +24,8 @@ function crateRelativePath(ref: string) { } } +export type EntityIds = ReadonlySet; + /** Entity id → its path in the crate */ export type LogicalPaths = Map; @@ -110,8 +112,21 @@ export class Indexer { throw new Error('Not Implemented'); } - /** `topCollection` is the top-level collection the crate sits under, the crate's own root if it has no parent */ - async index({ crateObject, crate, topCollection = crate.rootId }: { crateObject: CrateObject; crate: ROCrate; topCollection?: string }) { + /** + * `topCollection` is the top-level collection the crate sits under, the crate's own root if it has no parent. + * Returns the ids of the entity rows written, if this indexer writes them. + */ + async index({ + crateObject, + crate, + topCollection = crate.rootId, + rows, + }: { + crateObject: CrateObject; + crate: ROCrate; + topCollection?: string; + rows?: EntityIds | undefined; + }): Promise { const rootDataset = crate.root; const crateId = crate.rootId; const metadataLicense = firstStringOrId(crate.descriptor.license) || this.defaultMetadataLicense; @@ -128,15 +143,25 @@ export class Indexer { log.info({ crateId }, 'Indexing crate'); const paths = this.deriveUniqueEntityIds(crate); try { - await this._index({ crateObject, crate, paths, license, metadataLicense, topCollection }); + return await this._index({ crateObject, crate, paths, license, metadataLicense, topCollection, rows }); } catch (error) { log.error({ err: error, crateId, path: crateObject.root }, 'Error indexing crate'); + } finally { + log.info({ crateId }, 'Indexed crate'); } - log.info({ crateId }, 'Indexed crate'); } } - async _index(_param: { crateObject: CrateObject; crate: ROCrate; paths: LogicalPaths; license: string; metadataLicense: string; topCollection: string }) { + async _index(_param: { + crateObject: CrateObject; + crate: ROCrate; + paths: LogicalPaths; + license: string; + metadataLicense: string; + topCollection: string; + /** The ids of the crate's entity rows, once the database indexer has written them */ + rows?: EntityIds | undefined; + }): Promise { throw new Error('Not Implemented'); } diff --git a/src/indexer/search.ts b/src/indexer/search.ts index ef36808..64000db 100644 --- a/src/indexer/search.ts +++ b/src/indexer/search.ts @@ -183,14 +183,22 @@ export class SearchIndexer extends Indexer { return await countRows(crateId); } - async _index({ crateObject, crate, paths, license, topCollection }: Parameters[0]) { + /** Only entities with a row, as arocapi drops a hit without one but still counts it */ + async _index({ crateObject, crate, paths, license, topCollection, rows }: Parameters[0]): Promise { await this.search.createIndex(); + const crateId = crate.rootId; + const withRows = rows ?? new Set(await rowIds(crateId)); const docs: Document[] = []; for (const entity of crate.entities()) { if (!recordType(entity)) { continue; } - const doc = await searchDocument(crate, crateObject, entity, license, topCollection, paths.get(entity['@id'])); + const entityId = entity['@id']; + if (!withRows.has(entityId)) { + log.warn({ crateId, entityId }, 'Skipped: no database row'); + continue; + } + const doc = await searchDocument(crate, crateObject, entity, license, topCollection, paths.get(entityId)); if (doc.text) { // Indexed alone to keep each request small await this.indexDocuments([doc]); diff --git a/src/ocfl.ts b/src/ocfl.ts index 4cc2819..4bbac0f 100644 --- a/src/ocfl.ts +++ b/src/ocfl.ts @@ -7,7 +7,7 @@ import { OcflStorage } from 'arocapi/storage/ocfl'; import { type RawEntity, ROCrate } from 'ro-crate'; import { config } from './configuration.ts'; import { DatabaseIndexer } from './indexer/database.ts'; -import { type CrateObject, type Indexer, parentCollection } from './indexer/indexer.ts'; +import { type CrateObject, type EntityIds, type Indexer, parentCollection } from './indexer/indexer.ts'; import { createSearchBackend, SearchIndexer } from './indexer/search.ts'; import { logger } from './logger.ts'; import { type IndexState, State } from './repository.ts'; @@ -212,12 +212,13 @@ async function indexObject(crateId: string, types: string[], findTopCollection: } } const topCollection = await findTopCollection(crate); + let rows: EntityIds | undefined; for (const t of types) { const indexer = INDEXER[t]; if (indexer) { setState(ocflObject.id, t, State.INDEXING); try { - await indexer.index({ crateObject: wrap(ocflObject), crate, topCollection }); + rows = (await indexer.index({ crateObject: wrap(ocflObject), crate, topCollection, rows })) ?? rows; } catch (error) { log.error({ err: error, crateId: ocflObject.id, indexer: t }, 'Error indexing object'); } finally { diff --git a/src/search-total.test.ts b/src/search-total.test.ts new file mode 100644 index 0000000..8fa851c --- /dev/null +++ b/src/search-total.test.ts @@ -0,0 +1,95 @@ +import { execFileSync } from 'node:child_process'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Writable } from 'node:stream'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { ocflRepository } from '../test/admin.ts'; +import { buildTestApp } from '../test/app.ts'; + +const logs = vi.hoisted(() => [] as Record[]); + +vi.mock('./logger.ts', async () => { + const { pino } = await import('pino'); + const stream = new Writable({ + write(chunk, _encoding, done) { + logs.push(JSON.parse(chunk.toString())); + done(); + }, + }); + return { logger: pino({ level: 'info' }, stream) }; +}); + +const ROOT = 'arcp://name,unwritable'; +const WRITTEN = `${ROOT}/written`; +// Postgres can't store a NUL character, so this entity's row fails to be written +const UNWRITTEN = `${ROOT}/unwritten`; +const LICENSE = { '@id': 'https://creativecommons.org/licenses/by/4.0/' }; +const OBJECT = { '@id': 'https://w3id.org/ldac/profile#Object' }; + +const crate = { + '@context': ['https://w3id.org/ro/crate/1.1/context', 'https://w3id.org/ldac/context', { '@vocab': 'http://schema.org/' }], + '@graph': [ + { '@id': 'ro-crate-metadata.json', '@type': 'CreativeWork', license: LICENSE, about: { '@id': ROOT } }, + { + '@id': ROOT, + '@type': ['Dataset', 'RepositoryCollection'], + conformsTo: { '@id': 'https://w3id.org/ldac/profile#Collection' }, + name: 'Unwritable', + license: LICENSE, + }, + { '@id': WRITTEN, '@type': 'RepositoryObject', conformsTo: OBJECT, memberOf: { '@id': ROOT }, name: 'Written' }, + { '@id': UNWRITTEN, '@type': 'RepositoryObject', conformsTo: OBJECT, memberOf: { '@id': ROOT }, name: 'Unwritten\u0000' }, + ], +}; + +let storage: string; +let started: Awaited>; + +const search = async (body: Record) => { + const response = await started.app.inject({ method: 'POST', url: `${started.config.prefix}/search`, payload: body }); + expect(response.statusCode).toBe(200); + return response.json(); +}; + +beforeAll(async () => { + storage = await mkdtemp(join(tmpdir(), 'ldacapi-search-total-')); + const source = join(storage, 'crates'); + await mkdir(join(source, 'unwritable'), { recursive: true }); + await writeFile(join(source, 'unwritable', 'ro-crate-metadata.json'), JSON.stringify(crate)); + const ocflRoot = join(storage, 'ocfl'); + execFileSync('node', ['scripts/seed.ts', source], { env: { ...process.env, OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, stdio: 'ignore' }); + started = await buildTestApp({ OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, { repository: ocflRepository }); + await started.repository.deleteIndex(); + await started.repository.createIndex(ROOT); +}); + +afterAll(async () => { + await started?.repository.deleteIndex(); + await started?.app.close(); + await started?.prisma.$disconnect(); + await rm(storage, { recursive: true, force: true }); +}); + +describe('an entity whose row fails to be written', () => { + it('is not counted by search', async () => { + const { total, entities, facets } = await search({ query: '', limit: 100 }); + + expect(entities.map(({ id }: { id: string }) => id).sort()).toEqual([ROOT, WRITTEN]); + expect(total).toBe(2); + expect(facets.entityType.reduce((sum: number, { count }: { count: number }) => sum + count, 0)).toBe(2); + }); + + it('is not counted when the search index is rebuilt alone', async () => { + await started.repository.createIndex(ROOT, 'search', true); + + const { total, entities } = await search({ query: '', limit: 100 }); + + expect(entities.map(({ id }: { id: string }) => id).sort()).toEqual([ROOT, WRITTEN]); + expect(total).toBe(2); + }); + + it('logs the skip with the crate and entity ids', () => { + expect(logs).toContainEqual(expect.objectContaining({ level: 40, crateId: ROOT, entityId: UNWRITTEN, msg: 'Skipped: no database row' })); + }); +}); From e6d0ffe37826eaa69bc2470651569b171513ab98 Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 14:58:33 +1100 Subject: [PATCH 08/20] refactor: rename pnpm seed to pnpm crates:import (#55) --- README.md | 42 +++++++++++++-------------- package.json | 4 +-- scripts/{seed.ts => crates-import.ts} | 2 +- src/admin/actions.test.ts | 2 +- src/admin/crates.test.ts | 2 +- src/root-collection.test.ts | 2 +- src/search-total.test.ts | 2 +- src/search.test.ts | 2 +- test/bootstrap.test.ts | 4 +-- 9 files changed, 31 insertions(+), 31 deletions(-) rename scripts/{seed.ts => crates-import.ts} (90%) diff --git a/README.md b/README.md index ba69012..5c4bf1b 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ You need Node.js 26 (26.13 or later), [pnpm](https://pnpm.io/) and Docker. ```sh pnpm install docker compose up -d --wait # Postgres, OpenSearch, Oni and a fake upstream provider -pnpm bootstrap # migrate the database, seed OCFL from test-data, index +pnpm bootstrap # migrate the database, import test-data into OCFL, index pnpm dev ``` @@ -54,25 +54,25 @@ override a default, set it in `.env`. ## Scripts -| Script | Does | -| -------------------- | -------------------------------------------------------------------------------- | -| `pnpm dev` | Run the API, restarting on changes, and rebuild the admin pages' stylesheet | -| `pnpm build` | Build the admin pages' stylesheet (run before deploying) | -| `pnpm test` | Run the tests (needs `docker compose up`) | -| `pnpm lint` | Biome, Knip and TypeScript | -| `pnpm bootstrap` | `db:deploy`, `seed` and `index` in one go | -| `pnpm ocfl:init` | Create an empty OCFL repository at `OCFL_ROOT`, unless one exists already | -| `pnpm seed [dir]` | Import each crate folder in `dir` (default `test-data`) into the OCFL repository | -| `pnpm index` | Rebuild both indexes from the OCFL repository | -| `pnpm generate-keys` | Print new `OIDC_SIGNING_KEYS` and `OIDC_COOKIE_KEYS` for `.env` | -| `pnpm db:migrate` | Create and apply a migration after a schema change | -| `pnpm db:deploy` | Apply pending migrations | -| `pnpm db:reset` | Drop and recreate the dev database | -| `pnpm db:console` | Open `psql` on the dev database | +| Script | Does | +| -------------------------- | -------------------------------------------------------------------------------- | +| `pnpm dev` | Run the API, restarting on changes, and rebuild the admin pages' stylesheet | +| `pnpm build` | Build the admin pages' stylesheet (run before deploying) | +| `pnpm test` | Run the tests (needs `docker compose up`) | +| `pnpm lint` | Biome, Knip and TypeScript | +| `pnpm bootstrap` | `db:deploy`, `crates:import` and `index` in one go | +| `pnpm ocfl:init` | Create an empty OCFL repository at `OCFL_ROOT`, unless one exists already | +| `pnpm crates:import [dir]` | Import each crate folder in `dir` (default `test-data`) into the OCFL repository | +| `pnpm index` | Rebuild both indexes from the OCFL repository | +| `pnpm generate-keys` | Print new `OIDC_SIGNING_KEYS` and `OIDC_COOKIE_KEYS` for `.env` | +| `pnpm db:migrate` | Create and apply a migration after a schema change | +| `pnpm db:deploy` | Apply pending migrations | +| `pnpm db:reset` | Drop and recreate the dev database | +| `pnpm db:console` | Open `psql` on the dev database | ## Test data -Each folder in `test-data/` is one RO-Crate. `pnpm seed` imports them into the +Each folder in `test-data/` is one RO-Crate. `pnpm crates:import` imports them into the OCFL repository at `storage/ocfl` (gitignored), and re-running it only adds a new OCFL version for crates that changed. Run `pnpm index` afterwards. @@ -83,7 +83,7 @@ dataset) unless `DEFAULT_METADATA_LICENSE` or `DEFAULT_LICENSE` is set. ## Configuration Configuration comes only from environment variables, validated at startup -(`src/configuration.ts`). `pnpm dev`, `pnpm seed`, `pnpm index` and Prisma +(`src/configuration.ts`). `pnpm dev`, `pnpm crates:import`, `pnpm index` and Prisma also read `.env` if it exists, without overriding variables that are already set. @@ -287,7 +287,7 @@ docker run --rm --env-file ldacapi.env -v /srv/ldacapi:/storage ldacapi pnpm db: ``` `pnpm ocfl:init` (which creates an empty OCFL repository; the API won't start -without one), `pnpm seed [dir]` (mount `dir` into the container) and +without one), `pnpm crates:import [dir]` (mount `dir` into the container) and `pnpm index` run the same way, offline. Give them and `pnpm db:deploy` the API's environment file: Prisma and the scripts load the whole [configuration](#configuration). `pnpm generate-keys` reads no configuration, so it needs no environment file: @@ -302,7 +302,7 @@ The image has no default `OCFL_ROOT`. Mount your storage and set `OCFL_ROOT` (and `OCFL_SCRATCH`, if you need it elsewhere) to paths inside the container (above, `OCFL_ROOT=/storage/ocfl`). The API runs as the image's `node` user, uid 1000, so the OCFL repository must be readable by uid -1000 and the scratch directory writable by it. `pnpm seed` writes to the +1000 and the scratch directory writable by it. `pnpm crates:import` writes to the repository, so give it write access too when you run it. ## Production with Docker Compose @@ -389,7 +389,7 @@ Import crates and index them with the API's image, mounting the directory of crates read-only: ```sh -docker compose run --rm -v /path/to/crates:/import:ro api pnpm seed /import +docker compose run --rm -v /path/to/crates:/import:ro api pnpm crates:import /import docker compose run --rm api pnpm index ``` diff --git a/package.json b/package.json index 83a550d..9e3f74a 100644 --- a/package.json +++ b/package.json @@ -31,9 +31,9 @@ "dev:api": "node --env-file-if-exists=.env --watch src/index.ts", "dev:css": "tailwindcss -i src/admin/admin.css -o src/admin/public/admin.css --watch=always", "ocfl:init": "node --env-file-if-exists=.env scripts/ocfl-init.ts", - "seed": "node --env-file-if-exists=.env scripts/seed.ts", + "crates:import": "node --env-file-if-exists=.env scripts/crates-import.ts", "index": "node --env-file-if-exists=.env scripts/index.ts", - "bootstrap": "pnpm db:deploy && pnpm seed && pnpm index", + "bootstrap": "pnpm db:deploy && pnpm crates:import && pnpm index", "db:migrate": "prisma migrate dev", "db:deploy": "prisma migrate deploy", "db:reset": "prisma migrate reset", diff --git a/scripts/seed.ts b/scripts/crates-import.ts similarity index 90% rename from scripts/seed.ts rename to scripts/crates-import.ts index 7455d48..1551e0b 100644 --- a/scripts/seed.ts +++ b/scripts/crates-import.ts @@ -16,4 +16,4 @@ for (const directory of directories.filter((entry) => entry.isDirectory())) { await repository.object(crate.rootId).import(path); logger.info({ crateId: crate.rootId, path }, 'Imported crate'); } -logger.info({ root: config.ocfl.root }, 'Seeded the OCFL repository'); +logger.info({ root: config.ocfl.root }, 'Imported crates into the OCFL repository'); diff --git a/src/admin/actions.test.ts b/src/admin/actions.test.ts index 0b5ac78..b9e9246 100644 --- a/src/admin/actions.test.ts +++ b/src/admin/actions.test.ts @@ -45,7 +45,7 @@ const flash = (body: string) => beforeAll(async () => { storage = await mkdtemp(join(tmpdir(), 'ldacapi-actions-')); - execFileSync('node', ['scripts/seed.ts'], { env: { ...process.env, OCFL_ROOT: join(storage, 'ocfl') }, stdio: 'ignore' }); + execFileSync('node', ['scripts/crates-import.ts'], { env: { ...process.env, OCFL_ROOT: join(storage, 'ocfl') }, stdio: 'ignore' }); }); afterAll(async () => { diff --git a/src/admin/crates.test.ts b/src/admin/crates.test.ts index 68464d4..c75cd3d 100644 --- a/src/admin/crates.test.ts +++ b/src/admin/crates.test.ts @@ -83,7 +83,7 @@ beforeAll(async () => { await mkdir(join(source, directory)); await writeFile(join(source, directory, 'ro-crate-metadata.json'), JSON.stringify(crate(id, name))); } - execFileSync('node', ['scripts/seed.ts', source], { env: { ...process.env, OCFL_ROOT: join(storage, 'ocfl') }, stdio: 'ignore' }); + execFileSync('node', ['scripts/crates-import.ts', source], { env: { ...process.env, OCFL_ROOT: join(storage, 'ocfl') }, stdio: 'ignore' }); }); afterAll(async () => { diff --git a/src/root-collection.test.ts b/src/root-collection.test.ts index 72e4e63..4b5249b 100644 --- a/src/root-collection.test.ts +++ b/src/root-collection.test.ts @@ -29,7 +29,7 @@ const collectionFacet = async () => { beforeAll(async () => { storage = await mkdtemp(join(tmpdir(), 'ldacapi-root-collection-')); const ocflRoot = join(storage, 'ocfl'); - execFileSync('node', ['scripts/seed.ts'], { env: { ...process.env, OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, stdio: 'ignore' }); + execFileSync('node', ['scripts/crates-import.ts'], { env: { ...process.env, OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, stdio: 'ignore' }); started = await buildTestApp({ OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, { repository: ocflRepository }); }); diff --git a/src/search-total.test.ts b/src/search-total.test.ts index 8fa851c..a38fcb2 100644 --- a/src/search-total.test.ts +++ b/src/search-total.test.ts @@ -58,7 +58,7 @@ beforeAll(async () => { await mkdir(join(source, 'unwritable'), { recursive: true }); await writeFile(join(source, 'unwritable', 'ro-crate-metadata.json'), JSON.stringify(crate)); const ocflRoot = join(storage, 'ocfl'); - execFileSync('node', ['scripts/seed.ts', source], { env: { ...process.env, OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, stdio: 'ignore' }); + execFileSync('node', ['scripts/crates-import.ts', source], { env: { ...process.env, OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, stdio: 'ignore' }); started = await buildTestApp({ OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, { repository: ocflRepository }); await started.repository.deleteIndex(); await started.repository.createIndex(ROOT); diff --git a/src/search.test.ts b/src/search.test.ts index 3ee8189..a290229 100644 --- a/src/search.test.ts +++ b/src/search.test.ts @@ -23,7 +23,7 @@ const search = async (body: Record) => { beforeAll(async () => { storage = await mkdtemp(join(tmpdir(), 'ldacapi-search-')); const ocflRoot = join(storage, 'ocfl'); - execFileSync('node', ['scripts/seed.ts'], { env: { ...process.env, OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, stdio: 'ignore' }); + execFileSync('node', ['scripts/crates-import.ts'], { env: { ...process.env, OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, stdio: 'ignore' }); started = await buildTestApp({ OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, { repository: ocflRepository }); await started.repository.deleteIndex(); await started.repository.createIndex(BUNDLED); diff --git a/test/bootstrap.test.ts b/test/bootstrap.test.ts index 19026e1..be67ed8 100644 --- a/test/bootstrap.test.ts +++ b/test/bootstrap.test.ts @@ -15,13 +15,13 @@ const crateIds = [ 'arcp://name,distributed_metadata_restricted', ]; -describe('seed and index', () => { +describe('import and index', () => { let storage: string; beforeAll(async () => { storage = await mkdtemp(join(tmpdir(), 'ldacapi-')); const env = { ...process.env, OCFL_ROOT: join(storage, 'ocfl'), LOG_LEVEL: 'silent' }; - execFileSync('node', ['scripts/seed.ts'], { env, stdio: 'ignore' }); + execFileSync('node', ['scripts/crates-import.ts'], { env, stdio: 'ignore' }); execFileSync('node', ['scripts/index.ts'], { env, stdio: 'ignore' }); }); From 095acdb8599d0442325f6ea521194eb8d3e75cf6 Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 14:59:36 +1100 Subject: [PATCH 09/20] feat: facet search on recordType, a readable label for the entity's class (#56) --- deploy/oni/configuration.json | 2 +- docker/oni/configuration.json | 2 +- src/indexer/search.test.ts | 28 ++++++++++++++++++++++++++-- src/indexer/search.ts | 18 ++++++++++++++++-- src/search-total.test.ts | 2 +- src/search.test.ts | 27 ++++++++++++++++++--------- 6 files changed, 63 insertions(+), 16 deletions(-) diff --git a/deploy/oni/configuration.json b/deploy/oni/configuration.json index be73144..12c8146 100644 --- a/deploy/oni/configuration.json +++ b/deploy/oni/configuration.json @@ -325,7 +325,7 @@ }, { "display": "Record Type", - "name": "entityType" + "name": "recordType" }, { "display": "Language", diff --git a/docker/oni/configuration.json b/docker/oni/configuration.json index 4ec48e1..62644c1 100644 --- a/docker/oni/configuration.json +++ b/docker/oni/configuration.json @@ -325,7 +325,7 @@ }, { "display": "Record Type", - "name": "entityType" + "name": "recordType" }, { "display": "Language", diff --git a/src/indexer/search.test.ts b/src/indexer/search.test.ts index 19d3397..80510f4 100644 --- a/src/indexer/search.test.ts +++ b/src/indexer/search.test.ts @@ -6,9 +6,9 @@ import { searchDocument } from './search.ts'; const ROOT = 'arcp://name,test'; const LICENSE = 'https://creativecommons.org/licenses/by/4.0/'; -const crateWith = async (entity: Record) => { +const crateWith = async (entity: Record, terms: Record = {}) => { const crate = await ROCrate.create({ - '@context': ['https://w3id.org/ro/crate/1.1/context', { '@vocab': 'http://schema.org/', ldac: 'https://w3id.org/ldac/terms#' }], + '@context': ['https://w3id.org/ro/crate/1.1/context', { '@vocab': 'http://schema.org/', ldac: 'https://w3id.org/ldac/terms#', ...terms }], '@graph': [ { '@id': 'ro-crate-metadata.json', '@type': 'CreativeWork', about: { '@id': ROOT }, license: { '@id': LICENSE } }, { '@id': ROOT, '@type': ['Dataset', 'RepositoryCollection'], name: 'Root', hasPart: [{ '@id': 'data.txt' }] }, @@ -38,6 +38,7 @@ describe('searchDocument', () => { id: `${ROOT}/data.txt`, name: 'Data', entityType: 'http://schema.org/MediaObject', + recordType: 'File', inLanguage: ['English'], communicationMode: ['https://w3id.org/ldac/terms#SpokenLanguage'], encodingFormat: ['text/plain'], @@ -49,6 +50,29 @@ describe('searchDocument', () => { }); }); + it.each([ + [['Dataset', 'RepositoryCollection'], 'https://w3id.org/ldac/profile#Collection', 'Collection'], + [['Dataset', 'RepositoryObject'], 'https://w3id.org/ldac/profile#Object', 'Object'], + ['Person', 'https://w3id.org/ldac/profile#Person', 'Person'], + ['Organization', 'https://w3id.org/ldac/profile#Organization', 'Organisation'], + ['SoftwareApplication', 'https://w3id.org/ldac/profile#SoftwareApplication', 'Software Application'], + ])('labels a %s record as %s', async (type, profile, label) => { + const { crate, entity } = await crateWith({ '@type': type, conformsTo: { '@id': profile } }); + + expect((await searchDocument(crate, crateObject, entity, LICENSE, ROOT)).recordType).toBe(label); + }); + + it.each([ + ['http://schema.org/Thing', 'Thing'], + ['https://example.com/terms#Widget', 'Widget'], + ])('labels an unknown class %s by its last segment', async (uri, label) => { + const { crate, entity } = await crateWith({}, { File: uri }); + + const doc = await searchDocument(crate, crateObject, entity, LICENSE, ROOT); + + expect(doc).toMatchObject({ entityType: uri, recordType: label }); + }); + it('has no type field', async () => { const { crate, entity } = await crateWith({}); diff --git a/src/indexer/search.ts b/src/indexer/search.ts index 64000db..e1abc0f 100644 --- a/src/indexer/search.ts +++ b/src/indexer/search.ts @@ -15,7 +15,8 @@ const fields = { name: { type: 'text', label: 'Name', weight: 3, advanced: true, sort: true }, description: { type: 'text', label: 'Description', advanced: true }, text: { type: 'text', label: 'Text', advanced: true }, - entityType: { type: 'keyword', label: 'Record Type', facet: true }, + entityType: { type: 'keyword', label: 'Entity Type', filter: true }, + recordType: { type: 'keyword', label: 'Record Type', facet: true }, inLanguage: { type: 'keyword', label: 'Language', advanced: true, facet: true }, communicationMode: { type: 'keyword', label: 'Communication Mode', facet: true }, linguisticGenre: { type: 'keyword', label: 'Linguistic Genre', facet: true }, @@ -42,6 +43,18 @@ type SearchOptions = BaseOptions & { search: SearchBackend; }; +const CLASS_LABELS: Record = { + 'http://pcdm.org/models#Collection': 'Collection', + 'http://pcdm.org/models#Object': 'Object', + 'http://schema.org/MediaObject': 'File', + 'http://schema.org/Person': 'Person', + 'http://schema.org/Organization': 'Organisation', + 'http://schema.org/SoftwareApplication': 'Software Application', +}; + +/** A readable label for a class URI, falling back to its last segment */ +const classLabel = (uri: string) => CLASS_LABELS[uri] ?? uri.split(/[#/]/).at(-1) ?? uri; + const values = (entity: Entity, name: string): unknown[] => [...(entity[name] ?? []), ...(entity[`ldac:${name}`] ?? [])]; /** Linked entities by name, falling back to their id; literals as they are */ @@ -112,6 +125,7 @@ export async function searchDocument( ): Promise { const id: string = entity['@id']; const type = recordType(entity); + const entityType = type && entityClass(crate, type); const [datePublished] = dates(entity, 'datePublished'); const memberOf = pickSingleMemberOf(entity); const point = location(entity); @@ -120,7 +134,7 @@ export async function searchDocument( id, name: entity.name?.join('; ') || id, description: labels(entity, 'description'), - ...(type && { entityType: entityClass(crate, type) }), + ...(entityType && { entityType, recordType: classLabel(entityType) }), inLanguage: labels(entity, 'inLanguage'), communicationMode: labels(entity, 'communicationMode'), linguisticGenre: labels(entity, 'linguisticGenre'), diff --git a/src/search-total.test.ts b/src/search-total.test.ts index a38fcb2..c0d825e 100644 --- a/src/search-total.test.ts +++ b/src/search-total.test.ts @@ -77,7 +77,7 @@ describe('an entity whose row fails to be written', () => { expect(entities.map(({ id }: { id: string }) => id).sort()).toEqual([ROOT, WRITTEN]); expect(total).toBe(2); - expect(facets.entityType.reduce((sum: number, { count }: { count: number }) => sum + count, 0)).toBe(2); + expect(facets.recordType.reduce((sum: number, { count }: { count: number }) => sum + count, 0)).toBe(2); }); it('is not counted when the search index is rebuilt alone', async () => { diff --git a/src/search.test.ts b/src/search.test.ts index a290229..e575778 100644 --- a/src/search.test.ts +++ b/src/search.test.ts @@ -6,7 +6,6 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { ocflRepository } from '../test/admin.ts'; import { buildTestApp } from '../test/app.ts'; -const COLLECTION = 'http://pcdm.org/models#Collection'; const OBJECT = 'http://pcdm.org/models#Object'; const MEDIA = 'http://schema.org/MediaObject'; const BUNDLED = 'arcp://name,bundled'; @@ -36,21 +35,31 @@ afterAll(async () => { await rm(storage, { recursive: true, force: true }); }); -describe('search by entityType', () => { - it('declares an entityType facet and no type facet', async () => { +describe('search by recordType and entityType', () => { + it('declares a recordType facet, and entityType as a filter only', async () => { const response = await started.app.inject({ method: 'GET', url: `${started.config.prefix}/capabilities` }); - const { facets } = response.json().search; - expect(facets).toHaveProperty('entityType'); + const { facets, filters } = response.json().search; + expect(facets).toHaveProperty('recordType'); + expect(facets).not.toHaveProperty('entityType'); expect(facets).not.toHaveProperty('type'); + expect(filters).toHaveProperty('entityType'); }); - it("facets on each entity's class URI", async () => { + it("facets on a readable label for each entity's class", async () => { const { facets } = await search({ query: '' }); - const names = facets.entityType.map(({ name }: { name: string }) => name); - expect(names).toEqual(expect.arrayContaining([COLLECTION, OBJECT, MEDIA])); - expect(names.every((name: string) => /^https?:\/\//.test(name))).toBe(true); + expect(facets).not.toHaveProperty('entityType'); + const names = facets.recordType.map(({ name }: { name: string }) => name); + expect(names).toEqual(expect.arrayContaining(['Collection', 'Object', 'File'])); + expect(names.some((name: string) => name.includes('://'))).toBe(false); + }); + + it('filters on recordType', async () => { + const { total, entities } = await search({ query: '', filters: { recordType: ['File'] }, limit: 100 }); + + expect(total).toBeGreaterThan(0); + expect(entities.every(({ entityType }: { entityType: string }) => entityType === MEDIA)).toBe(true); }); it('filters on entityType, agreeing with listing entities by entityType', async () => { From 3b0609e68b082cd6f63a4dc69c015e15883ff914 Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 15:04:07 +1100 Subject: [PATCH 10/20] docs: rewrite the README, glossary and ADRs in arocapi's terms (#62) --- CLAUDE.md | 4 + GLOSSARY.md | 24 ++++ README.md | 127 +++++++++++++----- ...0004-entity-metadata-is-a-virtual-crate.md | 17 +++ 4 files changed, 142 insertions(+), 30 deletions(-) create mode 100644 docs/adr/0004-entity-metadata-is-a-virtual-crate.md diff --git a/CLAUDE.md b/CLAUDE.md index c280da8..c12c319 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,3 +17,7 @@ Single-context: `GLOSSARY.md` and `docs/adr/` at the repo root. See `docs/agents ## Database migrations A migration that creates or changes arocapi's models (`prisma/arocapi`, symlinked from the arocapi package) is named after the arocapi version: after an arocapi upgrade that changes the models, run `pnpm db:migrate --name arocapi-`. Name migrations for our own models (`prisma/models`) after what they change. + +## Upgrading arocapi + +After any arocapi upgrade, recheck the README's links to arocapi's docs (arocapi.crate-works.org is unversioned): each page and anchor should still exist and still say what the README relies on. diff --git a/GLOSSARY.md b/GLOSSARY.md index fbdd17c..16fa95a 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -4,6 +4,30 @@ Serves RO-Crates from OCFL storage through the arocapi API, and controls access ## Language +arocapi's terms, such as **Record**, **Metadata**, **Content** and **Row**, mean what [arocapi's glossary](https://github.com/crate-works/arocapi/blob/main/GLOSSARY.md) says. + +### Crates and indexes + +**Crate**: +One RO-Crate, stored as one OCFL object in the repository. The unit of import, indexing and admin actions. +_Avoid_: OCFL object (the storage), collection (what a crate may describe) + +**Import**: +Bringing RO-Crates into the OCFL repository: `pnpm crates:import` today, the deposit API in future. +_Avoid_: Seed + +**Index**: +Building the database index and the search index from the repository, for one crate or all of them. Both can be deleted and rebuilt at any time. +_Avoid_: Import + +**Database index**: +The Entity and File rows that arocapi reads, written by the indexer (arocapi's Rows). +_Avoid_: Structural index + +**Search index**: +The search documents in arocapi's search backend, one per entity that has a row. +_Avoid_: OpenSearch index (one backend's storage) + ### Authentication **Upstream provider**: diff --git a/README.md b/README.md index 5c4bf1b..652c315 100644 --- a/README.md +++ b/README.md @@ -7,26 +7,59 @@ RO-Crate API. It serves RO-Crates over the ## How it fits together -The core API (entities, files, search, RO-Crate metadata) comes from -[arocapi](https://github.com/crate-works/arocapi), which this -app mounts as a Fastify plugin. ldacapi adds what LDaCA needs on top: - -- **OCFL storage**: crates live in an [OCFL](https://ocfl.io/) repository on - disk (`src/ocfl.ts`). -- **Indexing**: crates are read from OCFL into the database (the database - index, used by arocapi's entity and file routes) and, as one search - document per entity, into arocapi's OpenSearch search backend - (`src/indexer/`). The search fields are declared in `src/indexer/search.ts`. +The API's routes (entities, files, search, RO-Crate metadata) come from +[arocapi](https://github.com/crate-works/arocapi), which ldacapi mounts as a +Fastify plugin under `/api` (`src/app.ts`). Its +[docs](https://arocapi.crate-works.org/) explain each part; ldacapi +configures them as follows: + +- **Storage backend, Content half**: arocapi's + [`OcflStorage`](https://arocapi.crate-works.org/configuration/storage/#ocflstorage) + serves crate files from the OCFL repository. +- **Storage backend, Metadata half**: arocapi's + [`EntityMetaMetadata`](https://arocapi.crate-works.org/configuration/storage/#entitymetametadata) + serves each entity's virtual crate, built at index time and held in its + database row ([ADR 0004](docs/adr/0004-entity-metadata-is-a-virtual-crate.md)). +- **Search backend**: arocapi's + [`OpenSearchBackend`](https://arocapi.crate-works.org/configuration/search/#opensearchbackend), + with the [fields](https://arocapi.crate-works.org/configuration/search/#fields) + declared in `src/indexer/search.ts`, where `entityType` filters on an + entity's class URI and the "Record Type" facet, `recordType`, shows a + readable label for it. +- **Visibility**: every entity is visible to everyone, through arocapi's + all-visible [visibility filters](https://arocapi.crate-works.org/configuration/visibility/) + ([ADR 0003](docs/adr/0003-every-entity-is-visible.md)). +- **Access**: arocapi's + [`LicenceAccess`](https://arocapi.crate-works.org/configuration/access/#access), + with the licences a user holds looked up in REMS + ([licence checks against an external authority](https://arocapi.crate-works.org/strategies/access/#licence-checks-against-an-external-authority)); + see [Licences](#licences) (`src/auth.ts`). + +ldacapi adds what LDaCA needs on top (terms as in [GLOSSARY.md](GLOSSARY.md)): + +- **OCFL repository**: each crate is one [OCFL](https://ocfl.io/) object on + disk, imported with `pnpm crates:import` (`src/ocfl.ts`). +- **Indexing**: `pnpm index` and the admin pages build the database index (the + Entity and File rows arocapi reads) and the search index (one search + document per entity) from the repository (`src/indexer/`). Both can be + rebuilt at any time. +- **Collections**: an entity's `rootCollection` is the top-level collection + it sits under, and is empty on a top-level collection + ([membership](https://arocapi.crate-works.org/concepts/entities/#membership)). + Indexing finds it by following `memberOf` up through the parent crates in + the repository, so a crate in a distributed collection gets the + distributed collection's root whatever order the crates are indexed in. If + a parent collection moves, reindex the crates under it: they keep the old + `rootCollection` until then. +- **Sign-in**: the API is its own OIDC provider, signing users in at an + upstream provider such as CILogon (`src/oidc/`, + [ADR 0001](docs/adr/0001-own-oidc-provider.md)). - **Admin pages**: server-rendered pages at `/admin/` on their own origin, where admins, who sign in through our own OIDC provider, see and rebuild or - delete indexes (`src/admin/`). -- **Sign-in**: the API is its own OIDC provider, signing users in at an - upstream provider such as CILogon (`src/oidc/`). -- **Access control**: every entity is visible, and what a request may read is - decided by licence, checked against REMS (`src/auth.ts`). -- **Files and metadata**: arocapi serves crate files from the OCFL repository - through its OCFL storage backend, and each entity's RO-Crate metadata from - the database index (`src/app.ts`). + delete indexes (`src/admin/`, + [ADR 0002](docs/adr/0002-server-rendered-admin-pages.md)). + +The ADRs linked here are ldacapi's own, in `docs/adr/`, not arocapi's. ## Getting started @@ -114,10 +147,28 @@ have no defaults and must be set. ### Licences -Content under a licence that isn't open needs an entitlement to it in REMS, -looked up at `REMS_ENDPOINT` with `REMS_USER` and `REMS_KEY`. A user without -one is offered an enrolment link: `ENROLLMENT_URL` with every `{licenseId}` -replaced by the URL-encoded licence id, e.g. +Every entity has a metadata licence, which guards its RO-Crate metadata +(`/entity/:id/metadata`), and a content licence, which guards a file's bytes +(`/file/:id`) +([two licences per entity](https://arocapi.crate-works.org/concepts/licences/#two-licences-per-entity)). +The indexer takes the metadata licence from the `license` of +`ro-crate-metadata.json`, and the content licence from the entity's own +`license` or else the root dataset's. + +A licence that isn't one of the open licences in `src/configuration.ts` needs +an entitlement to it in REMS, looked up at `REMS_ENDPOINT` with `REMS_USER` +and `REMS_KEY`, once per request. If REMS can't be reached or returns an +error, the API logs it and treats the user as holding no licences, so +restricted entities are refused until it recovers. + +A refused request gets 401 if it is anonymous, so the client can prompt a +sign-in, and 403 if it is signed in +([Denied is 401 or 403](https://arocapi.crate-works.org/concepts/visibility-vs-access/#denied-is-401-or-403-never-404)). +The enrolment link isn't on the error: every entity the API returns carries +an `access` object, and for each licence the user doesn't hold it has a +`metadataAuthorizationUrl` or `contentAuthorizationUrl`. That link is +`ENROLLMENT_URL` with every `{licenseId}` replaced by the URL-encoded licence +id, e.g. ```dotenv ENROLLMENT_URL=https://rems.example.org/apply-for?resource={licenseId} @@ -199,9 +250,11 @@ username to `ADMIN_USERS` in `.env`. Anyone else sees "Not an admin". The admin pages are at `/admin/`. `ADMIN_ORIGIN` is an origin, with no path, such as `https://admin.example.org`, and must differ from -`OIDC_ISSUER`: crate files are served on the API's origin with their own -content type, so HTML or SVG in a crate could run script there, and a separate -origin keeps that away from admin. It defaults to `http://admin.localhost:8080` +`OIDC_ISSUER`. Crate files are served on the API's origin with their own +content type. arocapi sandboxes them, except audio and video, but that rests +on each response's headers, so a separate origin keeps any script a crate file +does run away from admin (ldacapi's +[ADR 0002](docs/adr/0002-server-rendered-admin-pages.md)). It defaults to `http://admin.localhost:8080` outside production, which browsers resolve to this machine; in production it must be https. The admin pages are on only when sign-in is and `ADMIN_ORIGIN` is set; otherwise startup logs a warning and `/admin/` is not found. On the @@ -319,11 +372,15 @@ from Let's Encrypt on its own. It serves two hostnames: | `DOMAIN` | everything else | Oni | | `ADMIN_DOMAIN` | `/admin/*` | The API's admin pages | -The admin pages need their own hostname, so that script in a crate file can -never run on their origin (ADR 0002). arocapi serves crate files, other than -audio and video, with a `sandbox` Content-Security-Policy, which gives any HTML -or SVG in a crate file an opaque origin, so its scripts can't reach Oni's -sign-in on `DOMAIN`. +The admin pages need their own hostname (see [Admin pages](#admin-pages)). +Caddy sets no Content-Security-Policy on the API's responses: arocapi sends +crate files with `nosniff` and, except audio and video, a `sandbox` +Content-Security-Policy, which gives any HTML or SVG in a crate an opaque +origin, so its scripts can't reach Oni's sign-in on `DOMAIN` +([protective headers](https://arocapi.crate-works.org/configuration/storage/#protective-headers)). +Audio and video are left out so they play when opened directly. On both +hostnames Caddy adds HSTS, `nosniff` and a `strict-origin-when-cross-origin` +referrer policy, and Oni's pages also get `X-Frame-Options: SAMEORIGIN`. No other container publishes a port, on the host or on localhost, so Postgres, OpenSearch, the API and Oni are reachable only from within the stack. @@ -404,6 +461,16 @@ docker compose run --rm api pnpm db:deploy docker compose up -d ``` +If an upgrade changes the search fields, the API and `pnpm index` stop with an +error naming the changed field. Delete the search index (named by +`ENTITY_INDEX`, `entities` by default), rebuild both indexes, and start again: + +```sh +docker compose exec opensearch curl -sX DELETE localhost:9200/entities +docker compose run --rm api pnpm index +docker compose up -d +``` + To run a branch's image (see [Docker image](#docker-image)) on a dev deployment, change the API's `image` tag in `compose.yaml` from `main` to the branch's tag and follow the upgrade steps above. Migrations the branch adds diff --git a/docs/adr/0004-entity-metadata-is-a-virtual-crate.md b/docs/adr/0004-entity-metadata-is-a-virtual-crate.md new file mode 100644 index 0000000..35e6a26 --- /dev/null +++ b/docs/adr/0004-entity-metadata-is-a-virtual-crate.md @@ -0,0 +1,17 @@ +# Each entity's metadata is a virtual crate built at index time + +## Context + +arocapi serves an entity's Metadata, its RO-Crate metadata document, at `/entity/:id/metadata`, through the Metadata half of the storage backend, after checking the entity's metadata licence. A crate's `ro-crate-metadata.json` describes many entities, each with its own licences, and a large collection's runs to many megabytes. Oni asks for the metadata of the entity it is showing, not of the whole crate. + +## Decision + +Each entity's Metadata is a virtual crate: a new RO-Crate with the entity as its root, plus every entity it links to, directly or through others. The root also gets the `Dataset` type, the crate root's `conformsTo` if it has none, and the crate's content licence as its `license` if it has none. The database indexer builds it when it indexes the crate and stores it in the entity's row as `meta.rocrate`, and arocapi's `EntityMetaMetadata` serves it from there. Nothing serves a crate's `ro-crate-metadata.json` as it is. + +## Consequences + +- A request for one entity's Metadata gets only that entity and what it links to, guarded by that entity's metadata licence, rather than every entity in its crate. +- Metadata is only as fresh as the last index run: importing a new version of a crate changes nothing that is served until the crate is reindexed. +- Serving Metadata reads only the database, never the OCFL repository. +- The virtual crate isn't a file in the crate, so it has no checksum or version of its own, and it differs from what the depositor wrote by the properties added to its root. +- Rows are larger, as each holds a copy of what its entity links to. From db74df42ba1a73a3dca63de62da419f20cada6f6 Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 15:13:41 +1100 Subject: [PATCH 11/20] refactor: label record types from the Oni class constants (#56) --- src/indexer/search.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/indexer/search.ts b/src/indexer/search.ts index e1abc0f..b19ae87 100644 --- a/src/indexer/search.ts +++ b/src/indexer/search.ts @@ -4,6 +4,7 @@ import { OpenSearchBackend } from 'arocapi/search/opensearch'; import type { Entity, ROCrate } from 'ro-crate'; import { config } from '../configuration.ts'; import { logger } from '../logger.ts'; +import { COLLECTION, OBJECT } from '../oni.ts'; import { firstStringOrId } from '../utils.ts'; import { countRows, pickSingleMemberOf, rowIds } from './database.ts'; import type { BaseOptions, CrateObject } from './indexer.ts'; @@ -44,8 +45,8 @@ type SearchOptions = BaseOptions & { }; const CLASS_LABELS: Record = { - 'http://pcdm.org/models#Collection': 'Collection', - 'http://pcdm.org/models#Object': 'Object', + [COLLECTION]: 'Collection', + [OBJECT]: 'Object', 'http://schema.org/MediaObject': 'File', 'http://schema.org/Person': 'Person', 'http://schema.org/Organization': 'Organisation', From 19d387b2cb3801fed5f4cd9efa98aa4fd8b57f2b Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 15:13:48 +1100 Subject: [PATCH 12/20] refactor: search reads each crate's rows itself, and counts its documents The search indexer finds a crate's rows, and each document's rootCollection, with one query by meta.crateId, so the database indexer no longer hands it the ids it wrote. SearchIndexer.count counts search documents through arocapi's search backend, so the crates page shows OpenSearch's state again. Indexing runs database then search, and deleting search then database, by fixed lists; deleting a crate's database index deletes its search documents too. The top-collection walk reads parent crates as plain JSON, sharing the memberOf lookup with pickSingleMemberOf. --- src/indexer/database.test.ts | 2 +- src/indexer/database.ts | 17 +++++------- src/indexer/indexer.ts | 41 +++++++++------------------- src/indexer/search.ts | 48 ++++++++++++++++++++++++-------- src/ocfl.ts | 53 +++++++++++++++++++++++------------- test/bootstrap.test.ts | 14 ++-------- 6 files changed, 93 insertions(+), 82 deletions(-) diff --git a/src/indexer/database.test.ts b/src/indexer/database.test.ts index a0c0c1d..bb3458e 100644 --- a/src/indexer/database.test.ts +++ b/src/indexer/database.test.ts @@ -59,7 +59,7 @@ describe('DatabaseIndexer', () => { return { size, crc32: '' }; }, }; - await indexer.index({ crateObject, crate }); + await indexer.index({ crateObject, crate, topCollection: ROOT }); }; describe('file size', () => { diff --git a/src/indexer/database.ts b/src/indexer/database.ts index 96bf14d..5b041c8 100644 --- a/src/indexer/database.ts +++ b/src/indexer/database.ts @@ -5,7 +5,7 @@ import type { Prisma } from '../generated/prisma/client.ts'; import { logger } from '../logger.ts'; import { prisma } from '../prisma.ts'; import { firstStringOrId } from '../utils.ts'; -import { type BaseOptions, type CrateFile, entityClass, Indexer, recordType, rootCollection } from './indexer.ts'; +import { type BaseOptions, type CrateFile, entityClass, Indexer, memberOf, recordType, rootCollection } from './indexer.ts'; const log = logger.child({ module: 'indexer/database' }); @@ -25,7 +25,6 @@ export class DatabaseIndexer extends Indexer { override async _index({ crateObject, crate, paths, license, metadataLicense, topCollection }: Parameters[0]) { const crateId = crate.rootId; - const written = new Set(); let count = 0; for (const entity of crate.entities()) { const entityType = recordType(entity); @@ -70,7 +69,6 @@ export class DatabaseIndexer extends Indexer { } try { await prisma.entity.create({ data: param.entity }); - written.add(entityId); if (param.file) { await prisma.file.create({ data: param.file }); } @@ -80,7 +78,6 @@ export class DatabaseIndexer extends Indexer { } log.info({ crateId, entities: count }, 'Indexed crate'); - return written; } async delete(crateId?: string) { @@ -89,18 +86,19 @@ export class DatabaseIndexer extends Indexer { log.debug({ crateId }, 'Index deleted'); } - async count(crateId?: string) { - return await countRows(crateId); + async count(crateId: string) { + return await prisma.entity.count({ where: rowsOf(crateId) }); } } /** The entity rows indexed from `crateId`, or every row */ const rowsOf = (crateId?: string): Prisma.EntityWhereInput => (crateId ? { meta: { path: ['crateId'], equals: crateId } } : {}); -export const countRows = (crateId?: string) => prisma.entity.count({ where: rowsOf(crateId) }); +/** The id and rootCollection of each entity row indexed from `crateId` */ +export const crateRows = (crateId: string) => prisma.entity.findMany({ where: rowsOf(crateId), select: { id: true, rootCollection: true } }); /** The ids of the entity rows indexed from `crateId` */ -export const rowIds = async (crateId: string) => (await prisma.entity.findMany({ where: rowsOf(crateId), select: { id: true } })).map(({ id }) => id); +export const rowIds = async (crateId: string) => (await crateRows(crateId)).map(({ id }) => id); export function entityAsCrate(crate: ROCrate, entity: Entity, license: string) { const newCrate = new ROCrate({ '@context': crate['@context'] }, { array: true, link: true }); @@ -125,8 +123,7 @@ function statedSize(contentSize: unknown) { export function pickSingleMemberOf(entity: Entity) { return ( - entity['pcdm:memberOf']?.[0]['@id'] || - entity.memberOf?.[0]['@id'] || + memberOf(entity) || entity['@reverse']['pcdm:hasMember']?.[0]?.['@id'] || entity['@reverse'].hasMember?.[0]?.['@id'] || entity.isPartOf?.find((e: Entity) => e['@type'].includes('RepositoryObject'))?.['@id'] || diff --git a/src/indexer/indexer.ts b/src/indexer/indexer.ts index 3eecf23..71dc0ce 100644 --- a/src/indexer/indexer.ts +++ b/src/indexer/indexer.ts @@ -24,8 +24,6 @@ function crateRelativePath(ref: string) { } } -export type EntityIds = ReadonlySet; - /** Entity id → its path in the crate */ export type LogicalPaths = Map; @@ -99,7 +97,7 @@ export class Indexer { async init() {} - async count(_crateId?: string): Promise { + async count(_crateId: string): Promise { throw new Error('Not Implemented'); } @@ -112,21 +110,8 @@ export class Indexer { throw new Error('Not Implemented'); } - /** - * `topCollection` is the top-level collection the crate sits under, the crate's own root if it has no parent. - * Returns the ids of the entity rows written, if this indexer writes them. - */ - async index({ - crateObject, - crate, - topCollection = crate.rootId, - rows, - }: { - crateObject: CrateObject; - crate: ROCrate; - topCollection?: string; - rows?: EntityIds | undefined; - }): Promise { + /** `topCollection` is the top-level collection the crate sits under, the crate's own root if it has no parent */ + async index({ crateObject, crate, topCollection }: { crateObject: CrateObject; crate: ROCrate; topCollection: string }) { const rootDataset = crate.root; const crateId = crate.rootId; const metadataLicense = firstStringOrId(crate.descriptor.license) || this.defaultMetadataLicense; @@ -143,11 +128,9 @@ export class Indexer { log.info({ crateId }, 'Indexing crate'); const paths = this.deriveUniqueEntityIds(crate); try { - return await this._index({ crateObject, crate, paths, license, metadataLicense, topCollection, rows }); + await this._index({ crateObject, crate, paths, license, metadataLicense, topCollection }); } catch (error) { log.error({ err: error, crateId, path: crateObject.root }, 'Error indexing crate'); - } finally { - log.info({ crateId }, 'Indexed crate'); } } } @@ -159,9 +142,7 @@ export class Indexer { license: string; metadataLicense: string; topCollection: string; - /** The ids of the crate's entity rows, once the database indexer has written them */ - rows?: EntityIds | undefined; - }): Promise { + }): Promise { throw new Error('Not Implemented'); } @@ -174,11 +155,15 @@ export class Indexer { /** An entity's rootCollection: empty on the top-level collection itself */ export const rootCollection = (topCollection: string, entityId: string) => (entityId === topCollection ? null : topCollection); +const firstId = (value: unknown): string | undefined => ([value ?? []].flat()[0] as { '@id'?: string } | undefined)?.['@id']; + +/** The first collection `entity` is a member of, by `pcdm:memberOf` or `memberOf`, in a parsed crate or raw JSON-LD */ +export const memberOf = (entity: Record) => firstId(entity['pcdm:memberOf']) || firstId(entity.memberOf); + /** The id of the collection a crate's root is a member of, if any */ -export function parentCollection(crate: ROCrate): string | undefined { - const root = crate.root; - const parent: string | undefined = (root['pcdm:memberOf'] ?? root.memberOf)?.[0]?.['@id']; - return parent && parent !== crate.rootId ? parent : undefined; +export function parentCollection(root: Record, rootId: string): string | undefined { + const parent = memberOf(root); + return parent && parent !== rootId ? parent : undefined; } const RecordType: Record = config.indexType; diff --git a/src/indexer/search.ts b/src/indexer/search.ts index b19ae87..fb6c2ae 100644 --- a/src/indexer/search.ts +++ b/src/indexer/search.ts @@ -6,9 +6,9 @@ import { config } from '../configuration.ts'; import { logger } from '../logger.ts'; import { COLLECTION, OBJECT } from '../oni.ts'; import { firstStringOrId } from '../utils.ts'; -import { countRows, pickSingleMemberOf, rowIds } from './database.ts'; +import { crateRows, pickSingleMemberOf, rowIds } from './database.ts'; import type { BaseOptions, CrateObject } from './indexer.ts'; -import { entityClass, Indexer, recordType, rootCollection } from './indexer.ts'; +import { entityClass, Indexer, recordType } from './indexer.ts'; const log = logger.child({ module: 'indexer/search' }); @@ -39,6 +39,31 @@ export const createSearchBackend = (client: Client, options: { refresh?: boolean type SearchBackend = ReturnType; +/** How many of `ids` have a search document, 0 if there is no index */ +export async function countDocuments(search: SearchBackend, ids: string[]) { + if (!ids.length) { + return 0; + } + try { + const { total } = await search.search({ + searchType: 'basic', + query: '', + filters: { terms: {}, ranges: {} }, + limit: 1, + offset: 0, + sort: 'relevance', + order: 'asc', + visibility: { id: { in: ids } }, + }); + return total; + } catch (error) { + if ((error as { statusCode?: number }).statusCode === 404) { + return 0; + } + throw error; + } +} + type SearchOptions = BaseOptions & { client: Client; search: SearchBackend; @@ -121,7 +146,7 @@ export async function searchDocument( crateObject: CrateObject, entity: Entity, license: string, - topCollection: string, + rootCollection: string | null, path?: string, ): Promise { const id: string = entity['@id']; @@ -130,7 +155,6 @@ export async function searchDocument( const [datePublished] = dates(entity, 'datePublished'); const memberOf = pickSingleMemberOf(entity); const point = location(entity); - const root = rootCollection(topCollection, id); const doc: Document = { id, name: entity.name?.join('; ') || id, @@ -141,7 +165,7 @@ export async function searchDocument( linguisticGenre: labels(entity, 'linguisticGenre'), annotationType: labels(entity, 'annotationType'), encodingFormat: labels(entity, 'encodingFormat'), - ...(root && { rootCollection: root }), + ...(rootCollection && { rootCollection }), ...(memberOf && { memberOf }), license: firstStringOrId(entity.license) || license, ...(datePublished && { datePublished }), @@ -184,7 +208,6 @@ export class SearchIndexer extends Indexer { await this.search.createIndex(); } - /** A crate's documents are found by its database rows, so delete them before the rows */ async delete(crateId?: string) { if (crateId) { await this.search.remove(await rowIds(crateId)); @@ -194,26 +217,27 @@ export class SearchIndexer extends Indexer { log.debug({ crateId }, 'Index deleted'); } - async count(crateId?: string) { - return await countRows(crateId); + async count(crateId: string) { + return await countDocuments(this.search, await rowIds(crateId)); } /** Only entities with a row, as arocapi drops a hit without one but still counts it */ - async _index({ crateObject, crate, paths, license, topCollection, rows }: Parameters[0]): Promise { + async _index({ crateObject, crate, paths, license }: Parameters[0]) { await this.search.createIndex(); const crateId = crate.rootId; - const withRows = rows ?? new Set(await rowIds(crateId)); + const rows = new Map((await crateRows(crateId)).map(({ id, rootCollection }) => [id, rootCollection])); const docs: Document[] = []; for (const entity of crate.entities()) { if (!recordType(entity)) { continue; } const entityId = entity['@id']; - if (!withRows.has(entityId)) { + const rootCollection = rows.get(entityId); + if (rootCollection === undefined) { log.warn({ crateId, entityId }, 'Skipped: no database row'); continue; } - const doc = await searchDocument(crate, crateObject, entity, license, topCollection, paths.get(entityId)); + const doc = await searchDocument(crate, crateObject, entity, license, rootCollection, paths.get(entityId)); if (doc.text) { // Indexed alone to keep each request small await this.indexDocuments([doc]); diff --git a/src/ocfl.ts b/src/ocfl.ts index 4bbac0f..2f14b83 100644 --- a/src/ocfl.ts +++ b/src/ocfl.ts @@ -7,7 +7,7 @@ import { OcflStorage } from 'arocapi/storage/ocfl'; import { type RawEntity, ROCrate } from 'ro-crate'; import { config } from './configuration.ts'; import { DatabaseIndexer } from './indexer/database.ts'; -import { type CrateObject, type EntityIds, type Indexer, parentCollection } from './indexer/indexer.ts'; +import { type CrateObject, type Indexer, parentCollection } from './indexer/indexer.ts'; import { createSearchBackend, SearchIndexer } from './indexer/search.ts'; import { logger } from './logger.ts'; import { type IndexState, State } from './repository.ts'; @@ -142,12 +142,19 @@ export async function getState(crateId: string, type?: string) { return type ? { [type]: stateCache[crateId]?.[type] } : { ...stateCache[crateId] }; } -/** The search index finds a crate's documents by its database rows, so is deleted from first */ +/** + * The search index finds a crate's documents, and their rootCollection, by its database rows. So the rows are indexed + * first and deleted last, deleting stops at the first failure, and deleting the rows deletes the documents too. + */ +const INDEX_ORDER = ['database', 'search']; const DELETE_ORDER = ['search', 'database']; -/** Stops at the first failure, so the rows that find a crate's search documents outlive them */ +const typesOf = (type?: string | string[]) => (type ? [type].flat() : INDEX_ORDER); + +const withDocuments = (types: string[]) => (types.includes('database') && !types.includes('search') ? [...types, 'search'] : types); + async function deleteFrom(types: string[], crateId?: string) { - for (const t of types.toSorted((a, b) => DELETE_ORDER.indexOf(a) - DELETE_ORDER.indexOf(b))) { + for (const t of DELETE_ORDER.filter((t) => types.includes(t))) { try { await INDEXER[t]?.delete(crateId); } catch (error) { @@ -157,12 +164,14 @@ async function deleteFrom(types: string[], crateId?: string) { } } -/** A crate's OCFL object, whose id is the crate's root id, and its metadata */ -async function readCrate(crateId: string) { +const readCrateJson = async (ocflObject: OcflObject): Promise<{ '@graph': RawEntity[] }> => + JSON.parse(await ocflObject.getFile({ logicalPath: 'ro-crate-metadata.json' }).text()); + +/** A crate's OCFL object, whose id is the crate's root id */ +async function loadObject(crateId: string) { const ocflObject = repository.object(crateId); await ocflObject.load(); - const jsonContent = await ocflObject.getFile({ logicalPath: 'ro-crate-metadata.json' }).text(); - return { ocflObject, crate: await ROCrate.create(JSON.parse(jsonContent)) }; + return ocflObject; } /** @@ -174,14 +183,22 @@ function topCollectionFinder() { const parentOf = (crateId: string) => { let parent = parents.get(crateId); if (!parent) { - parent = readCrate(crateId).then(({ crate }) => parentCollection(crate)); + parent = loadObject(crateId) + .then(readCrateJson) + .then((json) => { + const root = json['@graph'].find((e) => e['@id'] === crateId); + return root && parentCollection(root, crateId); + }); parents.set(crateId, parent); } return parent; }; return async (crate: ROCrate) => { let top = crate.rootId; - let parent = parentCollection(crate); + let parent = parentCollection(crate.root, crate.rootId); + if (!parents.has(top)) { + parents.set(top, Promise.resolve(parent)); + } const seen = new Set([top]); while (parent && !seen.has(parent)) { seen.add(parent); @@ -200,7 +217,8 @@ function topCollectionFinder() { async function indexObject(crateId: string, types: string[], findTopCollection: ReturnType, force?: boolean) { try { - const { ocflObject, crate } = await readCrate(crateId); + const ocflObject = await loadObject(crateId); + const crate = await ROCrate.create(await readCrateJson(ocflObject)); log.debug({ crateId: ocflObject.id }, 'Found OCFL object'); if (force) { setState(ocflObject.id, types, State.INDEXING); @@ -212,13 +230,12 @@ async function indexObject(crateId: string, types: string[], findTopCollection: } } const topCollection = await findTopCollection(crate); - let rows: EntityIds | undefined; - for (const t of types) { + for (const t of INDEX_ORDER.filter((t) => types.includes(t))) { const indexer = INDEXER[t]; if (indexer) { setState(ocflObject.id, t, State.INDEXING); try { - rows = (await indexer.index({ crateObject: wrap(ocflObject), crate, topCollection, rows })) ?? rows; + await indexer.index({ crateObject: wrap(ocflObject), crate, topCollection }); } catch (error) { log.error({ err: error, crateId: ocflObject.id, indexer: t }, 'Error indexing object'); } finally { @@ -246,7 +263,7 @@ async function idsMatching(pattern?: RegExp) { /** Indexes `crateIds`, the crates matching it, or every crate, marking them all queued before indexing any */ export async function createIndex(crateIds?: string | string[] | RegExp, type?: string | string[], force?: boolean) { log.debug('Indexing started'); - const types = type ? ([] as string[]).concat(type) : Object.keys(INDEXER); + const types = force ? withDocuments(typesOf(type)) : typesOf(type); // Given ids are queued before the first await, so callers see them in progress at once const ids = typeof crateIds === 'string' || Array.isArray(crateIds) ? [crateIds].flat() : await idsMatching(crateIds); for (const id of ids) { @@ -260,7 +277,7 @@ export async function createIndex(crateIds?: string | string[] | RegExp, type?: } export async function deleteIndex(crateId?: string, type?: string | string[]) { - const types = type ? ([] as string[]).concat(type) : Object.keys(INDEXER); + const types = withDocuments(typesOf(type)); setState(crateId, types, State.DELETING); try { await deleteFrom(types, crateId); @@ -279,9 +296,7 @@ export async function* objects(prefix?: string, refresh?: boolean) { for await (const ocflObject of repository) { try { const inv = await ocflObject.getInventory(); - const jsonContent = await ocflObject.getFile({ logicalPath: 'ro-crate-metadata.json' }).text(); - const jsonParsed = JSON.parse(jsonContent); - const name = jsonParsed['@graph'].find((e: RawEntity) => e['@id'] === inv.id)?.name?.toString(); + const name = (await readCrateJson(ocflObject))['@graph'].find((e) => e['@id'] === inv.id)?.name?.toString(); yield { id: inv.id, name, path: ocflObject.root, state: await getState(inv.id) }; } catch (error) { log.error({ err: error, path: ocflObject.root }, 'Error reading object'); diff --git a/test/bootstrap.test.ts b/test/bootstrap.test.ts index be67ed8..c82f780 100644 --- a/test/bootstrap.test.ts +++ b/test/bootstrap.test.ts @@ -5,7 +5,7 @@ import { join } from 'node:path'; import { Client } from '@opensearch-project/opensearch'; import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; import { config } from '../src/configuration.ts'; -import { createSearchBackend } from '../src/indexer/search.ts'; +import { countDocuments, createSearchBackend } from '../src/indexer/search.ts'; import { prisma } from '../src/prisma.ts'; const crateIds = [ @@ -45,17 +45,7 @@ describe('import and index', () => { // Documents are searchable after OpenSearch's next refresh await vi.waitFor(async () => { - const { total } = await search.search({ - searchType: 'basic', - query: '', - filters: { terms: {}, ranges: {} }, - limit: 1, - offset: 0, - sort: 'relevance', - order: 'asc', - visibility: { id: { in: ids } }, - }); - expect(total).toBe(ids.length); + expect(await countDocuments(search, ids)).toBe(ids.length); }); await opensearch.close(); }); From d2c39d20b43062c17c826f6b4e9b6b97cd541d17 Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 15:13:54 +1100 Subject: [PATCH 13/20] test: share the OCFL testbed, log capture and row helpers ocflStorage and ocflTestbed in test/app.ts import crates into a new OCFL repository and build the app over it; capturingLogger in test/logs.ts is the logger mock's factory. The admin action tests use the indexer's rowIds and countDocuments, and one test for reindexing or deleting a crate of a distributed collection. --- src/admin/actions.test.ts | 98 ++++++++++--------------------------- src/admin/crates.test.ts | 51 +++++++++---------- src/licence.test.ts | 12 +---- src/root-collection.test.ts | 39 +++------------ src/search-total.test.ts | 58 ++++++---------------- src/search.test.ts | 38 +++----------- test/admin.ts | 5 +- test/app.ts | 61 ++++++++++++++++++++++- test/logs.ts | 13 +++++ 9 files changed, 155 insertions(+), 220 deletions(-) create mode 100644 test/logs.ts diff --git a/src/admin/actions.test.ts b/src/admin/actions.test.ts index b9e9246..b6c30ba 100644 --- a/src/admin/actions.test.ts +++ b/src/admin/actions.test.ts @@ -1,12 +1,9 @@ -import { execFileSync } from 'node:child_process'; -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { Client } from '@opensearch-project/opensearch'; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; -import { ALICE, adminTestbed, ocflRepository, PEOPLE, repositoryIn, repositoryOf } from '../../test/admin.ts'; -import type { buildTestApp, TestApp, TestRepository } from '../../test/app.ts'; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { ALICE, adminTestbed, PEOPLE, repositoryIn, repositoryOf } from '../../test/admin.ts'; +import { type buildTestApp, ocflRepository, ocflStorage, type TestApp, type TestRepository } from '../../test/app.ts'; import type { Browser } from '../../test/oidc.ts'; +import type { rowIds as crateRowIds } from '../indexer/database.ts'; +import type { countDocuments } from '../indexer/search.ts'; import { hasState, type Repository, State } from '../repository.ts'; const BUNDLED = 'arcp://name,bundled'; @@ -17,10 +14,9 @@ const CONTENT_RESTRICTED = 'arcp://name,distributed_content_restricted'; const METADATA_RESTRICTED = 'arcp://name,distributed_metadata_restricted'; const testbed = adminTestbed(); -let storage: string; -let opensearch: Client; +const { ocflRoot } = ocflStorage(); -const start = (repository: TestRepository) => testbed.start({ OCFL_ROOT: join(storage, 'ocfl'), ADMIN_USERS: ALICE.sub }, { repository }); +const start = (repository: TestRepository) => testbed.start({ OCFL_ROOT: ocflRoot(), ADMIN_USERS: ALICE.sub }, { repository }); const signIn = async (app: TestApp) => { const { browser, page } = await testbed.signInAsAdmin(app, ALICE); @@ -43,46 +39,20 @@ const flash = (body: string) => .replace(/\s+/g, ' ') .trim(); -beforeAll(async () => { - storage = await mkdtemp(join(tmpdir(), 'ldacapi-actions-')); - execFileSync('node', ['scripts/crates-import.ts'], { env: { ...process.env, OCFL_ROOT: join(storage, 'ocfl') }, stdio: 'ignore' }); -}); - -afterAll(async () => { - await opensearch?.close(); - await rm(storage, { recursive: true, force: true }); -}); - describe('index and delete actions', () => { let app: TestApp; let repository: Repository; let prisma: Awaited>['prisma']; - let entityIndex: string; let search: Awaited>['search']; + let crateRows: typeof crateRowIds; + let countSearchDocuments: typeof countDocuments; - /** The ids of the entity rows indexed from `crateId` */ - const rowIds = async (crateId: string) => - (await prisma.entity.findMany({ where: { meta: { path: ['crateId'], equals: crateId } }, select: { id: true } })).map(({ id }) => id).sort(); + /** The ids of the entity rows indexed from `crateIds`, sorted */ + const rowIds = async (...crateIds: string[]) => (await Promise.all(crateIds.map(crateRows))).flat().sort(); const entityCount = async (crateId: string) => (await rowIds(crateId)).length; - /** How many of `ids` have a search document */ - const searchCount = async (ids: string[]) => { - if (!ids.length || !(await opensearch.indices.exists({ index: entityIndex })).body) { - return 0; - } - const { total } = await search.search({ - searchType: 'basic', - query: '', - filters: { terms: {}, ranges: {} }, - limit: 1, - offset: 0, - sort: 'relevance', - order: 'asc', - visibility: { id: { in: ids } }, - }); - return total; - }; + const searchCount = (ids: string[]) => countSearchDocuments(search, ids); /** Waits until no crate is queued, indexing or deleting, so an earlier action can't leak into the next */ const waitUntilSettled = () => @@ -102,10 +72,10 @@ describe('index and delete actions', () => { }; beforeAll(async () => { - let config: Awaited>['config']; - ({ app, repository, prisma, config, search } = await start(ocflRepository)); - entityIndex = config.search.entityIndex; - opensearch = new Client({ node: config.opensearchUrl }); + ({ app, repository, prisma, search } = await start(ocflRepository)); + // Imported once the app has read its configuration + ({ rowIds: crateRows } = await import('../indexer/database.ts')); + ({ countDocuments: countSearchDocuments } = await import('../indexer/search.ts')); }); beforeEach(async () => { @@ -162,19 +132,22 @@ describe('index and delete actions', () => { expect(await searchCount(ids)).toBe(ids.length); }); - it('reindexes one crate of a distributed collection, leaving the rest of it alone', async () => { + it.each([ + ['reindexes', 'index', { force: 'true' }, true], + ['deletes', 'delete', {}, false], + ])('%s one crate of a distributed collection, leaving the rest of it alone', async (_, action, form, kept) => { const browser = await signIn(app); await act(browser, 'index'); await waitUntilIndexed(DISTRIBUTED_ROOT, CONTENT_RESTRICTED, METADATA_RESTRICTED); const ids = await rowIds(CONTENT_RESTRICTED); - const siblings = [...(await rowIds(DISTRIBUTED_ROOT)), ...(await rowIds(METADATA_RESTRICTED))]; + const siblings = await rowIds(DISTRIBUTED_ROOT, METADATA_RESTRICTED); - await act(browser, 'index', { crateId: CONTENT_RESTRICTED, force: 'true' }); + await act(browser, action, { crateId: CONTENT_RESTRICTED, ...form }); - await waitUntilIndexed(CONTENT_RESTRICTED); - expect(await rowIds(CONTENT_RESTRICTED)).toEqual(ids); - expect(await searchCount(ids)).toBe(ids.length); - expect([...(await rowIds(DISTRIBUTED_ROOT)), ...(await rowIds(METADATA_RESTRICTED))]).toEqual(siblings); + await waitUntilSettled(); + expect(await rowIds(CONTENT_RESTRICTED)).toEqual(kept ? ids : []); + expect(await searchCount(ids)).toBe(kept ? ids.length : 0); + expect(await rowIds(DISTRIBUTED_ROOT, METADATA_RESTRICTED)).toEqual(siblings); expect(await searchCount(siblings)).toBe(siblings.length); }); @@ -247,23 +220,6 @@ describe('index and delete actions', () => { expect(await searchCount(others)).toBe(others.length); }); - it('deletes one crate of a distributed collection, leaving the rest of it alone', async () => { - const browser = await signIn(app); - await act(browser, 'index'); - await waitUntilIndexed(DISTRIBUTED_ROOT, CONTENT_RESTRICTED, METADATA_RESTRICTED); - const ids = await rowIds(CONTENT_RESTRICTED); - const siblings = [...(await rowIds(DISTRIBUTED_ROOT)), ...(await rowIds(METADATA_RESTRICTED))]; - - await act(browser, 'delete', { crateId: CONTENT_RESTRICTED }); - - await vi.waitFor(async () => { - expect(await prisma.entity.count({ where: { id: { in: ids } } })).toBe(0); - expect(await searchCount(ids)).toBe(0); - }); - expect([...(await rowIds(DISTRIBUTED_ROOT)), ...(await rowIds(METADATA_RESTRICTED))]).toEqual(siblings); - expect(await searchCount(siblings)).toBe(siblings.length); - }); - it('keeps a crate’s database rows when deleting its search documents fails', async () => { const browser = await signIn(app); await act(browser, 'index'); @@ -288,7 +244,7 @@ describe('index and delete actions', () => { const browser = await signIn(app); await act(browser, 'index'); await waitUntilIndexed(BUNDLED, PEOPLE); - const ids = [...(await rowIds(BUNDLED)), ...(await rowIds(PEOPLE))]; + const ids = await rowIds(BUNDLED, PEOPLE); const page = await act(browser, 'delete'); diff --git a/src/admin/crates.test.ts b/src/admin/crates.test.ts index c75cd3d..92ef5bc 100644 --- a/src/admin/crates.test.ts +++ b/src/admin/crates.test.ts @@ -1,10 +1,8 @@ -import { execFileSync } from 'node:child_process'; -import { cp, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; +import { cp, mkdir, writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; -import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; -import { ALICE, adminTestbed, ocflRepository, PEOPLE, repositoryIn } from '../../test/admin.ts'; -import type { TestApp, TestRepository } from '../../test/app.ts'; +import { beforeAll, describe, expect, it, vi } from 'vitest'; +import { ALICE, adminTestbed, PEOPLE, repositoryIn } from '../../test/admin.ts'; +import { ocflRepository, ocflStorage, type TestApp, type TestRepository } from '../../test/app.ts'; import type { Repository } from '../repository.ts'; const BUNDLED = 'arcp://name,bundled'; @@ -59,9 +57,25 @@ const crate = (id: string, name: string) => ({ }); const testbed = adminTestbed(); -let storage: string; +const { ocflRoot } = ocflStorage({ + source: async (scratch) => { + const source = join(scratch, 'source'); + await mkdir(source); + for (const name of ['bundled', 'people']) { + await cp(resolve('test-data', name), join(source, name), { recursive: true }); + } + for (const [directory, id, name] of [ + ['extra', EXTRA, EXTRA_NAME], + ['more', MORE, 'More people'], + ] as const) { + await mkdir(join(source, directory)); + await writeFile(join(source, directory, 'ro-crate-metadata.json'), JSON.stringify(crate(id, name))); + } + return source; + }, +}); -const start = (repository: TestRepository) => testbed.start({ OCFL_ROOT: join(storage, 'ocfl'), ADMIN_USERS: ALICE.sub }, { repository }); +const start = (repository: TestRepository) => testbed.start({ OCFL_ROOT: ocflRoot(), ADMIN_USERS: ALICE.sub }, { repository }); const signIn = async (app: TestApp) => { const signedIn = await testbed.signInAsAdmin(app, ALICE); @@ -69,27 +83,6 @@ const signIn = async (app: TestApp) => { return signedIn; }; -beforeAll(async () => { - storage = await mkdtemp(join(tmpdir(), 'ldacapi-crates-')); - const source = join(storage, 'source'); - await mkdir(source); - for (const name of ['bundled', 'people']) { - await cp(resolve('test-data', name), join(source, name), { recursive: true }); - } - for (const [directory, id, name] of [ - ['extra', EXTRA, EXTRA_NAME], - ['more', MORE, 'More people'], - ] as const) { - await mkdir(join(source, directory)); - await writeFile(join(source, directory, 'ro-crate-metadata.json'), JSON.stringify(crate(id, name))); - } - execFileSync('node', ['scripts/crates-import.ts', source], { env: { ...process.env, OCFL_ROOT: join(storage, 'ocfl') }, stdio: 'ignore' }); -}); - -afterAll(async () => { - await rm(storage, { recursive: true, force: true }); -}); - describe('the crate tree', () => { let app: TestApp; let repository: Repository; diff --git a/src/licence.test.ts b/src/licence.test.ts index 4e15738..11d25ef 100644 --- a/src/licence.test.ts +++ b/src/licence.test.ts @@ -1,6 +1,5 @@ import { createServer, type IncomingMessage, type Server } from 'node:http'; import type { AddressInfo } from 'node:net'; -import { Writable } from 'node:stream'; import { generateKeyPair, importJWK, type JWTPayload, SignJWT } from 'jose'; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { buildTestApp, repositoryWithContent, type TestApp } from '../test/app.ts'; @@ -22,16 +21,7 @@ const RESTRICTED_ENTITY = 'http://test/licence/restricted'; const logs = vi.hoisted(() => [] as Record[]); -vi.mock('./logger.ts', async () => { - const { pino } = await import('pino'); - const stream = new Writable({ - write(chunk, _encoding, done) { - logs.push(JSON.parse(chunk.toString())); - done(); - }, - }); - return { logger: pino({ level: 'error' }, stream) }; -}); +vi.mock('./logger.ts', async () => (await import('../test/logs.ts')).capturingLogger(logs, 'error')); let app: TestApp; let prisma: Awaited>['prisma']; diff --git a/src/root-collection.test.ts b/src/root-collection.test.ts index 4b5249b..5a75b5d 100644 --- a/src/root-collection.test.ts +++ b/src/root-collection.test.ts @@ -1,52 +1,29 @@ -import { execFileSync } from 'node:child_process'; -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { ocflRepository } from '../test/admin.ts'; -import { buildTestApp } from '../test/app.ts'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { ocflTestbed } from '../test/app.ts'; const BUNDLED = 'arcp://name,bundled'; const ROOT = 'arcp://name,distributed_root'; const CONTENT = 'arcp://name,distributed_content_restricted'; const METADATA = 'arcp://name,distributed_metadata_restricted'; -let storage: string; -let started: Awaited>; +const testbed = ocflTestbed(); const rootCollectionOf = async (id: string) => { - const response = await started.app.inject({ method: 'GET', url: `${started.config.prefix}/entity/${encodeURIComponent(id)}` }); + const { app, config } = testbed.started; + const response = await app.inject({ method: 'GET', url: `${config.prefix}/entity/${encodeURIComponent(id)}` }); expect(response.statusCode).toBe(200); return response.json().rootCollection?.id ?? null; }; -const collectionFacet = async () => { - const response = await started.app.inject({ method: 'POST', url: `${started.config.prefix}/search`, payload: { query: '' } }); - expect(response.statusCode).toBe(200); - return response.json().facets.rootCollection.map(({ name }: { name: string }) => name); -}; - -beforeAll(async () => { - storage = await mkdtemp(join(tmpdir(), 'ldacapi-root-collection-')); - const ocflRoot = join(storage, 'ocfl'); - execFileSync('node', ['scripts/crates-import.ts'], { env: { ...process.env, OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, stdio: 'ignore' }); - started = await buildTestApp({ OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, { repository: ocflRepository }); -}); - -afterAll(async () => { - await started?.repository.deleteIndex(); - await started?.app.close(); - await started?.prisma.$disconnect(); - await rm(storage, { recursive: true, force: true }); -}); +const collectionFacet = async () => (await testbed.search({ query: '' })).facets.rootCollection.map(({ name }: { name: string }) => name); describe.each([ ['before', [ROOT, CONTENT, METADATA, BUNDLED]], ['after', [CONTENT, METADATA, ROOT, BUNDLED]], ])('rootCollection, with the distributed parent crate indexed %s its children', (_order, crates) => { beforeAll(async () => { - await started.repository.deleteIndex(); - await started.repository.createIndex(crates); + await testbed.started.repository.deleteIndex(); + await testbed.started.repository.createIndex(crates); }); it('is empty on a top-level collection', async () => { diff --git a/src/search-total.test.ts b/src/search-total.test.ts index c0d825e..4f802f3 100644 --- a/src/search-total.test.ts +++ b/src/search-total.test.ts @@ -1,24 +1,11 @@ -import { execFileSync } from 'node:child_process'; -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; +import { mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; -import { Writable } from 'node:stream'; -import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; -import { ocflRepository } from '../test/admin.ts'; -import { buildTestApp } from '../test/app.ts'; +import { beforeAll, describe, expect, it, vi } from 'vitest'; +import { ocflTestbed } from '../test/app.ts'; const logs = vi.hoisted(() => [] as Record[]); -vi.mock('./logger.ts', async () => { - const { pino } = await import('pino'); - const stream = new Writable({ - write(chunk, _encoding, done) { - logs.push(JSON.parse(chunk.toString())); - done(); - }, - }); - return { logger: pino({ level: 'info' }, stream) }; -}); +vi.mock('./logger.ts', async () => (await import('../test/logs.ts')).capturingLogger(logs, 'info')); const ROOT = 'arcp://name,unwritable'; const WRITTEN = `${ROOT}/written`; @@ -43,32 +30,19 @@ const crate = { ], }; -let storage: string; -let started: Awaited>; - -const search = async (body: Record) => { - const response = await started.app.inject({ method: 'POST', url: `${started.config.prefix}/search`, payload: body }); - expect(response.statusCode).toBe(200); - return response.json(); -}; - -beforeAll(async () => { - storage = await mkdtemp(join(tmpdir(), 'ldacapi-search-total-')); - const source = join(storage, 'crates'); - await mkdir(join(source, 'unwritable'), { recursive: true }); - await writeFile(join(source, 'unwritable', 'ro-crate-metadata.json'), JSON.stringify(crate)); - const ocflRoot = join(storage, 'ocfl'); - execFileSync('node', ['scripts/crates-import.ts', source], { env: { ...process.env, OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, stdio: 'ignore' }); - started = await buildTestApp({ OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, { repository: ocflRepository }); - await started.repository.deleteIndex(); - await started.repository.createIndex(ROOT); +const testbed = ocflTestbed({ + source: async (scratch) => { + const source = join(scratch, 'crates'); + await mkdir(join(source, 'unwritable'), { recursive: true }); + await writeFile(join(source, 'unwritable', 'ro-crate-metadata.json'), JSON.stringify(crate)); + return source; + }, }); +const { search } = testbed; -afterAll(async () => { - await started?.repository.deleteIndex(); - await started?.app.close(); - await started?.prisma.$disconnect(); - await rm(storage, { recursive: true, force: true }); +beforeAll(async () => { + await testbed.started.repository.deleteIndex(); + await testbed.started.repository.createIndex(ROOT); }); describe('an entity whose row fails to be written', () => { @@ -81,7 +55,7 @@ describe('an entity whose row fails to be written', () => { }); it('is not counted when the search index is rebuilt alone', async () => { - await started.repository.createIndex(ROOT, 'search', true); + await testbed.started.repository.createIndex(ROOT, 'search', true); const { total, entities } = await search({ query: '', limit: 100 }); diff --git a/src/search.test.ts b/src/search.test.ts index e575778..f4b0b6b 100644 --- a/src/search.test.ts +++ b/src/search.test.ts @@ -1,43 +1,21 @@ -import { execFileSync } from 'node:child_process'; -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { ocflRepository } from '../test/admin.ts'; -import { buildTestApp } from '../test/app.ts'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { ocflTestbed } from '../test/app.ts'; const OBJECT = 'http://pcdm.org/models#Object'; const MEDIA = 'http://schema.org/MediaObject'; const BUNDLED = 'arcp://name,bundled'; -let storage: string; -let started: Awaited>; - -const search = async (body: Record) => { - const response = await started.app.inject({ method: 'POST', url: `${started.config.prefix}/search`, payload: body }); - expect(response.statusCode).toBe(200); - return response.json(); -}; +const testbed = ocflTestbed(); +const { search } = testbed; beforeAll(async () => { - storage = await mkdtemp(join(tmpdir(), 'ldacapi-search-')); - const ocflRoot = join(storage, 'ocfl'); - execFileSync('node', ['scripts/crates-import.ts'], { env: { ...process.env, OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, stdio: 'ignore' }); - started = await buildTestApp({ OCFL_ROOT: ocflRoot, LOG_LEVEL: 'silent' }, { repository: ocflRepository }); - await started.repository.deleteIndex(); - await started.repository.createIndex(BUNDLED); -}); - -afterAll(async () => { - await started?.repository.deleteIndex(); - await started?.app.close(); - await started?.prisma.$disconnect(); - await rm(storage, { recursive: true, force: true }); + await testbed.started.repository.deleteIndex(); + await testbed.started.repository.createIndex(BUNDLED); }); describe('search by recordType and entityType', () => { it('declares a recordType facet, and entityType as a filter only', async () => { - const response = await started.app.inject({ method: 'GET', url: `${started.config.prefix}/capabilities` }); + const response = await testbed.started.app.inject({ method: 'GET', url: `${testbed.started.config.prefix}/capabilities` }); const { facets, filters } = response.json().search; expect(facets).toHaveProperty('recordType'); @@ -65,7 +43,7 @@ describe('search by recordType and entityType', () => { it('filters on entityType, agreeing with listing entities by entityType', async () => { const { total, entities } = await search({ query: '', filters: { entityType: [OBJECT] }, limit: 100 }); const listed = ( - await started.app.inject({ method: 'GET', url: `${started.config.prefix}/entities?entityType=${encodeURIComponent(OBJECT)}&limit=100` }) + await testbed.started.app.inject({ method: 'GET', url: `${testbed.started.config.prefix}/entities?entityType=${encodeURIComponent(OBJECT)}&limit=100` }) ).json(); expect(total).toBeGreaterThan(0); diff --git a/test/admin.ts b/test/admin.ts index 00b41b3..849ff84 100644 --- a/test/admin.ts +++ b/test/admin.ts @@ -1,5 +1,5 @@ import { afterAll, beforeAll, vi } from 'vitest'; -import { initRepository, type Repository } from '../src/repository.ts'; +import type { Repository } from '../src/repository.ts'; import { buildTestApp, type TestApp } from './app.ts'; import { startFakeUpstream, type UpstreamUser } from './fake-upstream.ts'; import { Browser } from './oidc.ts'; @@ -19,9 +19,6 @@ const oidcEnv = (upstreamIssuer: string) => ({ OIDC_UPSTREAM_CLIENT_SECRET: UPSTREAM_CLIENT.clientSecret, }); -/** The OCFL repository at `OCFL_ROOT` */ -export const ocflRepository = (deps: Record) => initRepository('ocfl', deps); - /** A repository of `crates`, each with its index state, recording what it is asked to do */ export const repositoryOf = (crates: { id: string; name?: string; state: Record }[]) => { const fake = { diff --git a/test/app.ts b/test/app.ts index 636456b..3a94c57 100644 --- a/test/app.ts +++ b/test/app.ts @@ -1,8 +1,12 @@ +import { execFileSync } from 'node:child_process'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { Readable } from 'node:stream'; import type { Client } from '@opensearch-project/opensearch'; -import { vi } from 'vitest'; +import { afterAll, beforeAll, expect, vi } from 'vitest'; import type { createSearchBackend } from '../src/indexer/search.ts'; -import type { Repository } from '../src/repository.ts'; +import { initRepository, type Repository } from '../src/repository.ts'; type RepositoryDeps = { opensearchClient: Client; search: ReturnType }; @@ -24,6 +28,9 @@ export const repositoryWithContent = { }), } as unknown as Repository; +/** The OCFL repository at `OCFL_ROOT` */ +export const ocflRepository = (deps: Record) => initRepository('ocfl', deps); + /** Builds the app with `env` set while configuration is read, which happens on first import */ export const buildTestApp = async (env: Record, options: { repository?: TestRepository } = {}) => { for (const [name, value] of Object.entries(env)) { @@ -47,3 +54,53 @@ export const buildTestApp = async (env: Record, options: { repos }; export type TestApp = Awaited>['app']; + +/** + * A new OCFL repository for a file's tests, imported from the directory `source` returns (test-data by default), + * which it may create under the scratch directory it is given + */ +export const ocflStorage = ({ source }: { source?: (scratch: string) => Promise } = {}) => { + let storage: string; + const ocflRoot = () => join(storage, 'ocfl'); + + beforeAll(async () => { + storage = await mkdtemp(join(tmpdir(), 'ldacapi-')); + const sources = source ? [await source(storage)] : []; + execFileSync('node', ['scripts/crates-import.ts', ...sources], { env: { ...process.env, OCFL_ROOT: ocflRoot(), LOG_LEVEL: 'silent' }, stdio: 'ignore' }); + }); + + afterAll(async () => { + await rm(storage, { recursive: true, force: true }); + }); + + return { ocflRoot }; +}; + +/** The app over a new OCFL repository (see `ocflStorage`), its indexes deleted after the file's tests */ +export const ocflTestbed = (options: Parameters[0] = {}) => { + const { ocflRoot } = ocflStorage(options); + let started: Awaited>; + + beforeAll(async () => { + started = await buildTestApp({ OCFL_ROOT: ocflRoot(), LOG_LEVEL: 'silent' }, { repository: ocflRepository }); + }); + + afterAll(async () => { + await started?.repository.deleteIndex(); + await started?.app.close(); + await started?.prisma.$disconnect(); + }); + + return { + get started() { + return started; + }, + + /** POSTs `body` to the search endpoint, expecting success */ + async search(body: Record) { + const response = await started.app.inject({ method: 'POST', url: `${started.config.prefix}/search`, payload: body }); + expect(response.statusCode).toBe(200); + return response.json(); + }, + }; +}; diff --git a/test/logs.ts b/test/logs.ts new file mode 100644 index 0000000..d07d7d7 --- /dev/null +++ b/test/logs.ts @@ -0,0 +1,13 @@ +import { Writable } from 'node:stream'; +import { pino } from 'pino'; + +/** The module `vi.mock('./logger.ts', …)` returns: a logger at `level` that writes each record into `logs` */ +export const capturingLogger = (logs: Record[], level: string) => { + const stream = new Writable({ + write(chunk, _encoding, done) { + logs.push(JSON.parse(chunk.toString())); + done(); + }, + }); + return { logger: pino({ level }, stream) }; +}; From 75f4ba524bca8af1ed8d33c5de7240522a587da4 Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 15:23:43 +1100 Subject: [PATCH 14/20] fix: pnpm index deletes every index before rebuilding them Rows from before meta.crateId don't record their crate, so deleting by crate missed them and indexing them again failed on duplicate ids. --- README.md | 8 +++++--- scripts/index.ts | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 652c315..d551be0 100644 --- a/README.md +++ b/README.md @@ -461,9 +461,11 @@ docker compose run --rm api pnpm db:deploy docker compose up -d ``` -If an upgrade changes the search fields, the API and `pnpm index` stop with an -error naming the changed field. Delete the search index (named by -`ENTITY_INDEX`, `entities` by default), rebuild both indexes, and start again: +`pnpm index` deletes both indexes and rebuilds them, so run it when an upgrade +changes what they hold. If an upgrade changes the search fields, the API and +`pnpm index` stop with an error naming the changed field, as they check the +search index's mapping before anything else. Delete the search index (named by +`ENTITY_INDEX`, `entities` by default) first, then rebuild and start again: ```sh docker compose exec opensearch curl -sX DELETE localhost:9200/entities diff --git a/scripts/index.ts b/scripts/index.ts index 4829d29..6ec9ab2 100644 --- a/scripts/index.ts +++ b/scripts/index.ts @@ -1,9 +1,11 @@ import { Client } from '@opensearch-project/opensearch'; import { config } from '../src/configuration.ts'; -import { createIndex, init } from '../src/ocfl.ts'; +import { createIndex, deleteIndex, init } from '../src/ocfl.ts'; import { prisma } from '../src/prisma.ts'; const opensearch = new Client({ node: config.opensearchUrl }); await init({ opensearchClient: opensearch }); -await createIndex(undefined, undefined, true); +// A full rebuild, as deleting by crate misses rows that don't record their crate +await deleteIndex(); +await createIndex(); await Promise.all([prisma.$disconnect(), opensearch.close()]); From 7abfd0a9c17dc4ba52f78d549dc8fb04f12fca60 Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 15:23:57 +1100 Subject: [PATCH 15/20] refactor: one list of index types, one way to read a crate's raw root and first id INDEX_TYPES and IndexType type the indexers, their orders and states, and the admin page's columns. memberOf uses firstStringOrId, which now takes a single value or an array. --- src/admin/actions.test.ts | 6 ++-- src/admin/crates.ts | 6 ++-- src/indexer/indexer.ts | 4 +-- src/ocfl.ts | 72 +++++++++++++++++---------------------- src/repository.ts | 18 ++++++---- src/utils.ts | 5 +-- 6 files changed, 54 insertions(+), 57 deletions(-) diff --git a/src/admin/actions.test.ts b/src/admin/actions.test.ts index b6c30ba..fb77e15 100644 --- a/src/admin/actions.test.ts +++ b/src/admin/actions.test.ts @@ -44,11 +44,11 @@ describe('index and delete actions', () => { let repository: Repository; let prisma: Awaited>['prisma']; let search: Awaited>['search']; - let crateRows: typeof crateRowIds; + let rowIdsOfCrate: typeof crateRowIds; let countSearchDocuments: typeof countDocuments; /** The ids of the entity rows indexed from `crateIds`, sorted */ - const rowIds = async (...crateIds: string[]) => (await Promise.all(crateIds.map(crateRows))).flat().sort(); + const rowIds = async (...crateIds: string[]) => (await Promise.all(crateIds.map(rowIdsOfCrate))).flat().sort(); const entityCount = async (crateId: string) => (await rowIds(crateId)).length; @@ -74,7 +74,7 @@ describe('index and delete actions', () => { beforeAll(async () => { ({ app, repository, prisma, search } = await start(ocflRepository)); // Imported once the app has read its configuration - ({ rowIds: crateRows } = await import('../indexer/database.ts')); + ({ rowIds: rowIdsOfCrate } = await import('../indexer/database.ts')); ({ countDocuments: countSearchDocuments } = await import('../indexer/search.ts')); }); diff --git a/src/admin/crates.ts b/src/admin/crates.ts index 0f9ebb8..6080c56 100644 --- a/src/admin/crates.ts +++ b/src/admin/crates.ts @@ -1,8 +1,8 @@ -import { hasState, type RepositoryObject, State } from '../repository.ts'; +import { hasState, INDEX_TYPES, type IndexType, type RepositoryObject, State } from '../repository.ts'; type Crate = RepositoryObject & { label: string; children: Crate[] }; -const INDEXES = { database: 'Database', search: 'Search' } as const; +const INDEX_LABELS: Record = { database: 'Database', search: 'Search' }; /** How often the page reloads while an index is changing */ const REFRESH_SECONDS = 5; @@ -59,7 +59,7 @@ const crateRow = (crate: Crate): CrateRow => { return { id: crate.id, label: crate.label, - states: Object.entries(INDEXES).map(([index, label]) => ({ label, value: crate.state[index] ?? 'not indexed' })), + states: INDEX_TYPES.map((index) => ({ label: INDEX_LABELS[index], value: crate.state[index] ?? 'not indexed' })), actions: [ { action: 'index', label: indexed ? 'Reindex' : 'Index', crateId: crate.id, force: indexed }, { action: 'delete', label: 'Delete', crateId: crate.id, confirm }, diff --git a/src/indexer/indexer.ts b/src/indexer/indexer.ts index 71dc0ce..3ee7ee6 100644 --- a/src/indexer/indexer.ts +++ b/src/indexer/indexer.ts @@ -155,10 +155,8 @@ export class Indexer { /** An entity's rootCollection: empty on the top-level collection itself */ export const rootCollection = (topCollection: string, entityId: string) => (entityId === topCollection ? null : topCollection); -const firstId = (value: unknown): string | undefined => ([value ?? []].flat()[0] as { '@id'?: string } | undefined)?.['@id']; - /** The first collection `entity` is a member of, by `pcdm:memberOf` or `memberOf`, in a parsed crate or raw JSON-LD */ -export const memberOf = (entity: Record) => firstId(entity['pcdm:memberOf']) || firstId(entity.memberOf); +export const memberOf = (entity: Record) => firstStringOrId(entity['pcdm:memberOf']) || firstStringOrId(entity.memberOf); /** The id of the collection a crate's root is a member of, if any */ export function parentCollection(root: Record, rootId: string): string | undefined { diff --git a/src/ocfl.ts b/src/ocfl.ts index 2f14b83..234d49a 100644 --- a/src/ocfl.ts +++ b/src/ocfl.ts @@ -10,14 +10,14 @@ import { DatabaseIndexer } from './indexer/database.ts'; import { type CrateObject, type Indexer, parentCollection } from './indexer/indexer.ts'; import { createSearchBackend, SearchIndexer } from './indexer/search.ts'; import { logger } from './logger.ts'; -import { type IndexState, State } from './repository.ts'; +import { INDEX_TYPES, type IndexState, type IndexStates, type IndexType, State } from './repository.ts'; const log = logger.child({ module: 'ocfl' }); const { defaultLicense, defaultMetadataLicense } = config; -const stateCache: { [key: string]: { [key: string]: IndexState | undefined } } = {}; -let INDEXER: { [key: string]: Indexer }; +const stateCache: Record = {}; +let INDEXER: Record; let repository: ReturnType; let backend: StorageBackend; @@ -107,14 +107,14 @@ function wrap(ocflObject: OcflObject): CrateObject { }; } -function _setState(crateId: string, types: string | string[], state?: IndexState) { - stateCache[crateId] = stateCache[crateId] || {}; - for (const type of ([] as string[]).concat(types)) { +function _setState(crateId: string, types: IndexType | IndexType[], state?: IndexState) { + stateCache[crateId] ??= {}; + for (const type of [types].flat()) { stateCache[crateId][type] = state; } } -function setState(crateId: string | undefined, types: string | string[], state?: IndexState) { +function setState(crateId: string | undefined, types: IndexType | IndexType[], state?: IndexState) { if (crateId) { _setState(crateId, types, state); } else { @@ -124,18 +124,10 @@ function setState(crateId: string | undefined, types: string | string[], state?: } } -export async function getState(crateId: string, type?: string) { - const indexers: [string, Indexer | undefined][] = type ? [[type, INDEXER[type]]] : Object.entries(INDEXER); - for (const [name, indexer] of indexers) { - if (!indexer) { - return; - } - if (!stateCache[crateId]?.[name]) { - const count = await indexer.count(crateId); - if (count > 0) { - stateCache[crateId] = stateCache[crateId] || {}; - stateCache[crateId][name] = State.INDEXED; - } +export async function getState(crateId: string, type?: IndexType): Promise { + for (const name of type ? [type] : INDEX_TYPES) { + if (!stateCache[crateId]?.[name] && (await INDEXER[name].count(crateId)) > 0) { + _setState(crateId, name, State.INDEXED); } } // A copy, as indexing goes on changing the cache after this returns @@ -146,17 +138,17 @@ export async function getState(crateId: string, type?: string) { * The search index finds a crate's documents, and their rootCollection, by its database rows. So the rows are indexed * first and deleted last, deleting stops at the first failure, and deleting the rows deletes the documents too. */ -const INDEX_ORDER = ['database', 'search']; -const DELETE_ORDER = ['search', 'database']; +const INDEX_ORDER: readonly IndexType[] = INDEX_TYPES; +const DELETE_ORDER: readonly IndexType[] = ['search', 'database']; -const typesOf = (type?: string | string[]) => (type ? [type].flat() : INDEX_ORDER); +const typesOf = (type?: IndexType | IndexType[]) => (type ? [type].flat() : [...INDEX_ORDER]); -const withDocuments = (types: string[]) => (types.includes('database') && !types.includes('search') ? [...types, 'search'] : types); +const withDocuments = (types: IndexType[]): IndexType[] => (types.includes('database') && !types.includes('search') ? [...types, 'search'] : types); -async function deleteFrom(types: string[], crateId?: string) { +async function deleteFrom(types: IndexType[], crateId?: string) { for (const t of DELETE_ORDER.filter((t) => types.includes(t))) { try { - await INDEXER[t]?.delete(crateId); + await INDEXER[t].delete(crateId); } catch (error) { log.error({ err: error, crateId, indexer: t }, 'Error deleting index'); throw error; @@ -167,6 +159,9 @@ async function deleteFrom(types: string[], crateId?: string) { const readCrateJson = async (ocflObject: OcflObject): Promise<{ '@graph': RawEntity[] }> => JSON.parse(await ocflObject.getFile({ logicalPath: 'ro-crate-metadata.json' }).text()); +/** The root entity of a crate's raw JSON-LD, whose id is the OCFL object's */ +const rawRoot = (json: { '@graph': RawEntity[] }, crateId: string) => json['@graph'].find((e) => e['@id'] === crateId); + /** A crate's OCFL object, whose id is the crate's root id */ async function loadObject(crateId: string) { const ocflObject = repository.object(crateId); @@ -186,7 +181,7 @@ function topCollectionFinder() { parent = loadObject(crateId) .then(readCrateJson) .then((json) => { - const root = json['@graph'].find((e) => e['@id'] === crateId); + const root = rawRoot(json, crateId); return root && parentCollection(root, crateId); }); parents.set(crateId, parent); @@ -215,7 +210,7 @@ function topCollectionFinder() { }; } -async function indexObject(crateId: string, types: string[], findTopCollection: ReturnType, force?: boolean) { +async function indexObject(crateId: string, types: IndexType[], findTopCollection: ReturnType, force?: boolean) { try { const ocflObject = await loadObject(crateId); const crate = await ROCrate.create(await readCrateJson(ocflObject)); @@ -231,16 +226,13 @@ async function indexObject(crateId: string, types: string[], findTopCollection: } const topCollection = await findTopCollection(crate); for (const t of INDEX_ORDER.filter((t) => types.includes(t))) { - const indexer = INDEXER[t]; - if (indexer) { - setState(ocflObject.id, t, State.INDEXING); - try { - await indexer.index({ crateObject: wrap(ocflObject), crate, topCollection }); - } catch (error) { - log.error({ err: error, crateId: ocflObject.id, indexer: t }, 'Error indexing object'); - } finally { - setState(ocflObject.id, t); - } + setState(ocflObject.id, t, State.INDEXING); + try { + await INDEXER[t].index({ crateObject: wrap(ocflObject), crate, topCollection }); + } catch (error) { + log.error({ err: error, crateId: ocflObject.id, indexer: t }, 'Error indexing object'); + } finally { + setState(ocflObject.id, t); } } } catch (error) { @@ -261,7 +253,7 @@ async function idsMatching(pattern?: RegExp) { } /** Indexes `crateIds`, the crates matching it, or every crate, marking them all queued before indexing any */ -export async function createIndex(crateIds?: string | string[] | RegExp, type?: string | string[], force?: boolean) { +export async function createIndex(crateIds?: string | string[] | RegExp, type?: IndexType | IndexType[], force?: boolean) { log.debug('Indexing started'); const types = force ? withDocuments(typesOf(type)) : typesOf(type); // Given ids are queued before the first await, so callers see them in progress at once @@ -276,7 +268,7 @@ export async function createIndex(crateIds?: string | string[] | RegExp, type?: log.debug('Indexing finished'); } -export async function deleteIndex(crateId?: string, type?: string | string[]) { +export async function deleteIndex(crateId?: string, type?: IndexType | IndexType[]) { const types = withDocuments(typesOf(type)); setState(crateId, types, State.DELETING); try { @@ -296,7 +288,7 @@ export async function* objects(prefix?: string, refresh?: boolean) { for await (const ocflObject of repository) { try { const inv = await ocflObject.getInventory(); - const name = (await readCrateJson(ocflObject))['@graph'].find((e) => e['@id'] === inv.id)?.name?.toString(); + const name = rawRoot(await readCrateJson(ocflObject), inv.id)?.name?.toString(); yield { id: inv.id, name, path: ocflObject.root, state: await getState(inv.id) }; } catch (error) { log.error({ err: error, path: ocflObject.root }, 'Error reading object'); diff --git a/src/repository.ts b/src/repository.ts index 1fc72a5..93f01f3 100644 --- a/src/repository.ts +++ b/src/repository.ts @@ -9,21 +9,27 @@ export const State = { export type IndexState = (typeof State)[keyof typeof State]; +export const INDEX_TYPES = ['database', 'search'] as const; + +export type IndexType = (typeof INDEX_TYPES)[number]; + +/** Each index's state, absent or undefined while it is idle */ +export type IndexStates = Partial>; + /** Whether any index is in one of `values` */ -export const hasState = (state: Partial>, ...values: IndexState[]) => - Object.values(state).some((value) => value !== undefined && values.includes(value)); +export const hasState = (state: IndexStates, ...values: IndexState[]) => Object.values(state).some((value) => value !== undefined && values.includes(value)); /** A crate in the repository, with the state of each index that isn't idle */ -export type RepositoryObject = { id: string; path: string; name?: string | undefined; state: Partial> }; +export type RepositoryObject = { id: string; path: string; name?: string | undefined; state: IndexStates }; /** * Abstraction for repository operations, such as indexing and listing objects. This is used by the API routes to interact with the underlying data store, which can be an OCFL storage or a database. */ export interface Repository { init(): Promise; - getState(crateId: string, type?: string): Promise>; - createIndex(crateIds?: string | string[] | RegExp, type?: string, force?: boolean): Promise; - deleteIndex(crateId?: string, type?: string | string[]): Promise; + getState(crateId: string, type?: IndexType): Promise; + createIndex(crateIds?: string | string[] | RegExp, type?: IndexType | IndexType[], force?: boolean): Promise; + deleteIndex(crateId?: string, type?: IndexType | IndexType[]): Promise; objects(prefix?: string, refresh?: boolean): AsyncIterable; storage(): StorageBackend; } diff --git a/src/utils.ts b/src/utils.ts index bc1b51c..753ae31 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,6 @@ -export function firstStringOrId(values: unknown[]): string | undefined { - for (const value of values || []) { +/** The first string or `@id` of a JSON-LD value, which may be a single value or an array */ +export function firstStringOrId(values: unknown): string | undefined { + for (const value of [values ?? []].flat()) { if (typeof value === 'string') { return value; } else if (typeof value === 'object' && value !== null && '@id' in value && typeof value['@id'] === 'string') { From 7625dcdc2340a75906db4a2ddae7f7547c6d5696 Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 15:24:06 +1100 Subject: [PATCH 16/20] fix: count and remove a crate's search documents in batches of 10,000 One request for all of a crate's ids fails past OpenSearch's default max_terms_count of 65,536, which broke the admin listing. --- src/indexer/search.test.ts | 14 ++++++++++++-- src/indexer/search.ts | 38 +++++++++++++++++++++++--------------- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/src/indexer/search.test.ts b/src/indexer/search.test.ts index 80510f4..6bae526 100644 --- a/src/indexer/search.test.ts +++ b/src/indexer/search.test.ts @@ -1,7 +1,7 @@ import { ROCrate } from 'ro-crate'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { type CrateObject, Indexer } from './indexer.ts'; -import { searchDocument } from './search.ts'; +import { countDocuments, searchDocument } from './search.ts'; const ROOT = 'arcp://name,test'; const LICENSE = 'https://creativecommons.org/licenses/by/4.0/'; @@ -88,3 +88,13 @@ describe('searchDocument', () => { expect(doc.dateCreated).toEqual(['1901-02-03']); }); }); + +describe('countDocuments', () => { + it('counts in batches under OpenSearch’s terms limit, summing them', async () => { + const search = vi.fn(async ({ visibility }: { visibility: { id: { in: string[] } } }) => ({ total: visibility.id.in.length, hits: [] })); + const ids = Array.from({ length: 25_000 }, (_, i) => `id-${i}`); + + expect(await countDocuments({ search } as unknown as Parameters[0], ids)).toBe(25_000); + expect(search.mock.calls.map(([{ visibility }]) => visibility.id.in.length)).toEqual([10_000, 10_000, 5_000]); + }); +}); diff --git a/src/indexer/search.ts b/src/indexer/search.ts index fb6c2ae..5783e6e 100644 --- a/src/indexer/search.ts +++ b/src/indexer/search.ts @@ -39,23 +39,29 @@ export const createSearchBackend = (client: Client, options: { refresh?: boolean type SearchBackend = ReturnType; +/** Well under OpenSearch's default `index.max_terms_count` of 65,536 */ +const BATCH_SIZE = 10_000; + +const batches = (items: T[]) => Array.from({ length: Math.ceil(items.length / BATCH_SIZE) }, (_, i) => items.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE)); + /** How many of `ids` have a search document, 0 if there is no index */ export async function countDocuments(search: SearchBackend, ids: string[]) { - if (!ids.length) { - return 0; - } try { - const { total } = await search.search({ - searchType: 'basic', - query: '', - filters: { terms: {}, ranges: {} }, - limit: 1, - offset: 0, - sort: 'relevance', - order: 'asc', - visibility: { id: { in: ids } }, - }); - return total; + let count = 0; + for (const batch of batches(ids)) { + const { total } = await search.search({ + searchType: 'basic', + query: '', + filters: { terms: {}, ranges: {} }, + limit: 1, + offset: 0, + sort: 'relevance', + order: 'asc', + visibility: { id: { in: batch } }, + }); + count += total; + } + return count; } catch (error) { if ((error as { statusCode?: number }).statusCode === 404) { return 0; @@ -210,7 +216,9 @@ export class SearchIndexer extends Indexer { async delete(crateId?: string) { if (crateId) { - await this.search.remove(await rowIds(crateId)); + for (const batch of batches(await rowIds(crateId))) { + await this.search.remove(batch); + } } else { await this.client.indices.delete({ index: this.indexName, ignore_unavailable: true }); } From b80c361b7c2f3e46b07c1437c34bf079e4704754 Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 15:24:20 +1100 Subject: [PATCH 17/20] fix: refresh the search index once after each crate's search delete and indexing Without the refresh, reading a crate's state straight after deleting its documents counted them again and cached it as indexed, and indexing finished before its documents were searchable. One refresh per crate, rather than waiting on every bulk request, as each text file is indexed in its own. Tests now use the backend as production does. --- src/admin/actions.test.ts | 14 +++++++++++- src/indexer/search.ts | 47 +++++++++++++++++++++++---------------- test/app.ts | 2 +- 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/src/admin/actions.test.ts b/src/admin/actions.test.ts index fb77e15..4e9308b 100644 --- a/src/admin/actions.test.ts +++ b/src/admin/actions.test.ts @@ -4,7 +4,7 @@ import { type buildTestApp, ocflRepository, ocflStorage, type TestApp, type Test import type { Browser } from '../../test/oidc.ts'; import type { rowIds as crateRowIds } from '../indexer/database.ts'; import type { countDocuments } from '../indexer/search.ts'; -import { hasState, type Repository, State } from '../repository.ts'; +import { hasState, type IndexType, type Repository, State } from '../repository.ts'; const BUNDLED = 'arcp://name,bundled'; const EXTRA = `${PEOPLE}/extra`; @@ -240,6 +240,18 @@ describe('index and delete actions', () => { expect(await repository.getState(PEOPLE)).toEqual({ database: 'indexed', search: 'indexed' }); }); + it.each<[string, IndexType[] | undefined]>([ + ['search', ['search']], + ['database and search', undefined], + ])('reads a crate as not indexed as soon as its %s indexes are deleted', async (_, type) => { + await repository.createIndex(BUNDLED); + expect(await repository.getState(BUNDLED)).toEqual({ database: 'indexed', search: 'indexed' }); + + await repository.deleteIndex(BUNDLED, type); + + expect(hasState(await repository.getState(BUNDLED, 'search'), State.INDEXED)).toBe(false); + }); + it('deletes every index', async () => { const browser = await signIn(app); await act(browser, 'index'); diff --git a/src/indexer/search.ts b/src/indexer/search.ts index 5783e6e..23ac827 100644 --- a/src/indexer/search.ts +++ b/src/indexer/search.ts @@ -34,8 +34,7 @@ const fields = { type Document = SearchDocument; -export const createSearchBackend = (client: Client, options: { refresh?: boolean | 'wait_for' } = {}) => - OpenSearchBackend({ client, fields, indexName: config.search.entityIndex, ...options }); +export const createSearchBackend = (client: Client) => OpenSearchBackend({ client, fields, indexName: config.search.entityIndex }); type SearchBackend = ReturnType; @@ -219,6 +218,7 @@ export class SearchIndexer extends Indexer { for (const batch of batches(await rowIds(crateId))) { await this.search.remove(batch); } + await this.refresh(); } else { await this.client.indices.delete({ index: this.indexName, ignore_unavailable: true }); } @@ -229,31 +229,40 @@ export class SearchIndexer extends Indexer { return await countDocuments(this.search, await rowIds(crateId)); } + /** Makes changes searchable now, rather than at OpenSearch's next periodic refresh */ + async refresh() { + await this.client.indices.refresh({ index: this.indexName, ignore_unavailable: true }); + } + /** Only entities with a row, as arocapi drops a hit without one but still counts it */ async _index({ crateObject, crate, paths, license }: Parameters[0]) { await this.search.createIndex(); const crateId = crate.rootId; const rows = new Map((await crateRows(crateId)).map(({ id, rootCollection }) => [id, rootCollection])); const docs: Document[] = []; - for (const entity of crate.entities()) { - if (!recordType(entity)) { - continue; - } - const entityId = entity['@id']; - const rootCollection = rows.get(entityId); - if (rootCollection === undefined) { - log.warn({ crateId, entityId }, 'Skipped: no database row'); - continue; - } - const doc = await searchDocument(crate, crateObject, entity, license, rootCollection, paths.get(entityId)); - if (doc.text) { - // Indexed alone to keep each request small - await this.indexDocuments([doc]); - } else { - docs.push(doc); + try { + for (const entity of crate.entities()) { + if (!recordType(entity)) { + continue; + } + const entityId = entity['@id']; + const rootCollection = rows.get(entityId); + if (rootCollection === undefined) { + log.warn({ crateId, entityId }, 'Skipped: no database row'); + continue; + } + const doc = await searchDocument(crate, crateObject, entity, license, rootCollection, paths.get(entityId)); + if (doc.text) { + // Indexed alone to keep each request small + await this.indexDocuments([doc]); + } else { + docs.push(doc); + } } + await this.indexDocuments(docs); + } finally { + await this.refresh(); } - await this.indexDocuments(docs); } /** Index a batch, falling back to one at a time so one bad document doesn't lose the rest */ diff --git a/test/app.ts b/test/app.ts index 3a94c57..8fc3b14 100644 --- a/test/app.ts +++ b/test/app.ts @@ -43,7 +43,7 @@ export const buildTestApp = async (env: Record, options: { repos const { Client } = await import('@opensearch-project/opensearch'); const { createSearchBackend } = await import('../src/indexer/search.ts'); const opensearchClient = new Client({ node: config.opensearchUrl }); - const search = createSearchBackend(opensearchClient, { refresh: true }); + const search = createSearchBackend(opensearchClient); const repo = typeof options.repository === 'function' ? await options.repository({ opensearchClient, search }) : (options.repository ?? repository); /** Builds another app, as another server process would */ const build = () => buildApp({ repository: repo, search }); From 69ec0f25d1e3cba6e8c4b7117e1edbe9aa8302b3 Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 15:24:26 +1100 Subject: [PATCH 18/20] fix: label https://schema.org classes as their http://schema.org ones --- src/indexer/search.test.ts | 6 ++++++ src/indexer/search.ts | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/indexer/search.test.ts b/src/indexer/search.test.ts index 6bae526..45c6a71 100644 --- a/src/indexer/search.test.ts +++ b/src/indexer/search.test.ts @@ -62,6 +62,12 @@ describe('searchDocument', () => { expect((await searchDocument(crate, crateObject, entity, LICENSE, ROOT)).recordType).toBe(label); }); + it.each(['http://schema.org/MediaObject', 'https://schema.org/MediaObject'])('labels %s as File, whichever the scheme', async (uri) => { + const { crate, entity } = await crateWith({}, { File: uri }); + + expect(await searchDocument(crate, crateObject, entity, LICENSE, ROOT)).toMatchObject({ entityType: uri, recordType: 'File' }); + }); + it.each([ ['http://schema.org/Thing', 'Thing'], ['https://example.com/terms#Widget', 'Widget'], diff --git a/src/indexer/search.ts b/src/indexer/search.ts index 23ac827..b15a3ce 100644 --- a/src/indexer/search.ts +++ b/src/indexer/search.ts @@ -83,8 +83,8 @@ const CLASS_LABELS: Record = { 'http://schema.org/SoftwareApplication': 'Software Application', }; -/** A readable label for a class URI, falling back to its last segment */ -const classLabel = (uri: string) => CLASS_LABELS[uri] ?? uri.split(/[#/]/).at(-1) ?? uri; +/** A readable label for a class URI, either schema.org scheme alike, falling back to its last segment */ +const classLabel = (uri: string) => CLASS_LABELS[uri.replace(/^https:\/\/schema\.org\//, 'http://schema.org/')] ?? uri.split(/[#/]/).at(-1) ?? uri; const values = (entity: Entity, name: string): unknown[] => [...(entity[name] ?? []), ...(entity[`ldac:${name}`] ?? [])]; From 41d8991c059f753b5ffc505f9a055f0120657b3b Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 15:24:38 +1100 Subject: [PATCH 19/20] fix: log each crate's indexed search documents, and its skip once when it has no rows A crate whose database indexing failed logged a warning for every entity. --- src/indexer/search.ts | 29 +++++++++++++++++++++-------- src/search-total.test.ts | 16 ++++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/indexer/search.ts b/src/indexer/search.ts index b15a3ce..4982464 100644 --- a/src/indexer/search.ts +++ b/src/indexer/search.ts @@ -239,7 +239,9 @@ export class SearchIndexer extends Indexer { await this.search.createIndex(); const crateId = crate.rootId; const rows = new Map((await crateRows(crateId)).map(({ id, rootCollection }) => [id, rootCollection])); + const unrowed: string[] = []; const docs: Document[] = []; + let count = 0; try { for (const entity of crate.entities()) { if (!recordType(entity)) { @@ -248,38 +250,49 @@ export class SearchIndexer extends Indexer { const entityId = entity['@id']; const rootCollection = rows.get(entityId); if (rootCollection === undefined) { - log.warn({ crateId, entityId }, 'Skipped: no database row'); + unrowed.push(entityId); continue; } const doc = await searchDocument(crate, crateObject, entity, license, rootCollection, paths.get(entityId)); if (doc.text) { // Indexed alone to keep each request small - await this.indexDocuments([doc]); + count += await this.indexDocuments([doc]); } else { docs.push(doc); } } - await this.indexDocuments(docs); + count += await this.indexDocuments(docs); } finally { await this.refresh(); } + if (rows.size === 0 && unrowed.length > 0) { + log.warn({ crateId, skipped: unrowed.length }, 'Skipped crate: no database rows'); + } else { + for (const entityId of unrowed) { + log.warn({ crateId, entityId }, 'Skipped: no database row'); + } + } + log.info({ crateId, documents: count }, 'Indexed crate'); } - /** Index a batch, falling back to one at a time so one bad document doesn't lose the rest */ - async indexDocuments(docs: Document[]) { + /** Index a batch, falling back to one at a time so one bad document doesn't lose the rest; returns how many were indexed */ + async indexDocuments(docs: Document[]): Promise { if (!docs.length) { - return; + return 0; } try { await this.search.index(docs); + return docs.length; } catch (error) { if (docs.length === 1) { log.error({ err: error, entityId: docs[0]?.id }, 'Error indexing document'); - return; + return 0; } + let count = 0; for (const doc of docs) { - await this.indexDocuments([doc]); + count += await this.indexDocuments([doc]); } + return count; } } } diff --git a/src/search-total.test.ts b/src/search-total.test.ts index 4f802f3..c1317c9 100644 --- a/src/search-total.test.ts +++ b/src/search-total.test.ts @@ -66,4 +66,20 @@ describe('an entity whose row fails to be written', () => { it('logs the skip with the crate and entity ids', () => { expect(logs).toContainEqual(expect.objectContaining({ level: 40, crateId: ROOT, entityId: UNWRITTEN, msg: 'Skipped: no database row' })); }); + + it('logs the documents it indexed', () => { + expect(logs).toContainEqual(expect.objectContaining({ module: 'indexer/search', crateId: ROOT, documents: 2, msg: 'Indexed crate' })); + }); +}); + +describe('a crate with no database rows', () => { + it('logs the skip once for the crate', async () => { + await testbed.started.repository.deleteIndex(ROOT); + logs.length = 0; + + await testbed.started.repository.createIndex(ROOT, 'search'); + + const skips = logs.filter(({ msg }) => typeof msg === 'string' && msg.startsWith('Skipped')); + expect(skips).toEqual([expect.objectContaining({ level: 40, crateId: ROOT, skipped: 3, msg: 'Skipped crate: no database rows' })]); + }); }); From a08be87d6ba6f7755a7a1697a93a7d3521696b72 Mon Sep 17 00:00:00 2001 From: John Ferlito Date: Wed, 4 Nov 2026 15:24:38 +1100 Subject: [PATCH 20/20] fix: warn when collections are members of each other, naming them --- src/collection-cycle.test.ts | 65 ++++++++++++++++++++++++++++++++++++ src/ocfl.ts | 5 ++- 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 src/collection-cycle.test.ts diff --git a/src/collection-cycle.test.ts b/src/collection-cycle.test.ts new file mode 100644 index 0000000..5c75f4a --- /dev/null +++ b/src/collection-cycle.test.ts @@ -0,0 +1,65 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { beforeAll, describe, expect, it, vi } from 'vitest'; +import { ocflTestbed } from '../test/app.ts'; + +const logs = vi.hoisted(() => [] as Record[]); + +vi.mock('./logger.ts', async () => (await import('../test/logs.ts')).capturingLogger(logs, 'warn')); + +const FIRST = 'arcp://name,cycle_first'; +const SECOND = 'arcp://name,cycle_second'; +const LICENSE = { '@id': 'https://creativecommons.org/licenses/by/4.0/' }; + +const collection = (id: string, parent: string) => ({ + '@context': ['https://w3id.org/ro/crate/1.1/context', 'https://w3id.org/ldac/context', { '@vocab': 'http://schema.org/' }], + '@graph': [ + { '@id': 'ro-crate-metadata.json', '@type': 'CreativeWork', license: LICENSE, about: { '@id': id } }, + { + '@id': id, + '@type': ['Dataset', 'RepositoryCollection'], + conformsTo: { '@id': 'https://w3id.org/ldac/profile#Collection' }, + memberOf: { '@id': parent }, + name: id, + license: LICENSE, + }, + ], +}); + +const testbed = ocflTestbed({ + source: async (scratch) => { + const source = join(scratch, 'crates'); + for (const [name, id, parent] of [ + ['first', FIRST, SECOND], + ['second', SECOND, FIRST], + ] as const) { + await mkdir(join(source, name), { recursive: true }); + await writeFile(join(source, name, 'ro-crate-metadata.json'), JSON.stringify(collection(id, parent))); + } + return source; + }, +}); + +beforeAll(async () => { + await testbed.started.repository.deleteIndex(); + await testbed.started.repository.createIndex(FIRST); +}); + +describe('collections that are members of each other', () => { + it('logs the cycle, naming the crates', () => { + expect(logs).toContainEqual( + expect.objectContaining({ + level: 40, + crateId: FIRST, + cycle: [FIRST, SECOND, FIRST], + msg: 'Collections are members of each other; using the highest found', + }), + ); + }); + + it('still indexes the crate under the highest collection found', async () => { + const row = await testbed.started.prisma.entity.findUnique({ where: { id: FIRST }, select: { rootCollection: true } }); + + expect(row?.rootCollection).toBe(SECOND); + }); +}); diff --git a/src/ocfl.ts b/src/ocfl.ts index 234d49a..a19bf45 100644 --- a/src/ocfl.ts +++ b/src/ocfl.ts @@ -203,9 +203,12 @@ function topCollectionFinder() { parent = next; } catch (error) { log.warn({ err: error, crateId: crate.rootId, parent, topCollection: top }, 'Cannot read parent collection; using the highest found'); - break; + return top; } } + if (parent) { + log.warn({ crateId: crate.rootId, cycle: [...seen, parent], topCollection: top }, 'Collections are members of each other; using the highest found'); + } return top; }; }