diff --git a/CLAUDE.md b/CLAUDE.md index c280da8..c12c319 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,3 +17,7 @@ Single-context: `GLOSSARY.md` and `docs/adr/` at the repo root. See `docs/agents ## Database migrations A migration that creates or changes arocapi's models (`prisma/arocapi`, symlinked from the arocapi package) is named after the arocapi version: after an arocapi upgrade that changes the models, run `pnpm db:migrate --name arocapi-`. Name migrations for our own models (`prisma/models`) after what they change. + +## Upgrading arocapi + +After any arocapi upgrade, recheck the README's links to arocapi's docs (arocapi.crate-works.org is unversioned): each page and anchor should still exist and still say what the README relies on. diff --git a/GLOSSARY.md b/GLOSSARY.md index fbdd17c..16fa95a 100644 --- a/GLOSSARY.md +++ b/GLOSSARY.md @@ -4,6 +4,30 @@ Serves RO-Crates from OCFL storage through the arocapi API, and controls access ## Language +arocapi's terms, such as **Record**, **Metadata**, **Content** and **Row**, mean what [arocapi's glossary](https://github.com/crate-works/arocapi/blob/main/GLOSSARY.md) says. + +### Crates and indexes + +**Crate**: +One RO-Crate, stored as one OCFL object in the repository. The unit of import, indexing and admin actions. +_Avoid_: OCFL object (the storage), collection (what a crate may describe) + +**Import**: +Bringing RO-Crates into the OCFL repository: `pnpm crates:import` today, the deposit API in future. +_Avoid_: Seed + +**Index**: +Building the database index and the search index from the repository, for one crate or all of them. Both can be deleted and rebuilt at any time. +_Avoid_: Import + +**Database index**: +The Entity and File rows that arocapi reads, written by the indexer (arocapi's Rows). +_Avoid_: Structural index + +**Search index**: +The search documents in arocapi's search backend, one per entity that has a row. +_Avoid_: OpenSearch index (one backend's storage) + ### Authentication **Upstream provider**: diff --git a/README.md b/README.md index 2a000ef..d551be0 100644 --- a/README.md +++ b/README.md @@ -7,26 +7,59 @@ RO-Crate API. It serves RO-Crates over the ## How it fits together -The core API (entities, files, search, RO-Crate metadata) comes from -[arocapi](https://github.com/crate-works/arocapi), which this -app mounts as a Fastify plugin. ldacapi adds what LDaCA needs on top: - -- **OCFL storage**: crates live in an [OCFL](https://ocfl.io/) repository on - disk (`src/ocfl.ts`). -- **Indexing**: crates are read from OCFL into the database (the structural - index, used by arocapi's entity and file routes) and, as one search - document per entity, into arocapi's OpenSearch search backend - (`src/indexer/`). The search fields are declared in `src/indexer/search.ts`. +The API's routes (entities, files, search, RO-Crate metadata) come from +[arocapi](https://github.com/crate-works/arocapi), which ldacapi mounts as a +Fastify plugin under `/api` (`src/app.ts`). Its +[docs](https://arocapi.crate-works.org/) explain each part; ldacapi +configures them as follows: + +- **Storage backend, Content half**: arocapi's + [`OcflStorage`](https://arocapi.crate-works.org/configuration/storage/#ocflstorage) + serves crate files from the OCFL repository. +- **Storage backend, Metadata half**: arocapi's + [`EntityMetaMetadata`](https://arocapi.crate-works.org/configuration/storage/#entitymetametadata) + serves each entity's virtual crate, built at index time and held in its + database row ([ADR 0004](docs/adr/0004-entity-metadata-is-a-virtual-crate.md)). +- **Search backend**: arocapi's + [`OpenSearchBackend`](https://arocapi.crate-works.org/configuration/search/#opensearchbackend), + with the [fields](https://arocapi.crate-works.org/configuration/search/#fields) + declared in `src/indexer/search.ts`, where `entityType` filters on an + entity's class URI and the "Record Type" facet, `recordType`, shows a + readable label for it. +- **Visibility**: every entity is visible to everyone, through arocapi's + all-visible [visibility filters](https://arocapi.crate-works.org/configuration/visibility/) + ([ADR 0003](docs/adr/0003-every-entity-is-visible.md)). +- **Access**: arocapi's + [`LicenceAccess`](https://arocapi.crate-works.org/configuration/access/#access), + with the licences a user holds looked up in REMS + ([licence checks against an external authority](https://arocapi.crate-works.org/strategies/access/#licence-checks-against-an-external-authority)); + see [Licences](#licences) (`src/auth.ts`). + +ldacapi adds what LDaCA needs on top (terms as in [GLOSSARY.md](GLOSSARY.md)): + +- **OCFL repository**: each crate is one [OCFL](https://ocfl.io/) object on + disk, imported with `pnpm crates:import` (`src/ocfl.ts`). +- **Indexing**: `pnpm index` and the admin pages build the database index (the + Entity and File rows arocapi reads) and the search index (one search + document per entity) from the repository (`src/indexer/`). Both can be + rebuilt at any time. +- **Collections**: an entity's `rootCollection` is the top-level collection + it sits under, and is empty on a top-level collection + ([membership](https://arocapi.crate-works.org/concepts/entities/#membership)). + Indexing finds it by following `memberOf` up through the parent crates in + the repository, so a crate in a distributed collection gets the + distributed collection's root whatever order the crates are indexed in. If + a parent collection moves, reindex the crates under it: they keep the old + `rootCollection` until then. +- **Sign-in**: the API is its own OIDC provider, signing users in at an + upstream provider such as CILogon (`src/oidc/`, + [ADR 0001](docs/adr/0001-own-oidc-provider.md)). - **Admin pages**: server-rendered pages at `/admin/` on their own origin, where admins, who sign in through our own OIDC provider, see and rebuild or - delete indexes (`src/admin/`). -- **Sign-in**: the API is its own OIDC provider, signing users in at an - upstream provider such as CILogon (`src/oidc/`). -- **Access control**: every entity is visible, and what a request may read is - decided by licence, checked against REMS (`src/auth.ts`). -- **Files and metadata**: arocapi serves crate files from the OCFL repository - through its OCFL storage backend, and each entity's RO-Crate metadata from - the structural index (`src/app.ts`). + delete indexes (`src/admin/`, + [ADR 0002](docs/adr/0002-server-rendered-admin-pages.md)). + +The ADRs linked here are ldacapi's own, in `docs/adr/`, not arocapi's. ## Getting started @@ -35,7 +68,7 @@ You need Node.js 26 (26.13 or later), [pnpm](https://pnpm.io/) and Docker. ```sh pnpm install docker compose up -d --wait # Postgres, OpenSearch, Oni and a fake upstream provider -pnpm bootstrap # migrate the database, seed OCFL from test-data, index +pnpm bootstrap # migrate the database, import test-data into OCFL, index pnpm dev ``` @@ -54,25 +87,25 @@ override a default, set it in `.env`. ## Scripts -| Script | Does | -| -------------------- | -------------------------------------------------------------------------------- | -| `pnpm dev` | Run the API, restarting on changes, and rebuild the admin pages' stylesheet | -| `pnpm build` | Build the admin pages' stylesheet (run before deploying) | -| `pnpm test` | Run the tests (needs `docker compose up`) | -| `pnpm lint` | Biome, Knip and TypeScript | -| `pnpm bootstrap` | `db:deploy`, `seed` and `index` in one go | -| `pnpm ocfl:init` | Create an empty OCFL repository at `OCFL_ROOT`, unless one exists already | -| `pnpm seed [dir]` | Import each crate folder in `dir` (default `test-data`) into the OCFL repository | -| `pnpm index` | Rebuild both indexes from the OCFL repository | -| `pnpm generate-keys` | Print new `OIDC_SIGNING_KEYS` and `OIDC_COOKIE_KEYS` for `.env` | -| `pnpm db:migrate` | Create and apply a migration after a schema change | -| `pnpm db:deploy` | Apply pending migrations | -| `pnpm db:reset` | Drop and recreate the dev database | -| `pnpm db:console` | Open `psql` on the dev database | +| Script | Does | +| -------------------------- | -------------------------------------------------------------------------------- | +| `pnpm dev` | Run the API, restarting on changes, and rebuild the admin pages' stylesheet | +| `pnpm build` | Build the admin pages' stylesheet (run before deploying) | +| `pnpm test` | Run the tests (needs `docker compose up`) | +| `pnpm lint` | Biome, Knip and TypeScript | +| `pnpm bootstrap` | `db:deploy`, `crates:import` and `index` in one go | +| `pnpm ocfl:init` | Create an empty OCFL repository at `OCFL_ROOT`, unless one exists already | +| `pnpm crates:import [dir]` | Import each crate folder in `dir` (default `test-data`) into the OCFL repository | +| `pnpm index` | Rebuild both indexes from the OCFL repository | +| `pnpm generate-keys` | Print new `OIDC_SIGNING_KEYS` and `OIDC_COOKIE_KEYS` for `.env` | +| `pnpm db:migrate` | Create and apply a migration after a schema change | +| `pnpm db:deploy` | Apply pending migrations | +| `pnpm db:reset` | Drop and recreate the dev database | +| `pnpm db:console` | Open `psql` on the dev database | ## Test data -Each folder in `test-data/` is one RO-Crate. `pnpm seed` imports them into the +Each folder in `test-data/` is one RO-Crate. `pnpm crates:import` imports them into the OCFL repository at `storage/ocfl` (gitignored), and re-running it only adds a new OCFL version for crates that changed. Run `pnpm index` afterwards. @@ -83,7 +116,7 @@ dataset) unless `DEFAULT_METADATA_LICENSE` or `DEFAULT_LICENSE` is set. ## Configuration Configuration comes only from environment variables, validated at startup -(`src/configuration.ts`). `pnpm dev`, `pnpm seed`, `pnpm index` and Prisma +(`src/configuration.ts`). `pnpm dev`, `pnpm crates:import`, `pnpm index` and Prisma also read `.env` if it exists, without overriding variables that are already set. @@ -114,10 +147,28 @@ have no defaults and must be set. ### Licences -Content under a licence that isn't open needs an entitlement to it in REMS, -looked up at `REMS_ENDPOINT` with `REMS_USER` and `REMS_KEY`. A user without -one is offered an enrolment link: `ENROLLMENT_URL` with every `{licenseId}` -replaced by the URL-encoded licence id, e.g. +Every entity has a metadata licence, which guards its RO-Crate metadata +(`/entity/:id/metadata`), and a content licence, which guards a file's bytes +(`/file/:id`) +([two licences per entity](https://arocapi.crate-works.org/concepts/licences/#two-licences-per-entity)). +The indexer takes the metadata licence from the `license` of +`ro-crate-metadata.json`, and the content licence from the entity's own +`license` or else the root dataset's. + +A licence that isn't one of the open licences in `src/configuration.ts` needs +an entitlement to it in REMS, looked up at `REMS_ENDPOINT` with `REMS_USER` +and `REMS_KEY`, once per request. If REMS can't be reached or returns an +error, the API logs it and treats the user as holding no licences, so +restricted entities are refused until it recovers. + +A refused request gets 401 if it is anonymous, so the client can prompt a +sign-in, and 403 if it is signed in +([Denied is 401 or 403](https://arocapi.crate-works.org/concepts/visibility-vs-access/#denied-is-401-or-403-never-404)). +The enrolment link isn't on the error: every entity the API returns carries +an `access` object, and for each licence the user doesn't hold it has a +`metadataAuthorizationUrl` or `contentAuthorizationUrl`. That link is +`ENROLLMENT_URL` with every `{licenseId}` replaced by the URL-encoded licence +id, e.g. ```dotenv ENROLLMENT_URL=https://rems.example.org/apply-for?resource={licenseId} @@ -199,9 +250,11 @@ username to `ADMIN_USERS` in `.env`. Anyone else sees "Not an admin". The admin pages are at `/admin/`. `ADMIN_ORIGIN` is an origin, with no path, such as `https://admin.example.org`, and must differ from -`OIDC_ISSUER`: crate files are served on the API's origin with their own -content type, so HTML or SVG in a crate could run script there, and a separate -origin keeps that away from admin. It defaults to `http://admin.localhost:8080` +`OIDC_ISSUER`. Crate files are served on the API's origin with their own +content type. arocapi sandboxes them, except audio and video, but that rests +on each response's headers, so a separate origin keeps any script a crate file +does run away from admin (ldacapi's +[ADR 0002](docs/adr/0002-server-rendered-admin-pages.md)). It defaults to `http://admin.localhost:8080` outside production, which browsers resolve to this machine; in production it must be https. The admin pages are on only when sign-in is and `ADMIN_ORIGIN` is set; otherwise startup logs a warning and `/admin/` is not found. On the @@ -287,7 +340,7 @@ docker run --rm --env-file ldacapi.env -v /srv/ldacapi:/storage ldacapi pnpm db: ``` `pnpm ocfl:init` (which creates an empty OCFL repository; the API won't start -without one), `pnpm seed [dir]` (mount `dir` into the container) and +without one), `pnpm crates:import [dir]` (mount `dir` into the container) and `pnpm index` run the same way, offline. Give them and `pnpm db:deploy` the API's environment file: Prisma and the scripts load the whole [configuration](#configuration). `pnpm generate-keys` reads no configuration, so it needs no environment file: @@ -302,7 +355,7 @@ The image has no default `OCFL_ROOT`. Mount your storage and set `OCFL_ROOT` (and `OCFL_SCRATCH`, if you need it elsewhere) to paths inside the container (above, `OCFL_ROOT=/storage/ocfl`). The API runs as the image's `node` user, uid 1000, so the OCFL repository must be readable by uid -1000 and the scratch directory writable by it. `pnpm seed` writes to the +1000 and the scratch directory writable by it. `pnpm crates:import` writes to the repository, so give it write access too when you run it. ## Production with Docker Compose @@ -319,11 +372,15 @@ from Let's Encrypt on its own. It serves two hostnames: | `DOMAIN` | everything else | Oni | | `ADMIN_DOMAIN` | `/admin/*` | The API's admin pages | -The admin pages need their own hostname, so that script in a crate file can -never run on their origin (ADR 0002). Caddy also serves everything under -`/api/` with `Content-Security-Policy: sandbox`, which gives any HTML or SVG in -a crate file an opaque origin, so its scripts can't reach Oni's sign-in on -`DOMAIN`. +The admin pages need their own hostname (see [Admin pages](#admin-pages)). +Caddy sets no Content-Security-Policy on the API's responses: arocapi sends +crate files with `nosniff` and, except audio and video, a `sandbox` +Content-Security-Policy, which gives any HTML or SVG in a crate an opaque +origin, so its scripts can't reach Oni's sign-in on `DOMAIN` +([protective headers](https://arocapi.crate-works.org/configuration/storage/#protective-headers)). +Audio and video are left out so they play when opened directly. On both +hostnames Caddy adds HSTS, `nosniff` and a `strict-origin-when-cross-origin` +referrer policy, and Oni's pages also get `X-Frame-Options: SAMEORIGIN`. No other container publishes a port, on the host or on localhost, so Postgres, OpenSearch, the API and Oni are reachable only from within the stack. @@ -389,7 +446,7 @@ Import crates and index them with the API's image, mounting the directory of crates read-only: ```sh -docker compose run --rm -v /path/to/crates:/import:ro api pnpm seed /import +docker compose run --rm -v /path/to/crates:/import:ro api pnpm crates:import /import docker compose run --rm api pnpm index ``` @@ -404,6 +461,18 @@ docker compose run --rm api pnpm db:deploy docker compose up -d ``` +`pnpm index` deletes both indexes and rebuilds them, so run it when an upgrade +changes what they hold. If an upgrade changes the search fields, the API and +`pnpm index` stop with an error naming the changed field, as they check the +search index's mapping before anything else. Delete the search index (named by +`ENTITY_INDEX`, `entities` by default) first, then rebuild and start again: + +```sh +docker compose exec opensearch curl -sX DELETE localhost:9200/entities +docker compose run --rm api pnpm index +docker compose up -d +``` + To run a branch's image (see [Docker image](#docker-image)) on a dev deployment, change the API's `image` tag in `compose.yaml` from `main` to the branch's tag and follow the upgrade steps above. Migrations the branch adds diff --git a/deploy/Caddyfile b/deploy/Caddyfile index e64641c..3168413 100644 --- a/deploy/Caddyfile +++ b/deploy/Caddyfile @@ -25,14 +25,7 @@ file_server } - # Crate files are served inline with their own content type. The sandbox gives - # any HTML or SVG among them an opaque origin, so its scripts can't reach Oni's sign-in. - handle /api/* { - header Content-Security-Policy "sandbox allow-scripts allow-popups allow-downloads" - reverse_proxy api:8080 - } - - @provider path /api /oauth/* /.well-known/openid-configuration + @provider path /api /api/* /oauth/* /.well-known/openid-configuration handle @provider { reverse_proxy api:8080 } diff --git a/deploy/oni/configuration.json b/deploy/oni/configuration.json index 51fac8f..12c8146 100644 --- a/deploy/oni/configuration.json +++ b/deploy/oni/configuration.json @@ -325,7 +325,7 @@ }, { "display": "Record Type", - "name": "type" + "name": "recordType" }, { "display": "Language", diff --git a/docker/oni/configuration.json b/docker/oni/configuration.json index 85dea36..62644c1 100644 --- a/docker/oni/configuration.json +++ b/docker/oni/configuration.json @@ -325,7 +325,7 @@ }, { "display": "Record Type", - "name": "type" + "name": "recordType" }, { "display": "Language", diff --git a/docs/adr/0002-server-rendered-admin-pages.md b/docs/adr/0002-server-rendered-admin-pages.md index 2e55d01..16af409 100644 --- a/docs/adr/0002-server-rendered-admin-pages.md +++ b/docs/adr/0002-server-rendered-admin-pages.md @@ -2,11 +2,11 @@ ## Context -Admins manage the API's indexes. A separate browser app calling an admin API with one shared key exposes the key to anyone who loads the app, records nobody's identity, and can only take access away from everyone at once. The API is already an OIDC provider that signs users in at the upstream provider (ADR 0001), and it can reach the repository and indexer directly. It also serves crate files inline with their own content type and no sandbox, so HTML or SVG in a crate runs script on the API's origin. +Admins manage the API's indexes. A separate browser app calling an admin API with one shared key exposes the key to anyone who loads the app, records nobody's identity, and can only take access away from everyone at once. The API is already an OIDC provider that signs users in at the upstream provider (ADR 0001), and it can reach the repository and indexer directly. It also serves crate files inline, on the API's origin, with their own content type. arocapi sends them with `nosniff` and a `sandbox` Content-Security-Policy, which gives HTML or SVG in a crate an opaque origin, but leaves the CSP off audio and video so they play when opened directly. That protection rests on each response's headers and content type, and doesn't cover media. ## Decision -The API serves its own admin pages at `/admin/`, rendered on the server, in the API's process but on their own origin, `ADMIN_ORIGIN`, which must differ from the issuer's. Every admin route is constrained to the admin origin's host, and nothing else is served there, so script in a crate file runs on another origin than admin, with the browser's boundary between them. They are registered only when sign-in and `ADMIN_ORIGIN` are configured. They are the only way into admin: there is no admin API and no shared admin key, and the pages call the repository directly. +The API serves its own admin pages at `/admin/`, rendered on the server, in the API's process but on their own origin, `ADMIN_ORIGIN`, which must differ from the issuer's. Every admin route is constrained to the admin origin's host, and nothing else is served there, so any script a crate file does run is on another origin than admin, with the browser's boundary between them. They are registered only when sign-in and `ADMIN_ORIGIN` are configured. They are the only way into admin: there is no admin API and no shared admin key, and the pages call the repository directly. - The admin pages are a built-in confidential client of our provider, not listed in `OIDC_CLIENTS`. Its client secret is derived from `OIDC_COOKIE_KEYS` with HKDF, and it uses only the authorisation code grant, asking for `openid profile`. - Sign-in is a server-side authorisation code flow with PKCE, state and nonce. The browser goes through the provider as usual, but the token request reaches the provider in-process, with its metadata built from configuration rather than discovery, so the API never calls its own public URL. Only the ID token is kept. diff --git a/docs/adr/0004-entity-metadata-is-a-virtual-crate.md b/docs/adr/0004-entity-metadata-is-a-virtual-crate.md new file mode 100644 index 0000000..35e6a26 --- /dev/null +++ b/docs/adr/0004-entity-metadata-is-a-virtual-crate.md @@ -0,0 +1,17 @@ +# Each entity's metadata is a virtual crate built at index time + +## Context + +arocapi serves an entity's Metadata, its RO-Crate metadata document, at `/entity/:id/metadata`, through the Metadata half of the storage backend, after checking the entity's metadata licence. A crate's `ro-crate-metadata.json` describes many entities, each with its own licences, and a large collection's runs to many megabytes. Oni asks for the metadata of the entity it is showing, not of the whole crate. + +## Decision + +Each entity's Metadata is a virtual crate: a new RO-Crate with the entity as its root, plus every entity it links to, directly or through others. The root also gets the `Dataset` type, the crate root's `conformsTo` if it has none, and the crate's content licence as its `license` if it has none. The database indexer builds it when it indexes the crate and stores it in the entity's row as `meta.rocrate`, and arocapi's `EntityMetaMetadata` serves it from there. Nothing serves a crate's `ro-crate-metadata.json` as it is. + +## Consequences + +- A request for one entity's Metadata gets only that entity and what it links to, guarded by that entity's metadata licence, rather than every entity in its crate. +- Metadata is only as fresh as the last index run: importing a new version of a crate changes nothing that is served until the crate is reindexed. +- Serving Metadata reads only the database, never the OCFL repository. +- The virtual crate isn't a file in the crate, so it has no checksum or version of its own, and it differs from what the depositor wrote by the properties added to its root. +- Rows are larger, as each holds a copy of what its entity links to. diff --git a/package.json b/package.json index 83a550d..9e3f74a 100644 --- a/package.json +++ b/package.json @@ -31,9 +31,9 @@ "dev:api": "node --env-file-if-exists=.env --watch src/index.ts", "dev:css": "tailwindcss -i src/admin/admin.css -o src/admin/public/admin.css --watch=always", "ocfl:init": "node --env-file-if-exists=.env scripts/ocfl-init.ts", - "seed": "node --env-file-if-exists=.env scripts/seed.ts", + "crates:import": "node --env-file-if-exists=.env scripts/crates-import.ts", "index": "node --env-file-if-exists=.env scripts/index.ts", - "bootstrap": "pnpm db:deploy && pnpm seed && pnpm index", + "bootstrap": "pnpm db:deploy && pnpm crates:import && pnpm index", "db:migrate": "prisma migrate dev", "db:deploy": "prisma migrate deploy", "db:reset": "prisma migrate reset", diff --git a/scripts/seed.ts b/scripts/crates-import.ts similarity index 90% rename from scripts/seed.ts rename to scripts/crates-import.ts index 7455d48..1551e0b 100644 --- a/scripts/seed.ts +++ b/scripts/crates-import.ts @@ -16,4 +16,4 @@ for (const directory of directories.filter((entry) => entry.isDirectory())) { await repository.object(crate.rootId).import(path); logger.info({ crateId: crate.rootId, path }, 'Imported crate'); } -logger.info({ root: config.ocfl.root }, 'Seeded the OCFL repository'); +logger.info({ root: config.ocfl.root }, 'Imported crates into the OCFL repository'); diff --git a/scripts/index.ts b/scripts/index.ts index 4829d29..6ec9ab2 100644 --- a/scripts/index.ts +++ b/scripts/index.ts @@ -1,9 +1,11 @@ import { Client } from '@opensearch-project/opensearch'; import { config } from '../src/configuration.ts'; -import { createIndex, init } from '../src/ocfl.ts'; +import { createIndex, deleteIndex, init } from '../src/ocfl.ts'; import { prisma } from '../src/prisma.ts'; const opensearch = new Client({ node: config.opensearchUrl }); await init({ opensearchClient: opensearch }); -await createIndex(undefined, undefined, true); +// A full rebuild, as deleting by crate misses rows that don't record their crate +await deleteIndex(); +await createIndex(); await Promise.all([prisma.$disconnect(), opensearch.close()]); diff --git a/src/admin/actions.test.ts b/src/admin/actions.test.ts index ad88490..4e9308b 100644 --- a/src/admin/actions.test.ts +++ b/src/admin/actions.test.ts @@ -1,23 +1,22 @@ -import { execFileSync } from 'node:child_process'; -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { Client } from '@opensearch-project/opensearch'; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; -import { ALICE, adminTestbed, ocflRepository, PEOPLE, repositoryIn, repositoryOf } from '../../test/admin.ts'; -import type { buildTestApp, TestApp, TestRepository } from '../../test/app.ts'; +import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { ALICE, adminTestbed, PEOPLE, repositoryIn, repositoryOf } from '../../test/admin.ts'; +import { type buildTestApp, ocflRepository, ocflStorage, type TestApp, type TestRepository } from '../../test/app.ts'; import type { Browser } from '../../test/oidc.ts'; -import { hasState, type Repository, State } from '../repository.ts'; +import type { rowIds as crateRowIds } from '../indexer/database.ts'; +import type { countDocuments } from '../indexer/search.ts'; +import { hasState, type IndexType, type Repository, State } from '../repository.ts'; const BUNDLED = 'arcp://name,bundled'; const EXTRA = `${PEOPLE}/extra`; const PEOPLE_TOO = `${PEOPLE}2`; +const DISTRIBUTED_ROOT = 'arcp://name,distributed_root'; +const CONTENT_RESTRICTED = 'arcp://name,distributed_content_restricted'; +const METADATA_RESTRICTED = 'arcp://name,distributed_metadata_restricted'; const testbed = adminTestbed(); -let storage: string; -let opensearch: Client; +const { ocflRoot } = ocflStorage(); -const start = (repository: TestRepository) => testbed.start({ OCFL_ROOT: join(storage, 'ocfl'), ADMIN_USERS: ALICE.sub }, { repository }); +const start = (repository: TestRepository) => testbed.start({ OCFL_ROOT: ocflRoot(), ADMIN_USERS: ALICE.sub }, { repository }); const signIn = async (app: TestApp) => { const { browser, page } = await testbed.signInAsAdmin(app, ALICE); @@ -40,29 +39,20 @@ const flash = (body: string) => .replace(/\s+/g, ' ') .trim(); -beforeAll(async () => { - storage = await mkdtemp(join(tmpdir(), 'ldacapi-actions-')); - execFileSync('node', ['scripts/seed.ts'], { env: { ...process.env, OCFL_ROOT: join(storage, 'ocfl') }, stdio: 'ignore' }); -}); - -afterAll(async () => { - await opensearch?.close(); - await rm(storage, { recursive: true, force: true }); -}); - describe('index and delete actions', () => { let app: TestApp; let repository: Repository; let prisma: Awaited>['prisma']; - let entityIndex: string; + let search: Awaited>['search']; + let rowIdsOfCrate: typeof crateRowIds; + let countSearchDocuments: typeof countDocuments; - const entityCount = (crateId: string) => prisma.entity.count({ where: { rootCollection: crateId } }); + /** The ids of the entity rows indexed from `crateIds`, sorted */ + const rowIds = async (...crateIds: string[]) => (await Promise.all(crateIds.map(rowIdsOfCrate))).flat().sort(); - const searchCount = async (crateId: string) => { - await opensearch.indices.refresh({ index: entityIndex, ignore_unavailable: true }); - const { body } = await opensearch.count({ index: entityIndex, ignore_unavailable: true, body: { query: { term: { rootCollection: crateId } } } }); - return body.count; - }; + const entityCount = async (crateId: string) => (await rowIds(crateId)).length; + + const searchCount = (ids: string[]) => countSearchDocuments(search, ids); /** Waits until no crate is queued, indexing or deleting, so an earlier action can't leak into the next */ const waitUntilSettled = () => @@ -77,15 +67,15 @@ describe('index and delete actions', () => { const waitUntilIndexed = async (...crateIds: string[]) => { await waitUntilSettled(); for (const crateId of crateIds) { - expect(await repository.getState(crateId)).toEqual({ structural: 'indexed', search: 'indexed' }); + expect(await repository.getState(crateId)).toEqual({ database: 'indexed', search: 'indexed' }); } }; beforeAll(async () => { - let config: Awaited>['config']; - ({ app, repository, prisma, config } = await start(ocflRepository)); - entityIndex = config.search.entityIndex; - opensearch = new Client({ node: config.opensearchUrl }); + ({ app, repository, prisma, search } = await start(ocflRepository)); + // Imported once the app has read its configuration + ({ rowIds: rowIdsOfCrate } = await import('../indexer/database.ts')); + ({ countDocuments: countSearchDocuments } = await import('../indexer/search.ts')); }); beforeEach(async () => { @@ -101,10 +91,21 @@ describe('index and delete actions', () => { expect(flash(page.body)).toBe(`Indexing ${BUNDLED} has been queued.`); await waitUntilIndexed(BUNDLED); - expect(await entityCount(BUNDLED)).toBeGreaterThan(0); - expect(await searchCount(BUNDLED)).toBeGreaterThan(0); + const ids = await rowIds(BUNDLED); + expect(ids).toContain(BUNDLED); + expect(await searchCount(ids)).toBe(ids.length); expect(await entityCount(PEOPLE)).toBe(0); - expect(await searchCount(PEOPLE)).toBe(0); + }); + + it('records the crate each entity row came from', async () => { + const browser = await signIn(app); + + await act(browser, 'index'); + + await waitUntilIndexed(BUNDLED, PEOPLE, DISTRIBUTED_ROOT, CONTENT_RESTRICTED, METADATA_RESTRICTED); + const counts = await Promise.all([BUNDLED, PEOPLE, DISTRIBUTED_ROOT, CONTENT_RESTRICTED, METADATA_RESTRICTED].map(entityCount)); + expect(counts.every((count) => count > 0)).toBe(true); + expect(counts.reduce((sum, count) => sum + count)).toBe(await prisma.entity.count()); }); it('refuses to index an indexed crate, asking for a reindex', async () => { @@ -121,15 +122,33 @@ describe('index and delete actions', () => { const browser = await signIn(app); await act(browser, 'index', { crateId: BUNDLED }); await waitUntilIndexed(BUNDLED); - const entities = await entityCount(BUNDLED); - const documents = await searchCount(BUNDLED); + const ids = await rowIds(BUNDLED); const page = await act(browser, 'index', { crateId: BUNDLED, force: 'true' }); expect(flash(page.body)).toBe(`Reindexing ${BUNDLED} has been queued.`); await waitUntilIndexed(BUNDLED); - expect(await entityCount(BUNDLED)).toBe(entities); - expect(await searchCount(BUNDLED)).toBe(documents); + expect(await rowIds(BUNDLED)).toEqual(ids); + expect(await searchCount(ids)).toBe(ids.length); + }); + + it.each([ + ['reindexes', 'index', { force: 'true' }, true], + ['deletes', 'delete', {}, false], + ])('%s one crate of a distributed collection, leaving the rest of it alone', async (_, action, form, kept) => { + const browser = await signIn(app); + await act(browser, 'index'); + await waitUntilIndexed(DISTRIBUTED_ROOT, CONTENT_RESTRICTED, METADATA_RESTRICTED); + const ids = await rowIds(CONTENT_RESTRICTED); + const siblings = await rowIds(DISTRIBUTED_ROOT, METADATA_RESTRICTED); + + await act(browser, action, { crateId: CONTENT_RESTRICTED, ...form }); + + await waitUntilSettled(); + expect(await rowIds(CONTENT_RESTRICTED)).toEqual(kept ? ids : []); + expect(await searchCount(ids)).toBe(kept ? ids.length : 0); + expect(await rowIds(DISTRIBUTED_ROOT, METADATA_RESTRICTED)).toEqual(siblings); + expect(await searchCount(siblings)).toBe(siblings.length); }); it('indexes every crate', async () => { @@ -178,38 +197,73 @@ describe('index and delete actions', () => { const page = await act(browser, 'index'); expect(page.body).toMatch(/ { + it('deletes one crate’s indexes, including its people and organisations', async () => { const browser = await signIn(app); await act(browser, 'index'); await waitUntilIndexed(BUNDLED, PEOPLE); + const ids = await rowIds(PEOPLE); + expect(ids).toEqual(expect.arrayContaining([`${PEOPLE}/JohnSmith`, 'arcp://name,organisation/ldaca'])); + const others = await rowIds(BUNDLED); - const page = await act(browser, 'delete', { crateId: BUNDLED }); + const page = await act(browser, 'delete', { crateId: PEOPLE }); - expect(flash(page.body)).toBe(`Deleting the indexes of ${BUNDLED} has started.`); + expect(flash(page.body)).toBe(`Deleting the indexes of ${PEOPLE} has started.`); await vi.waitFor(async () => { - expect(await entityCount(BUNDLED)).toBe(0); - expect(await searchCount(BUNDLED)).toBe(0); + expect(await prisma.entity.count({ where: { id: { in: ids } } })).toBe(0); + expect(await searchCount(ids)).toBe(0); }); - expect(await entityCount(PEOPLE)).toBeGreaterThan(0); - expect(await searchCount(PEOPLE)).toBeGreaterThan(0); + expect(await rowIds(BUNDLED)).toEqual(others); + expect(await searchCount(others)).toBe(others.length); + }); + + it('keeps a crate’s database rows when deleting its search documents fails', async () => { + const browser = await signIn(app); + await act(browser, 'index'); + await waitUntilIndexed(BUNDLED, PEOPLE); + const ids = await rowIds(PEOPLE); + const remove = vi.spyOn(search, 'remove').mockRejectedValue(new Error('Search is down')); + + try { + await act(browser, 'delete', { crateId: PEOPLE }); + await waitUntilSettled(); + expect(remove).toHaveBeenCalled(); + } finally { + remove.mockRestore(); + } + + expect(await rowIds(PEOPLE)).toEqual(ids); + expect(await searchCount(ids)).toBe(ids.length); + expect(await repository.getState(PEOPLE)).toEqual({ database: 'indexed', search: 'indexed' }); + }); + + it.each<[string, IndexType[] | undefined]>([ + ['search', ['search']], + ['database and search', undefined], + ])('reads a crate as not indexed as soon as its %s indexes are deleted', async (_, type) => { + await repository.createIndex(BUNDLED); + expect(await repository.getState(BUNDLED)).toEqual({ database: 'indexed', search: 'indexed' }); + + await repository.deleteIndex(BUNDLED, type); + + expect(hasState(await repository.getState(BUNDLED, 'search'), State.INDEXED)).toBe(false); }); it('deletes every index', async () => { const browser = await signIn(app); await act(browser, 'index'); await waitUntilIndexed(BUNDLED, PEOPLE); + const ids = await rowIds(BUNDLED, PEOPLE); const page = await act(browser, 'delete'); expect(flash(page.body)).toBe('Deleting every index has started.'); await vi.waitFor(async () => { expect(await prisma.entity.count()).toBe(0); - expect(await searchCount(BUNDLED)).toBe(0); - expect(await searchCount(PEOPLE)).toBe(0); + expect(await searchCount(ids)).toBe(0); }); }); @@ -220,7 +274,7 @@ describe('index and delete actions', () => { await indexing; - expect(read).toEqual({ structural: 'queued', search: 'queued' }); + expect(read).toEqual({ database: 'queued', search: 'queued' }); expect(state).toEqual(read); }); @@ -307,7 +361,7 @@ describe('protecting the actions', () => { ])('refuses to delete %s while a crate it covers is %s', async (_, state, form, asked) => { const repository = repositoryOf([ { id: PEOPLE, state: {} }, - { id: EXTRA, state: { structural: state } }, + { id: EXTRA, state: { database: state } }, ]); const browser = await signIn((await start(repository)).app); diff --git a/src/admin/crates.test.ts b/src/admin/crates.test.ts index bdf8366..92ef5bc 100644 --- a/src/admin/crates.test.ts +++ b/src/admin/crates.test.ts @@ -1,10 +1,8 @@ -import { execFileSync } from 'node:child_process'; -import { cp, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; +import { cp, mkdir, writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; -import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; -import { ALICE, adminTestbed, ocflRepository, PEOPLE, repositoryIn } from '../../test/admin.ts'; -import type { TestApp, TestRepository } from '../../test/app.ts'; +import { beforeAll, describe, expect, it, vi } from 'vitest'; +import { ALICE, adminTestbed, PEOPLE, repositoryIn } from '../../test/admin.ts'; +import { ocflRepository, ocflStorage, type TestApp, type TestRepository } from '../../test/app.ts'; import type { Repository } from '../repository.ts'; const BUNDLED = 'arcp://name,bundled'; @@ -59,9 +57,25 @@ const crate = (id: string, name: string) => ({ }); const testbed = adminTestbed(); -let storage: string; +const { ocflRoot } = ocflStorage({ + source: async (scratch) => { + const source = join(scratch, 'source'); + await mkdir(source); + for (const name of ['bundled', 'people']) { + await cp(resolve('test-data', name), join(source, name), { recursive: true }); + } + for (const [directory, id, name] of [ + ['extra', EXTRA, EXTRA_NAME], + ['more', MORE, 'More people'], + ] as const) { + await mkdir(join(source, directory)); + await writeFile(join(source, directory, 'ro-crate-metadata.json'), JSON.stringify(crate(id, name))); + } + return source; + }, +}); -const start = (repository: TestRepository) => testbed.start({ OCFL_ROOT: join(storage, 'ocfl'), ADMIN_USERS: ALICE.sub }, { repository }); +const start = (repository: TestRepository) => testbed.start({ OCFL_ROOT: ocflRoot(), ADMIN_USERS: ALICE.sub }, { repository }); const signIn = async (app: TestApp) => { const signedIn = await testbed.signInAsAdmin(app, ALICE); @@ -69,27 +83,6 @@ const signIn = async (app: TestApp) => { return signedIn; }; -beforeAll(async () => { - storage = await mkdtemp(join(tmpdir(), 'ldacapi-crates-')); - const source = join(storage, 'source'); - await mkdir(source); - for (const name of ['bundled', 'people']) { - await cp(resolve('test-data', name), join(source, name), { recursive: true }); - } - for (const [directory, id, name] of [ - ['extra', EXTRA, EXTRA_NAME], - ['more', MORE, 'More people'], - ] as const) { - await mkdir(join(source, directory)); - await writeFile(join(source, directory, 'ro-crate-metadata.json'), JSON.stringify(crate(id, name))); - } - execFileSync('node', ['scripts/seed.ts', source], { env: { ...process.env, OCFL_ROOT: join(storage, 'ocfl') }, stdio: 'ignore' }); -}); - -afterAll(async () => { - await rm(storage, { recursive: true, force: true }); -}); - describe('the crate tree', () => { let app: TestApp; let repository: Repository; @@ -112,8 +105,8 @@ describe('the crate tree', () => { await vi.waitFor(async () => { const page = await browser.get('/admin/'); const tree = crateTree(page.body); - expect(find(tree, BUNDLED)?.row).toMatch(/Structural: indexed .*Search: indexed/); - expect(find(tree, PEOPLE)?.row).toMatch(/Structural: not indexed .*Search: not indexed/); + expect(find(tree, BUNDLED)?.row).toMatch(/Database: indexed .*Search: indexed/); + expect(find(tree, PEOPLE)?.row).toMatch(/Database: not indexed .*Search: not indexed/); expect(page.body).not.toContain('http-equiv="refresh"'); }); }); diff --git a/src/admin/crates.ts b/src/admin/crates.ts index 091de16..6080c56 100644 --- a/src/admin/crates.ts +++ b/src/admin/crates.ts @@ -1,8 +1,8 @@ -import { hasState, type RepositoryObject, State } from '../repository.ts'; +import { hasState, INDEX_TYPES, type IndexType, type RepositoryObject, State } from '../repository.ts'; type Crate = RepositoryObject & { label: string; children: Crate[] }; -const INDEXES = { structural: 'Structural', search: 'Search' } as const; +const INDEX_LABELS: Record = { database: 'Database', search: 'Search' }; /** How often the page reloads while an index is changing */ const REFRESH_SECONDS = 5; @@ -59,7 +59,7 @@ const crateRow = (crate: Crate): CrateRow => { return { id: crate.id, label: crate.label, - states: Object.entries(INDEXES).map(([index, label]) => ({ label, value: crate.state[index] ?? 'not indexed' })), + states: INDEX_TYPES.map((index) => ({ label: INDEX_LABELS[index], value: crate.state[index] ?? 'not indexed' })), actions: [ { action: 'index', label: indexed ? 'Reindex' : 'Index', crateId: crate.id, force: indexed }, { action: 'delete', label: 'Delete', crateId: crate.id, confirm }, diff --git a/src/app.test.ts b/src/app.test.ts index 2e11c28..f19cb80 100644 --- a/src/app.test.ts +++ b/src/app.test.ts @@ -1,7 +1,7 @@ import { Client } from '@opensearch-project/opensearch'; import { describe, expect, it } from 'vitest'; import pkg from '../package.json' with { type: 'json' }; -import { repository } from '../test/app.ts'; +import { buildTestApp, repository, repositoryWithContent } from '../test/app.ts'; import { buildApp } from './app.ts'; import { config } from './configuration.ts'; import { createSearchBackend } from './indexer/search.ts'; @@ -81,3 +81,38 @@ describe('buildApp', () => { await app.close(); }); }); + +describe('crate file delivery', () => { + const OPEN = 'https://creativecommons.org/licenses/by/4.0/'; + const files = { 'http://test/delivery/page.html': 'text/html', 'http://test/delivery/song.mp3': 'audio/mpeg' }; + + it('sandboxes crate files but not audio or video, so media plays when opened directly', async () => { + const { app, prisma, config } = await buildTestApp({}, { repository: repositoryWithContent }); + for (const [id, mediaType] of Object.entries(files)) { + await prisma.entity.create({ + data: { id, name: id, description: '', entityType: 'http://schema.org/MediaObject', metadataLicenseId: OPEN, contentLicenseId: OPEN }, + }); + await prisma.file.create({ data: { id, filename: id.split('/').at(-1) ?? id, mediaType, size: 7 } }); + } + + try { + const deliver = (id: string) => app.inject({ method: 'GET', url: `${config.prefix}/file/${encodeURIComponent(id)}` }); + + const html = await deliver('http://test/delivery/page.html'); + expect(html.statusCode).toBe(200); + expect(html.headers['x-content-type-options']).toBe('nosniff'); + expect(html.headers['content-security-policy']).toContain('sandbox'); + + const audio = await deliver('http://test/delivery/song.mp3'); + expect(audio.statusCode).toBe(200); + expect(audio.headers['x-content-type-options']).toBe('nosniff'); + expect(audio.headers['content-security-policy']).toBeUndefined(); + } finally { + const ids = Object.keys(files); + await prisma.file.deleteMany({ where: { id: { in: ids } } }); + await prisma.entity.deleteMany({ where: { id: { in: ids } } }); + await app.close(); + await prisma.$disconnect(); + } + }); +}); diff --git a/src/app.ts b/src/app.ts index 21a9c80..4a927a1 100644 --- a/src/app.ts +++ b/src/app.ts @@ -39,6 +39,7 @@ export function buildApp({ repository, search }: AppDeps) { fastify.register(fastifySensible); fastify.register( async (api) => { + // Here rather than at the root, to keep CORS headers off /oauth and /admin api.register(cors, { origin: '*', methods: ['GET', 'HEAD', 'POST'], maxAge: 2 * 60 * 60 }); api.register(arocapi, { prisma, diff --git a/src/auth.ts b/src/auth.ts index 04d9fe7..60bb0d4 100644 --- a/src/auth.ts +++ b/src/auth.ts @@ -44,19 +44,23 @@ async function fetchUserLicenses(request: FastifyRequest): Promise item.resource); + } catch (err) { + request.log.error({ err }, 'Failed to fetch user entitlements'); return; } - const data = await res.json(); - return data.map((item: RemsUserEntitlement) => item.resource); } const userIdsByRequest = new WeakMap>(); diff --git a/src/collection-cycle.test.ts b/src/collection-cycle.test.ts new file mode 100644 index 0000000..5c75f4a --- /dev/null +++ b/src/collection-cycle.test.ts @@ -0,0 +1,65 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { beforeAll, describe, expect, it, vi } from 'vitest'; +import { ocflTestbed } from '../test/app.ts'; + +const logs = vi.hoisted(() => [] as Record[]); + +vi.mock('./logger.ts', async () => (await import('../test/logs.ts')).capturingLogger(logs, 'warn')); + +const FIRST = 'arcp://name,cycle_first'; +const SECOND = 'arcp://name,cycle_second'; +const LICENSE = { '@id': 'https://creativecommons.org/licenses/by/4.0/' }; + +const collection = (id: string, parent: string) => ({ + '@context': ['https://w3id.org/ro/crate/1.1/context', 'https://w3id.org/ldac/context', { '@vocab': 'http://schema.org/' }], + '@graph': [ + { '@id': 'ro-crate-metadata.json', '@type': 'CreativeWork', license: LICENSE, about: { '@id': id } }, + { + '@id': id, + '@type': ['Dataset', 'RepositoryCollection'], + conformsTo: { '@id': 'https://w3id.org/ldac/profile#Collection' }, + memberOf: { '@id': parent }, + name: id, + license: LICENSE, + }, + ], +}); + +const testbed = ocflTestbed({ + source: async (scratch) => { + const source = join(scratch, 'crates'); + for (const [name, id, parent] of [ + ['first', FIRST, SECOND], + ['second', SECOND, FIRST], + ] as const) { + await mkdir(join(source, name), { recursive: true }); + await writeFile(join(source, name, 'ro-crate-metadata.json'), JSON.stringify(collection(id, parent))); + } + return source; + }, +}); + +beforeAll(async () => { + await testbed.started.repository.deleteIndex(); + await testbed.started.repository.createIndex(FIRST); +}); + +describe('collections that are members of each other', () => { + it('logs the cycle, naming the crates', () => { + expect(logs).toContainEqual( + expect.objectContaining({ + level: 40, + crateId: FIRST, + cycle: [FIRST, SECOND, FIRST], + msg: 'Collections are members of each other; using the highest found', + }), + ); + }); + + it('still indexes the crate under the highest collection found', async () => { + const row = await testbed.started.prisma.entity.findUnique({ where: { id: FIRST }, select: { rootCollection: true } }); + + expect(row?.rootCollection).toBe(SECOND); + }); +}); diff --git a/src/entity-meta.ts b/src/entity-meta.ts index 13fbde9..9c446c4 100644 --- a/src/entity-meta.ts +++ b/src/entity-meta.ts @@ -5,8 +5,8 @@ const ACCESS_CONTROLS = ['Public', 'AuthorizationByApplication'] as const; /** How Oni shows access is granted: at once, or by applying in REMS */ export type AccessControl = (typeof ACCESS_CONTROLS)[number]; -/** What the structural index keeps in `Entity.meta` */ -export type EntityMeta = { rocrate: unknown; accessControl: AccessControl }; +/** What the database index keeps in `Entity.meta`; `crateId` is the crate the entity was indexed from */ +export type EntityMeta = { crateId: string; rocrate: unknown; accessControl: AccessControl }; export const accessControlOf = (meta: unknown): AccessControl | undefined => { const value = (meta as Partial | null)?.accessControl; diff --git a/src/indexer/structural.test.ts b/src/indexer/database.test.ts similarity index 90% rename from src/indexer/structural.test.ts rename to src/indexer/database.test.ts index b8dbcfa..bb3458e 100644 --- a/src/indexer/structural.test.ts +++ b/src/indexer/database.test.ts @@ -3,8 +3,8 @@ import { ROCrate } from 'ro-crate'; import { afterEach, beforeAll, describe, expect, it } from 'vitest'; import { accessControlOf } from '../entity-meta.ts'; import { prisma } from '../prisma.ts'; +import { DatabaseIndexer, entityAsCrate } from './database.ts'; import type { CrateObject } from './indexer.ts'; -import { entityAsCrate, StructuralIndexer } from './structural.ts'; describe('entityAsCrate', () => { it('preserves the original RO-Crate context when serializing an entity', async () => { @@ -16,15 +16,15 @@ describe('entityAsCrate', () => { }); }); -describe('StructuralIndexer', () => { - const ROOT = 'arcp://name,structural-test'; +describe('DatabaseIndexer', () => { + const ROOT = 'arcp://name,database-test'; const OPEN = 'https://creativecommons.org/licenses/by/4.0/'; const RESTRICTED = 'https://www.ldaca.edu.au/licenses/restricted/v1/'; - let indexer: StructuralIndexer; + let indexer: DatabaseIndexer; beforeAll(async () => { - indexer = await StructuralIndexer.create({}); + indexer = await DatabaseIndexer.create({}); }); afterEach(async () => { @@ -40,7 +40,7 @@ describe('StructuralIndexer', () => { '@id': ROOT, '@type': ['Dataset', 'RepositoryCollection'], conformsTo: { '@id': 'https://w3id.org/ldac/profile#Collection' }, - name: 'Structural test', + name: 'Database test', license: OPEN, hasPart: files.map((f) => ({ '@id': f['@id'] })), }, @@ -48,7 +48,7 @@ describe('StructuralIndexer', () => { ], }); const crateObject: CrateObject = { - id: 'structural-test', + id: 'database-test', root: '/nowhere', text: async () => '', file: async (path) => { @@ -59,7 +59,7 @@ describe('StructuralIndexer', () => { return { size, crc32: '' }; }, }; - await indexer.index({ crateObject, crate }); + await indexer.index({ crateObject, crate, topCollection: ROOT }); }; describe('file size', () => { diff --git a/src/indexer/structural.ts b/src/indexer/database.ts similarity index 73% rename from src/indexer/structural.ts rename to src/indexer/database.ts index 94b27a4..5b041c8 100644 --- a/src/indexer/structural.ts +++ b/src/indexer/database.ts @@ -5,25 +5,25 @@ import type { Prisma } from '../generated/prisma/client.ts'; import { logger } from '../logger.ts'; import { prisma } from '../prisma.ts'; import { firstStringOrId } from '../utils.ts'; -import { type BaseOptions, type CrateFile, Indexer, RecordType, recordType } from './indexer.ts'; +import { type BaseOptions, type CrateFile, entityClass, Indexer, memberOf, recordType, rootCollection } from './indexer.ts'; -const log = logger.child({ module: 'indexer/structural' }); +const log = logger.child({ module: 'indexer/database' }); type IndexRecord = { entity: Prisma.EntityUncheckedCreateInput; file?: Prisma.FileUncheckedCreateInput }; -type StructuralOptions = BaseOptions & { +type DatabaseOptions = BaseOptions & { memberOfField?: string; }; -export class StructuralIndexer extends Indexer { +export class DatabaseIndexer extends Indexer { memberOfField: string; - constructor(opt: StructuralOptions) { + constructor(opt: DatabaseOptions) { super(opt); this.memberOfField = opt.memberOfField || 'pcdm:memberOf'; } - override async _index({ crateObject, crate, paths, license, metadataLicense }: Parameters[0]) { + override async _index({ crateObject, crate, paths, license, metadataLicense, topCollection }: Parameters[0]) { const crateId = crate.rootId; let count = 0; for (const entity of crate.entities()) { @@ -41,12 +41,12 @@ export class StructuralIndexer extends Indexer { id: entityId, name: entity.name?.join('; ') || entityId, description: entity.description?.join('; ') || '', - entityType: crate.getContextDefinition(entityType) || RecordType[entityType], + entityType: entityClass(crate, entityType), memberOf: pickSingleMemberOf(entity), - rootCollection: crate.rootId, + rootCollection: rootCollection(topCollection, entityId), metadataLicenseId: metadataLicense, contentLicenseId, - meta: { rocrate, accessControl: accessControlFor(contentLicenseId) } satisfies EntityMeta, + meta: { crateId, rocrate, accessControl: accessControlFor(contentLicenseId) } satisfies EntityMeta, }, }; if (entityType.endsWith('://schema.org/MediaObject') || entityType === 'File') { @@ -81,16 +81,25 @@ export class StructuralIndexer extends Indexer { } async delete(crateId?: string) { - await prisma.file.deleteMany({ where: crateId ? { entity: { rootCollection: crateId } } : {} }); - await prisma.entity.deleteMany({ where: crateId ? { rootCollection: crateId } : {} }); + await prisma.file.deleteMany({ where: { entity: rowsOf(crateId) } }); + await prisma.entity.deleteMany({ where: rowsOf(crateId) }); log.debug({ crateId }, 'Index deleted'); } - async count(crateId?: string) { - return await prisma.entity.count(crateId ? { where: { id: crateId } } : undefined); + async count(crateId: string) { + return await prisma.entity.count({ where: rowsOf(crateId) }); } } +/** The entity rows indexed from `crateId`, or every row */ +const rowsOf = (crateId?: string): Prisma.EntityWhereInput => (crateId ? { meta: { path: ['crateId'], equals: crateId } } : {}); + +/** The id and rootCollection of each entity row indexed from `crateId` */ +export const crateRows = (crateId: string) => prisma.entity.findMany({ where: rowsOf(crateId), select: { id: true, rootCollection: true } }); + +/** The ids of the entity rows indexed from `crateId` */ +export const rowIds = async (crateId: string) => (await crateRows(crateId)).map(({ id }) => id); + export function entityAsCrate(crate: ROCrate, entity: Entity, license: string) { const newCrate = new ROCrate({ '@context': crate['@context'] }, { array: true, link: true }); for (const key in entity) { @@ -114,8 +123,7 @@ function statedSize(contentSize: unknown) { export function pickSingleMemberOf(entity: Entity) { return ( - entity['pcdm:memberOf']?.[0]['@id'] || - entity.memberOf?.[0]['@id'] || + memberOf(entity) || entity['@reverse']['pcdm:hasMember']?.[0]?.['@id'] || entity['@reverse'].hasMember?.[0]?.['@id'] || entity.isPartOf?.find((e: Entity) => e['@type'].includes('RepositoryObject'))?.['@id'] || diff --git a/src/indexer/indexer.ts b/src/indexer/indexer.ts index 9bef0b2..3ee7ee6 100644 --- a/src/indexer/indexer.ts +++ b/src/indexer/indexer.ts @@ -97,7 +97,7 @@ export class Indexer { async init() {} - async count(_crateId?: string): Promise { + async count(_crateId: string): Promise { throw new Error('Not Implemented'); } @@ -110,7 +110,8 @@ export class Indexer { throw new Error('Not Implemented'); } - async index({ crateObject, crate }: { crateObject: CrateObject; crate: ROCrate }) { + /** `topCollection` is the top-level collection the crate sits under, the crate's own root if it has no parent */ + async index({ crateObject, crate, topCollection }: { crateObject: CrateObject; crate: ROCrate; topCollection: string }) { const rootDataset = crate.root; const crateId = crate.rootId; const metadataLicense = firstStringOrId(crate.descriptor.license) || this.defaultMetadataLicense; @@ -127,15 +128,21 @@ export class Indexer { log.info({ crateId }, 'Indexing crate'); const paths = this.deriveUniqueEntityIds(crate); try { - await this._index({ crateObject, crate, paths, license, metadataLicense }); + await this._index({ crateObject, crate, paths, license, metadataLicense, topCollection }); } catch (error) { log.error({ err: error, crateId, path: crateObject.root }, 'Error indexing crate'); } - log.info({ crateId }, 'Indexed crate'); } } - async _index(_param: { crateObject: CrateObject; crate: ROCrate; paths: LogicalPaths; license: string; metadataLicense: string }) { + async _index(_param: { + crateObject: CrateObject; + crate: ROCrate; + paths: LogicalPaths; + license: string; + metadataLicense: string; + topCollection: string; + }): Promise { throw new Error('Not Implemented'); } @@ -145,7 +152,19 @@ export class Indexer { } } -export const RecordType: Record = config.indexType; +/** An entity's rootCollection: empty on the top-level collection itself */ +export const rootCollection = (topCollection: string, entityId: string) => (entityId === topCollection ? null : topCollection); + +/** The first collection `entity` is a member of, by `pcdm:memberOf` or `memberOf`, in a parsed crate or raw JSON-LD */ +export const memberOf = (entity: Record) => firstStringOrId(entity['pcdm:memberOf']) || firstStringOrId(entity.memberOf); + +/** The id of the collection a crate's root is a member of, if any */ +export function parentCollection(root: Record, rootId: string): string | undefined { + const parent = memberOf(root); + return parent && parent !== rootId ? parent : undefined; +} + +const RecordType: Record = config.indexType; /** The entity's first type that we index, if it conforms to that type's profile */ export function recordType(entity: Entity): string | undefined { @@ -156,3 +175,6 @@ export function recordType(entity: Entity): string | undefined { } return type; } + +/** The class URI of a record type, as the entity's row holds it */ +export const entityClass = (crate: ROCrate, type: string): string => crate.getContextDefinition(type) || RecordType[type]; diff --git a/src/indexer/search.test.ts b/src/indexer/search.test.ts index 1ab291e..45c6a71 100644 --- a/src/indexer/search.test.ts +++ b/src/indexer/search.test.ts @@ -1,14 +1,14 @@ import { ROCrate } from 'ro-crate'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { type CrateObject, Indexer } from './indexer.ts'; -import { searchDocument } from './search.ts'; +import { countDocuments, searchDocument } from './search.ts'; const ROOT = 'arcp://name,test'; const LICENSE = 'https://creativecommons.org/licenses/by/4.0/'; -const crateWith = async (entity: Record) => { +const crateWith = async (entity: Record, terms: Record = {}) => { const crate = await ROCrate.create({ - '@context': ['https://w3id.org/ro/crate/1.1/context', { '@vocab': 'http://schema.org/', ldac: 'https://w3id.org/ldac/terms#' }], + '@context': ['https://w3id.org/ro/crate/1.1/context', { '@vocab': 'http://schema.org/', ldac: 'https://w3id.org/ldac/terms#', ...terms }], '@graph': [ { '@id': 'ro-crate-metadata.json', '@type': 'CreativeWork', about: { '@id': ROOT }, license: { '@id': LICENSE } }, { '@id': ROOT, '@type': ['Dataset', 'RepositoryCollection'], name: 'Root', hasPart: [{ '@id': 'data.txt' }] }, @@ -34,10 +34,11 @@ describe('searchDocument', () => { contentLocation: { '@id': '#place' }, }); - expect(await searchDocument(crate, crateObject, entity, LICENSE, path)).toMatchObject({ + expect(await searchDocument(crate, crateObject, entity, LICENSE, ROOT, path)).toMatchObject({ id: `${ROOT}/data.txt`, name: 'Data', - type: ['File'], + entityType: 'http://schema.org/MediaObject', + recordType: 'File', inLanguage: ['English'], communicationMode: ['https://w3id.org/ldac/terms#SpokenLanguage'], encodingFormat: ['text/plain'], @@ -49,12 +50,57 @@ describe('searchDocument', () => { }); }); + it.each([ + [['Dataset', 'RepositoryCollection'], 'https://w3id.org/ldac/profile#Collection', 'Collection'], + [['Dataset', 'RepositoryObject'], 'https://w3id.org/ldac/profile#Object', 'Object'], + ['Person', 'https://w3id.org/ldac/profile#Person', 'Person'], + ['Organization', 'https://w3id.org/ldac/profile#Organization', 'Organisation'], + ['SoftwareApplication', 'https://w3id.org/ldac/profile#SoftwareApplication', 'Software Application'], + ])('labels a %s record as %s', async (type, profile, label) => { + const { crate, entity } = await crateWith({ '@type': type, conformsTo: { '@id': profile } }); + + expect((await searchDocument(crate, crateObject, entity, LICENSE, ROOT)).recordType).toBe(label); + }); + + it.each(['http://schema.org/MediaObject', 'https://schema.org/MediaObject'])('labels %s as File, whichever the scheme', async (uri) => { + const { crate, entity } = await crateWith({}, { File: uri }); + + expect(await searchDocument(crate, crateObject, entity, LICENSE, ROOT)).toMatchObject({ entityType: uri, recordType: 'File' }); + }); + + it.each([ + ['http://schema.org/Thing', 'Thing'], + ['https://example.com/terms#Widget', 'Widget'], + ])('labels an unknown class %s by its last segment', async (uri, label) => { + const { crate, entity } = await crateWith({}, { File: uri }); + + const doc = await searchDocument(crate, crateObject, entity, LICENSE, ROOT); + + expect(doc).toMatchObject({ entityType: uri, recordType: label }); + }); + + it('has no type field', async () => { + const { crate, entity } = await crateWith({}); + + expect(await searchDocument(crate, crateObject, entity, LICENSE, ROOT)).not.toHaveProperty('type'); + }); + it('leaves out dates it cannot store', async () => { const { crate, entity } = await crateWith({ datePublished: 'circa 1900', dateCreated: ['1901-02-03', 'unknown'] }); - const doc = await searchDocument(crate, crateObject, entity, LICENSE); + const doc = await searchDocument(crate, crateObject, entity, LICENSE, ROOT); expect(doc).not.toHaveProperty('datePublished'); expect(doc.dateCreated).toEqual(['1901-02-03']); }); }); + +describe('countDocuments', () => { + it('counts in batches under OpenSearch’s terms limit, summing them', async () => { + const search = vi.fn(async ({ visibility }: { visibility: { id: { in: string[] } } }) => ({ total: visibility.id.in.length, hits: [] })); + const ids = Array.from({ length: 25_000 }, (_, i) => `id-${i}`); + + expect(await countDocuments({ search } as unknown as Parameters[0], ids)).toBe(25_000); + expect(search.mock.calls.map(([{ visibility }]) => visibility.id.in.length)).toEqual([10_000, 10_000, 5_000]); + }); +}); diff --git a/src/indexer/search.ts b/src/indexer/search.ts index fe23353..4982464 100644 --- a/src/indexer/search.ts +++ b/src/indexer/search.ts @@ -4,10 +4,11 @@ import { OpenSearchBackend } from 'arocapi/search/opensearch'; import type { Entity, ROCrate } from 'ro-crate'; import { config } from '../configuration.ts'; import { logger } from '../logger.ts'; +import { COLLECTION, OBJECT } from '../oni.ts'; import { firstStringOrId } from '../utils.ts'; +import { crateRows, pickSingleMemberOf, rowIds } from './database.ts'; import type { BaseOptions, CrateObject } from './indexer.ts'; -import { Indexer, recordType } from './indexer.ts'; -import { pickSingleMemberOf } from './structural.ts'; +import { entityClass, Indexer, recordType } from './indexer.ts'; const log = logger.child({ module: 'indexer/search' }); @@ -15,7 +16,8 @@ const fields = { name: { type: 'text', label: 'Name', weight: 3, advanced: true, sort: true }, description: { type: 'text', label: 'Description', advanced: true }, text: { type: 'text', label: 'Text', advanced: true }, - type: { type: 'keyword', label: 'Record Type', facet: true }, + entityType: { type: 'keyword', label: 'Entity Type', filter: true }, + recordType: { type: 'keyword', label: 'Record Type', facet: true }, inLanguage: { type: 'keyword', label: 'Language', advanced: true, facet: true }, communicationMode: { type: 'keyword', label: 'Communication Mode', facet: true }, linguisticGenre: { type: 'keyword', label: 'Linguistic Genre', facet: true }, @@ -36,11 +38,54 @@ export const createSearchBackend = (client: Client) => OpenSearchBackend({ clien type SearchBackend = ReturnType; +/** Well under OpenSearch's default `index.max_terms_count` of 65,536 */ +const BATCH_SIZE = 10_000; + +const batches = (items: T[]) => Array.from({ length: Math.ceil(items.length / BATCH_SIZE) }, (_, i) => items.slice(i * BATCH_SIZE, (i + 1) * BATCH_SIZE)); + +/** How many of `ids` have a search document, 0 if there is no index */ +export async function countDocuments(search: SearchBackend, ids: string[]) { + try { + let count = 0; + for (const batch of batches(ids)) { + const { total } = await search.search({ + searchType: 'basic', + query: '', + filters: { terms: {}, ranges: {} }, + limit: 1, + offset: 0, + sort: 'relevance', + order: 'asc', + visibility: { id: { in: batch } }, + }); + count += total; + } + return count; + } catch (error) { + if ((error as { statusCode?: number }).statusCode === 404) { + return 0; + } + throw error; + } +} + type SearchOptions = BaseOptions & { client: Client; search: SearchBackend; }; +const CLASS_LABELS: Record = { + [COLLECTION]: 'Collection', + [OBJECT]: 'Object', + 'http://schema.org/MediaObject': 'File', + 'http://schema.org/Person': 'Person', + 'http://schema.org/Organization': 'Organisation', + 'http://schema.org/SoftwareApplication': 'Software Application', +}; + +/** A readable label for a class URI, either schema.org scheme alike, falling back to its last segment */ +const classLabel = (uri: string) => CLASS_LABELS[uri.replace(/^https:\/\/schema\.org\//, 'http://schema.org/')] ?? uri.split(/[#/]/).at(-1) ?? uri; + const values = (entity: Entity, name: string): unknown[] => [...(entity[name] ?? []), ...(entity[`ldac:${name}`] ?? [])]; /** Linked entities by name, falling back to their id; literals as they are */ @@ -101,8 +146,17 @@ function isText(entity: Entity) { } /** One entity's search document; File entities with text content include it */ -export async function searchDocument(crate: ROCrate, crateObject: CrateObject, entity: Entity, license: string, path?: string): Promise { +export async function searchDocument( + crate: ROCrate, + crateObject: CrateObject, + entity: Entity, + license: string, + rootCollection: string | null, + path?: string, +): Promise { const id: string = entity['@id']; + const type = recordType(entity); + const entityType = type && entityClass(crate, type); const [datePublished] = dates(entity, 'datePublished'); const memberOf = pickSingleMemberOf(entity); const point = location(entity); @@ -110,13 +164,13 @@ export async function searchDocument(crate: ROCrate, crateObject: CrateObject, e id, name: entity.name?.join('; ') || id, description: labels(entity, 'description'), - type: entity['@type'], + ...(entityType && { entityType, recordType: classLabel(entityType) }), inLanguage: labels(entity, 'inLanguage'), communicationMode: labels(entity, 'communicationMode'), linguisticGenre: labels(entity, 'linguisticGenre'), annotationType: labels(entity, 'annotationType'), encodingFormat: labels(entity, 'encodingFormat'), - rootCollection: crate.rootId, + ...(rootCollection && { rootCollection }), ...(memberOf && { memberOf }), license: firstStringOrId(entity.license) || license, ...(datePublished && { datePublished }), @@ -161,59 +215,84 @@ export class SearchIndexer extends Indexer { async delete(crateId?: string) { if (crateId) { - await this.client.deleteByQuery({ index: this.indexName, refresh: true, body: { query: { term: { rootCollection: crateId } } } }); + for (const batch of batches(await rowIds(crateId))) { + await this.search.remove(batch); + } + await this.refresh(); } else { await this.client.indices.delete({ index: this.indexName, ignore_unavailable: true }); } log.debug({ crateId }, 'Index deleted'); } - async count(crateId?: string) { - try { - const res = await this.client.count({ - index: this.indexName, - ...(crateId && { body: { query: { term: { rootCollection: crateId } } } }), - }); - return res.body.count; - } catch { - return 0; - } + async count(crateId: string) { + return await countDocuments(this.search, await rowIds(crateId)); } + /** Makes changes searchable now, rather than at OpenSearch's next periodic refresh */ + async refresh() { + await this.client.indices.refresh({ index: this.indexName, ignore_unavailable: true }); + } + + /** Only entities with a row, as arocapi drops a hit without one but still counts it */ async _index({ crateObject, crate, paths, license }: Parameters[0]) { await this.search.createIndex(); + const crateId = crate.rootId; + const rows = new Map((await crateRows(crateId)).map(({ id, rootCollection }) => [id, rootCollection])); + const unrowed: string[] = []; const docs: Document[] = []; - for (const entity of crate.entities()) { - if (!recordType(entity)) { - continue; + let count = 0; + try { + for (const entity of crate.entities()) { + if (!recordType(entity)) { + continue; + } + const entityId = entity['@id']; + const rootCollection = rows.get(entityId); + if (rootCollection === undefined) { + unrowed.push(entityId); + continue; + } + const doc = await searchDocument(crate, crateObject, entity, license, rootCollection, paths.get(entityId)); + if (doc.text) { + // Indexed alone to keep each request small + count += await this.indexDocuments([doc]); + } else { + docs.push(doc); + } } - const doc = await searchDocument(crate, crateObject, entity, license, paths.get(entity['@id'])); - if (doc.text) { - // Indexed alone to keep each request small - await this.indexDocuments([doc]); - } else { - docs.push(doc); + count += await this.indexDocuments(docs); + } finally { + await this.refresh(); + } + if (rows.size === 0 && unrowed.length > 0) { + log.warn({ crateId, skipped: unrowed.length }, 'Skipped crate: no database rows'); + } else { + for (const entityId of unrowed) { + log.warn({ crateId, entityId }, 'Skipped: no database row'); } } - await this.indexDocuments(docs); - await this.client.indices.refresh({ index: this.indexName }); + log.info({ crateId, documents: count }, 'Indexed crate'); } - /** Index a batch, falling back to one at a time so one bad document doesn't lose the rest */ - async indexDocuments(docs: Document[]) { + /** Index a batch, falling back to one at a time so one bad document doesn't lose the rest; returns how many were indexed */ + async indexDocuments(docs: Document[]): Promise { if (!docs.length) { - return; + return 0; } try { await this.search.index(docs); + return docs.length; } catch (error) { if (docs.length === 1) { log.error({ err: error, entityId: docs[0]?.id }, 'Error indexing document'); - return; + return 0; } + let count = 0; for (const doc of docs) { - await this.indexDocuments([doc]); + count += await this.indexDocuments([doc]); } + return count; } } } diff --git a/src/licence.test.ts b/src/licence.test.ts index 43be883..11d25ef 100644 --- a/src/licence.test.ts +++ b/src/licence.test.ts @@ -1,7 +1,7 @@ import { createServer, type IncomingMessage, type Server } from 'node:http'; import type { AddressInfo } from 'node:net'; import { generateKeyPair, importJWK, type JWTPayload, SignJWT } from 'jose'; -import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { buildTestApp, repositoryWithContent, type TestApp } from '../test/app.ts'; import { generateSigningKey, type Jwk } from './oidc/keys.ts'; @@ -12,11 +12,17 @@ const RESTRICTED = 'https://example.org/licenses/restricted'; const RESTRICTED_METADATA = 'https://example.org/licenses/restricted-metadata'; const ENTITLED = 'upstream-user-entitled'; const UNENTITLED = 'upstream-user-unentitled'; +const UNREACHABLE = 'upstream-user-rems-unreachable'; +const GARBLED = 'upstream-user-rems-garbled'; const OPEN_FILE = 'http://test/licence/open.txt'; const RESTRICTED_FILE = 'http://test/licence/restricted.txt'; const RESTRICTED_ENTITY = 'http://test/licence/restricted'; +const logs = vi.hoisted(() => [] as Record[]); + +vi.mock('./logger.ts', async () => (await import('../test/logs.ts')).capturingLogger(logs, 'error')); + let app: TestApp; let prisma: Awaited>['prisma']; let prefix: string; @@ -59,6 +65,14 @@ beforeAll(async () => { server = createServer((request, response) => { received.push(request); const user = new URL(request.url ?? '', 'http://rems').searchParams.get('user'); + if (user === UNREACHABLE) { + request.socket.destroy(); + return; + } + if (user === GARBLED) { + response.writeHead(200, { 'content-type': 'application/json' }).end('{"entitlements": ['); + return; + } const entitlements = user === ENTITLED ? [{ resource: RESTRICTED }, { resource: RESTRICTED_METADATA }] : []; response.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify(entitlements)); }); @@ -74,6 +88,7 @@ beforeAll(async () => { OIDC_UPSTREAM_CLIENT_ID: 'ldacapi', OIDC_UPSTREAM_CLIENT_SECRET: 'secret', REMS_ENDPOINT: fake, + ENROLLMENT_URL: 'https://rems.example.org/apply-for?resource={licenseId}', OIDC_SIGNING_KEYS: JSON.stringify({ keys: SIGNING_KEYS }), }, { repository: repositoryWithContent }, @@ -89,6 +104,7 @@ beforeAll(async () => { beforeEach(() => { received = []; + logs.length = 0; }); afterAll(async () => { @@ -127,6 +143,29 @@ describe('licence checks with our access tokens', () => { expect(remsRequests()).toHaveLength(1); }); + describe.each([ + ['is unreachable', UNREACHABLE], + ['answers with a garbled body', GARBLED], + ])('when REMS %s', (_, sub) => { + it('denies restricted content with its enrolment link, and logs the failure', async () => { + const response = await get(`/entity/${encodeURIComponent(RESTRICTED_FILE)}`, await token({ sub })); + + expect(response.statusCode).toBe(200); + expect(response.json().access).toEqual({ + metadata: true, + content: false, + contentAuthorizationUrl: `https://rems.example.org/apply-for?resource=${encodeURIComponent(RESTRICTED)}`, + }); + expect(logs).toContainEqual(expect.objectContaining({ level: 50, msg: 'Failed to fetch user entitlements' })); + }); + + it('still serves content under an open licence', async () => { + const response = await file(OPEN_FILE, await token({ sub })); + + expect(response.statusCode).toBe(200); + }); + }); + it('does not call the upstream provider', async () => { await file(RESTRICTED_FILE, await token()); diff --git a/src/ocfl.ts b/src/ocfl.ts index 909f593..a19bf45 100644 --- a/src/ocfl.ts +++ b/src/ocfl.ts @@ -6,25 +6,25 @@ import type { StorageBackend } from 'arocapi'; import { OcflStorage } from 'arocapi/storage/ocfl'; import { type RawEntity, ROCrate } from 'ro-crate'; import { config } from './configuration.ts'; -import type { CrateObject, Indexer } from './indexer/indexer.ts'; +import { DatabaseIndexer } from './indexer/database.ts'; +import { type CrateObject, type Indexer, parentCollection } from './indexer/indexer.ts'; import { createSearchBackend, SearchIndexer } from './indexer/search.ts'; -import { StructuralIndexer } from './indexer/structural.ts'; import { logger } from './logger.ts'; -import { type IndexState, State } from './repository.ts'; +import { INDEX_TYPES, type IndexState, type IndexStates, type IndexType, State } from './repository.ts'; const log = logger.child({ module: 'ocfl' }); const { defaultLicense, defaultMetadataLicense } = config; -const stateCache: { [key: string]: { [key: string]: IndexState | undefined } } = {}; -let INDEXER: { [key: string]: Indexer }; +const stateCache: Record = {}; +let INDEXER: Record; let repository: ReturnType; let backend: StorageBackend; export async function init(opts: { opensearchClient: Client; search?: ReturnType }) { log.info('Initializing OCFL repository and indexers'); INDEXER = { - structural: await StructuralIndexer.create({ + database: await DatabaseIndexer.create({ defaultLicense, defaultMetadataLicense, }), @@ -107,14 +107,14 @@ function wrap(ocflObject: OcflObject): CrateObject { }; } -function _setState(crateId: string, types: string | string[], state?: IndexState) { - stateCache[crateId] = stateCache[crateId] || {}; - for (const type of ([] as string[]).concat(types)) { +function _setState(crateId: string, types: IndexType | IndexType[], state?: IndexState) { + stateCache[crateId] ??= {}; + for (const type of [types].flat()) { stateCache[crateId][type] = state; } } -function setState(crateId: string | undefined, types: string | string[], state?: IndexState) { +function setState(crateId: string | undefined, types: IndexType | IndexType[], state?: IndexState) { if (crateId) { _setState(crateId, types, state); } else { @@ -124,47 +124,118 @@ function setState(crateId: string | undefined, types: string | string[], state?: } } -export async function getState(crateId: string, type?: string) { - const indexers: [string, Indexer | undefined][] = type ? [[type, INDEXER[type]]] : Object.entries(INDEXER); - for (const [name, indexer] of indexers) { - if (!indexer) { - return; - } - if (!stateCache[crateId]?.[name]) { - const count = await indexer.count(crateId); - if (count > 0) { - stateCache[crateId] = stateCache[crateId] || {}; - stateCache[crateId][name] = State.INDEXED; - } +export async function getState(crateId: string, type?: IndexType): Promise { + for (const name of type ? [type] : INDEX_TYPES) { + if (!stateCache[crateId]?.[name] && (await INDEXER[name].count(crateId)) > 0) { + _setState(crateId, name, State.INDEXED); } } // A copy, as indexing goes on changing the cache after this returns return type ? { [type]: stateCache[crateId]?.[type] } : { ...stateCache[crateId] }; } -async function indexObject(crateId: string, types: string[], force?: boolean) { +/** + * The search index finds a crate's documents, and their rootCollection, by its database rows. So the rows are indexed + * first and deleted last, deleting stops at the first failure, and deleting the rows deletes the documents too. + */ +const INDEX_ORDER: readonly IndexType[] = INDEX_TYPES; +const DELETE_ORDER: readonly IndexType[] = ['search', 'database']; + +const typesOf = (type?: IndexType | IndexType[]) => (type ? [type].flat() : [...INDEX_ORDER]); + +const withDocuments = (types: IndexType[]): IndexType[] => (types.includes('database') && !types.includes('search') ? [...types, 'search'] : types); + +async function deleteFrom(types: IndexType[], crateId?: string) { + for (const t of DELETE_ORDER.filter((t) => types.includes(t))) { + try { + await INDEXER[t].delete(crateId); + } catch (error) { + log.error({ err: error, crateId, indexer: t }, 'Error deleting index'); + throw error; + } + } +} + +const readCrateJson = async (ocflObject: OcflObject): Promise<{ '@graph': RawEntity[] }> => + JSON.parse(await ocflObject.getFile({ logicalPath: 'ro-crate-metadata.json' }).text()); + +/** The root entity of a crate's raw JSON-LD, whose id is the OCFL object's */ +const rawRoot = (json: { '@graph': RawEntity[] }, crateId: string) => json['@graph'].find((e) => e['@id'] === crateId); + +/** A crate's OCFL object, whose id is the crate's root id */ +async function loadObject(crateId: string) { + const ocflObject = repository.object(crateId); + await ocflObject.load(); + return ocflObject; +} + +/** + * Finds the top-level collection a crate sits under by following memberOf up through the parent crates in the repository, + * so it doesn't depend on which crates are indexed yet. Each crate's parent is cached for the life of the finder. + */ +function topCollectionFinder() { + const parents = new Map>(); + const parentOf = (crateId: string) => { + let parent = parents.get(crateId); + if (!parent) { + parent = loadObject(crateId) + .then(readCrateJson) + .then((json) => { + const root = rawRoot(json, crateId); + return root && parentCollection(root, crateId); + }); + parents.set(crateId, parent); + } + return parent; + }; + return async (crate: ROCrate) => { + let top = crate.rootId; + let parent = parentCollection(crate.root, crate.rootId); + if (!parents.has(top)) { + parents.set(top, Promise.resolve(parent)); + } + const seen = new Set([top]); + while (parent && !seen.has(parent)) { + seen.add(parent); + try { + const next = await parentOf(parent); + top = parent; + parent = next; + } catch (error) { + log.warn({ err: error, crateId: crate.rootId, parent, topCollection: top }, 'Cannot read parent collection; using the highest found'); + return top; + } + } + if (parent) { + log.warn({ crateId: crate.rootId, cycle: [...seen, parent], topCollection: top }, 'Collections are members of each other; using the highest found'); + } + return top; + }; +} + +async function indexObject(crateId: string, types: IndexType[], findTopCollection: ReturnType, force?: boolean) { try { - const ocflObject = repository.object(crateId); - await ocflObject.load(); + const ocflObject = await loadObject(crateId); + const crate = await ROCrate.create(await readCrateJson(ocflObject)); log.debug({ crateId: ocflObject.id }, 'Found OCFL object'); - const jsonContent = await ocflObject.getFile({ logicalPath: 'ro-crate-metadata.json' }).text(); - const rawCrate = JSON.parse(jsonContent); - const crate = await ROCrate.create(rawCrate); - for (const t of types) { - const indexer = INDEXER[t]; - if (indexer) { - setState(ocflObject.id, t, State.INDEXING); - try { - if (force) { - await indexer.delete(crate.rootId); - } - const crateObject = wrap(ocflObject); - await indexer.index({ crateObject, crate }); - } catch (error) { - log.error({ err: error, crateId: ocflObject.id, indexer: t }, 'Error indexing object'); - } finally { - setState(ocflObject.id, t); - } + if (force) { + setState(ocflObject.id, types, State.INDEXING); + try { + await deleteFrom(types, crate.rootId); + } catch { + setState(ocflObject.id, types); + return; + } + } + const topCollection = await findTopCollection(crate); + for (const t of INDEX_ORDER.filter((t) => types.includes(t))) { + setState(ocflObject.id, t, State.INDEXING); + try { + await INDEXER[t].index({ crateObject: wrap(ocflObject), crate, topCollection }); + } catch (error) { + log.error({ err: error, crateId: ocflObject.id, indexer: t }, 'Error indexing object'); + } finally { + setState(ocflObject.id, t); } } } catch (error) { @@ -185,26 +256,29 @@ async function idsMatching(pattern?: RegExp) { } /** Indexes `crateIds`, the crates matching it, or every crate, marking them all queued before indexing any */ -export async function createIndex(crateIds?: string | string[] | RegExp, type?: string | string[], force?: boolean) { +export async function createIndex(crateIds?: string | string[] | RegExp, type?: IndexType | IndexType[], force?: boolean) { log.debug('Indexing started'); - const types = type ? ([] as string[]).concat(type) : Object.keys(INDEXER); + const types = force ? withDocuments(typesOf(type)) : typesOf(type); // Given ids are queued before the first await, so callers see them in progress at once const ids = typeof crateIds === 'string' || Array.isArray(crateIds) ? [crateIds].flat() : await idsMatching(crateIds); for (const id of ids) { setState(id, types, State.QUEUED); } + const findTopCollection = topCollectionFinder(); for (const id of ids) { - await indexObject(id, types, force); + await indexObject(id, types, findTopCollection, force); } log.debug('Indexing finished'); } -export async function deleteIndex(crateId?: string, type?: string | string[]) { - const types = type ? ([] as string[]).concat(type) : Object.keys(INDEXER); - const indexers = types.map((t) => INDEXER[t]).filter((i) => !!i); +export async function deleteIndex(crateId?: string, type?: IndexType | IndexType[]) { + const types = withDocuments(typesOf(type)); setState(crateId, types, State.DELETING); - await Promise.allSettled(indexers.map((indexer) => indexer.delete(crateId))); - setState(crateId, types); + try { + await deleteFrom(types, crateId); + } finally { + setState(crateId, types); + } } export async function* objects(prefix?: string, refresh?: boolean) { @@ -217,9 +291,7 @@ export async function* objects(prefix?: string, refresh?: boolean) { for await (const ocflObject of repository) { try { const inv = await ocflObject.getInventory(); - const jsonContent = await ocflObject.getFile({ logicalPath: 'ro-crate-metadata.json' }).text(); - const jsonParsed = JSON.parse(jsonContent); - const name = jsonParsed['@graph'].find((e: RawEntity) => e['@id'] === inv.id)?.name?.toString(); + const name = rawRoot(await readCrateJson(ocflObject), inv.id)?.name?.toString(); yield { id: inv.id, name, path: ocflObject.root, state: await getState(inv.id) }; } catch (error) { log.error({ err: error, path: ocflObject.root }, 'Error reading object'); diff --git a/src/repository.ts b/src/repository.ts index 1fc72a5..93f01f3 100644 --- a/src/repository.ts +++ b/src/repository.ts @@ -9,21 +9,27 @@ export const State = { export type IndexState = (typeof State)[keyof typeof State]; +export const INDEX_TYPES = ['database', 'search'] as const; + +export type IndexType = (typeof INDEX_TYPES)[number]; + +/** Each index's state, absent or undefined while it is idle */ +export type IndexStates = Partial>; + /** Whether any index is in one of `values` */ -export const hasState = (state: Partial>, ...values: IndexState[]) => - Object.values(state).some((value) => value !== undefined && values.includes(value)); +export const hasState = (state: IndexStates, ...values: IndexState[]) => Object.values(state).some((value) => value !== undefined && values.includes(value)); /** A crate in the repository, with the state of each index that isn't idle */ -export type RepositoryObject = { id: string; path: string; name?: string | undefined; state: Partial> }; +export type RepositoryObject = { id: string; path: string; name?: string | undefined; state: IndexStates }; /** * Abstraction for repository operations, such as indexing and listing objects. This is used by the API routes to interact with the underlying data store, which can be an OCFL storage or a database. */ export interface Repository { init(): Promise; - getState(crateId: string, type?: string): Promise>; - createIndex(crateIds?: string | string[] | RegExp, type?: string, force?: boolean): Promise; - deleteIndex(crateId?: string, type?: string | string[]): Promise; + getState(crateId: string, type?: IndexType): Promise; + createIndex(crateIds?: string | string[] | RegExp, type?: IndexType | IndexType[], force?: boolean): Promise; + deleteIndex(crateId?: string, type?: IndexType | IndexType[]): Promise; objects(prefix?: string, refresh?: boolean): AsyncIterable; storage(): StorageBackend; } diff --git a/src/root-collection.test.ts b/src/root-collection.test.ts new file mode 100644 index 0000000..5a75b5d --- /dev/null +++ b/src/root-collection.test.ts @@ -0,0 +1,51 @@ +import { beforeAll, describe, expect, it } from 'vitest'; +import { ocflTestbed } from '../test/app.ts'; + +const BUNDLED = 'arcp://name,bundled'; +const ROOT = 'arcp://name,distributed_root'; +const CONTENT = 'arcp://name,distributed_content_restricted'; +const METADATA = 'arcp://name,distributed_metadata_restricted'; + +const testbed = ocflTestbed(); + +const rootCollectionOf = async (id: string) => { + const { app, config } = testbed.started; + const response = await app.inject({ method: 'GET', url: `${config.prefix}/entity/${encodeURIComponent(id)}` }); + expect(response.statusCode).toBe(200); + return response.json().rootCollection?.id ?? null; +}; + +const collectionFacet = async () => (await testbed.search({ query: '' })).facets.rootCollection.map(({ name }: { name: string }) => name); + +describe.each([ + ['before', [ROOT, CONTENT, METADATA, BUNDLED]], + ['after', [CONTENT, METADATA, ROOT, BUNDLED]], +])('rootCollection, with the distributed parent crate indexed %s its children', (_order, crates) => { + beforeAll(async () => { + await testbed.started.repository.deleteIndex(); + await testbed.started.repository.createIndex(crates); + }); + + it('is empty on a top-level collection', async () => { + expect(await rootCollectionOf(ROOT)).toBeNull(); + expect(await rootCollectionOf(BUNDLED)).toBeNull(); + }); + + it("is the distributed collection's root for the entities of its nested crates", async () => { + for (const id of [CONTENT, `${CONTENT}/main`, METADATA, `${METADATA}/main`]) { + expect(await rootCollectionOf(id)).toBe(ROOT); + } + }); + + it("is a stand-alone crate's root for its other entities", async () => { + expect(await rootCollectionOf(`${BUNDLED}/ch1`)).toBe(BUNDLED); + }); + + it('lists the distributed collection once in the Collection facet', async () => { + const names = await collectionFacet(); + + expect(names.filter((name: string) => name === ROOT)).toHaveLength(1); + expect(names).not.toContain(CONTENT); + expect(names).not.toContain(METADATA); + }); +}); diff --git a/src/search-total.test.ts b/src/search-total.test.ts new file mode 100644 index 0000000..c1317c9 --- /dev/null +++ b/src/search-total.test.ts @@ -0,0 +1,85 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { beforeAll, describe, expect, it, vi } from 'vitest'; +import { ocflTestbed } from '../test/app.ts'; + +const logs = vi.hoisted(() => [] as Record[]); + +vi.mock('./logger.ts', async () => (await import('../test/logs.ts')).capturingLogger(logs, 'info')); + +const ROOT = 'arcp://name,unwritable'; +const WRITTEN = `${ROOT}/written`; +// Postgres can't store a NUL character, so this entity's row fails to be written +const UNWRITTEN = `${ROOT}/unwritten`; +const LICENSE = { '@id': 'https://creativecommons.org/licenses/by/4.0/' }; +const OBJECT = { '@id': 'https://w3id.org/ldac/profile#Object' }; + +const crate = { + '@context': ['https://w3id.org/ro/crate/1.1/context', 'https://w3id.org/ldac/context', { '@vocab': 'http://schema.org/' }], + '@graph': [ + { '@id': 'ro-crate-metadata.json', '@type': 'CreativeWork', license: LICENSE, about: { '@id': ROOT } }, + { + '@id': ROOT, + '@type': ['Dataset', 'RepositoryCollection'], + conformsTo: { '@id': 'https://w3id.org/ldac/profile#Collection' }, + name: 'Unwritable', + license: LICENSE, + }, + { '@id': WRITTEN, '@type': 'RepositoryObject', conformsTo: OBJECT, memberOf: { '@id': ROOT }, name: 'Written' }, + { '@id': UNWRITTEN, '@type': 'RepositoryObject', conformsTo: OBJECT, memberOf: { '@id': ROOT }, name: 'Unwritten\u0000' }, + ], +}; + +const testbed = ocflTestbed({ + source: async (scratch) => { + const source = join(scratch, 'crates'); + await mkdir(join(source, 'unwritable'), { recursive: true }); + await writeFile(join(source, 'unwritable', 'ro-crate-metadata.json'), JSON.stringify(crate)); + return source; + }, +}); +const { search } = testbed; + +beforeAll(async () => { + await testbed.started.repository.deleteIndex(); + await testbed.started.repository.createIndex(ROOT); +}); + +describe('an entity whose row fails to be written', () => { + it('is not counted by search', async () => { + const { total, entities, facets } = await search({ query: '', limit: 100 }); + + expect(entities.map(({ id }: { id: string }) => id).sort()).toEqual([ROOT, WRITTEN]); + expect(total).toBe(2); + expect(facets.recordType.reduce((sum: number, { count }: { count: number }) => sum + count, 0)).toBe(2); + }); + + it('is not counted when the search index is rebuilt alone', async () => { + await testbed.started.repository.createIndex(ROOT, 'search', true); + + const { total, entities } = await search({ query: '', limit: 100 }); + + expect(entities.map(({ id }: { id: string }) => id).sort()).toEqual([ROOT, WRITTEN]); + expect(total).toBe(2); + }); + + it('logs the skip with the crate and entity ids', () => { + expect(logs).toContainEqual(expect.objectContaining({ level: 40, crateId: ROOT, entityId: UNWRITTEN, msg: 'Skipped: no database row' })); + }); + + it('logs the documents it indexed', () => { + expect(logs).toContainEqual(expect.objectContaining({ module: 'indexer/search', crateId: ROOT, documents: 2, msg: 'Indexed crate' })); + }); +}); + +describe('a crate with no database rows', () => { + it('logs the skip once for the crate', async () => { + await testbed.started.repository.deleteIndex(ROOT); + logs.length = 0; + + await testbed.started.repository.createIndex(ROOT, 'search'); + + const skips = logs.filter(({ msg }) => typeof msg === 'string' && msg.startsWith('Skipped')); + expect(skips).toEqual([expect.objectContaining({ level: 40, crateId: ROOT, skipped: 3, msg: 'Skipped crate: no database rows' })]); + }); +}); diff --git a/src/search.test.ts b/src/search.test.ts new file mode 100644 index 0000000..f4b0b6b --- /dev/null +++ b/src/search.test.ts @@ -0,0 +1,53 @@ +import { beforeAll, describe, expect, it } from 'vitest'; +import { ocflTestbed } from '../test/app.ts'; + +const OBJECT = 'http://pcdm.org/models#Object'; +const MEDIA = 'http://schema.org/MediaObject'; +const BUNDLED = 'arcp://name,bundled'; + +const testbed = ocflTestbed(); +const { search } = testbed; + +beforeAll(async () => { + await testbed.started.repository.deleteIndex(); + await testbed.started.repository.createIndex(BUNDLED); +}); + +describe('search by recordType and entityType', () => { + it('declares a recordType facet, and entityType as a filter only', async () => { + const response = await testbed.started.app.inject({ method: 'GET', url: `${testbed.started.config.prefix}/capabilities` }); + + const { facets, filters } = response.json().search; + expect(facets).toHaveProperty('recordType'); + expect(facets).not.toHaveProperty('entityType'); + expect(facets).not.toHaveProperty('type'); + expect(filters).toHaveProperty('entityType'); + }); + + it("facets on a readable label for each entity's class", async () => { + const { facets } = await search({ query: '' }); + + expect(facets).not.toHaveProperty('entityType'); + const names = facets.recordType.map(({ name }: { name: string }) => name); + expect(names).toEqual(expect.arrayContaining(['Collection', 'Object', 'File'])); + expect(names.some((name: string) => name.includes('://'))).toBe(false); + }); + + it('filters on recordType', async () => { + const { total, entities } = await search({ query: '', filters: { recordType: ['File'] }, limit: 100 }); + + expect(total).toBeGreaterThan(0); + expect(entities.every(({ entityType }: { entityType: string }) => entityType === MEDIA)).toBe(true); + }); + + it('filters on entityType, agreeing with listing entities by entityType', async () => { + const { total, entities } = await search({ query: '', filters: { entityType: [OBJECT] }, limit: 100 }); + const listed = ( + await testbed.started.app.inject({ method: 'GET', url: `${testbed.started.config.prefix}/entities?entityType=${encodeURIComponent(OBJECT)}&limit=100` }) + ).json(); + + expect(total).toBeGreaterThan(0); + expect(entities.every(({ entityType }: { entityType: string }) => entityType === OBJECT)).toBe(true); + expect(entities.map(({ id }: { id: string }) => id).sort()).toEqual(listed.entities.map(({ id }: { id: string }) => id).sort()); + }); +}); diff --git a/src/utils.ts b/src/utils.ts index bc1b51c..753ae31 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,6 @@ -export function firstStringOrId(values: unknown[]): string | undefined { - for (const value of values || []) { +/** The first string or `@id` of a JSON-LD value, which may be a single value or an array */ +export function firstStringOrId(values: unknown): string | undefined { + for (const value of [values ?? []].flat()) { if (typeof value === 'string') { return value; } else if (typeof value === 'object' && value !== null && '@id' in value && typeof value['@id'] === 'string') { diff --git a/test/admin.ts b/test/admin.ts index c7951e5..849ff84 100644 --- a/test/admin.ts +++ b/test/admin.ts @@ -1,5 +1,5 @@ import { afterAll, beforeAll, vi } from 'vitest'; -import { initRepository, type Repository } from '../src/repository.ts'; +import type { Repository } from '../src/repository.ts'; import { buildTestApp, type TestApp } from './app.ts'; import { startFakeUpstream, type UpstreamUser } from './fake-upstream.ts'; import { Browser } from './oidc.ts'; @@ -19,9 +19,6 @@ const oidcEnv = (upstreamIssuer: string) => ({ OIDC_UPSTREAM_CLIENT_SECRET: UPSTREAM_CLIENT.clientSecret, }); -/** The OCFL repository at `OCFL_ROOT` */ -export const ocflRepository = (deps: Record) => initRepository('ocfl', deps); - /** A repository of `crates`, each with its index state, recording what it is asked to do */ export const repositoryOf = (crates: { id: string; name?: string; state: Record }[]) => { const fake = { @@ -39,7 +36,7 @@ export const repositoryOf = (crates: { id: string; name?: string; state: Record< }; /** A repository with one crate, `PEOPLE`, whose search index is in `state` */ -export const repositoryIn = (state: string) => repositoryOf([{ id: PEOPLE, name: 'People', state: { structural: 'indexed', search: state } }]); +export const repositoryIn = (state: string) => repositoryOf([{ id: PEOPLE, name: 'People', state: { database: 'indexed', search: state } }]); /** Runs a fake upstream provider for a file's tests, then closes it and every app started with `start` */ export const adminTestbed = () => { diff --git a/test/app.ts b/test/app.ts index 762b168..8fc3b14 100644 --- a/test/app.ts +++ b/test/app.ts @@ -1,8 +1,12 @@ +import { execFileSync } from 'node:child_process'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { Readable } from 'node:stream'; import type { Client } from '@opensearch-project/opensearch'; -import { vi } from 'vitest'; +import { afterAll, beforeAll, expect, vi } from 'vitest'; import type { createSearchBackend } from '../src/indexer/search.ts'; -import type { Repository } from '../src/repository.ts'; +import { initRepository, type Repository } from '../src/repository.ts'; type RepositoryDeps = { opensearchClient: Client; search: ReturnType }; @@ -12,13 +16,21 @@ export type TestRepository = Repository | ((deps: RepositoryDeps) => Repository /** A repository with no content */ export const repository = { storage: () => ({ content: {} }), async *objects() {} } as unknown as Repository; -/** A repository whose every file holds `content` */ +/** A repository whose every file holds `content`, served as the file's media type */ export const repositoryWithContent = { storage: () => ({ - content: () => ({ type: 'stream', open: () => Readable.from(['content']), contentLength: 7, contentType: 'text/plain' }), + content: (file: { mediaType?: string }) => ({ + type: 'stream', + open: () => Readable.from(['content']), + contentLength: 7, + contentType: file.mediaType ?? 'text/plain', + }), }), } as unknown as Repository; +/** The OCFL repository at `OCFL_ROOT` */ +export const ocflRepository = (deps: Record) => initRepository('ocfl', deps); + /** Builds the app with `env` set while configuration is read, which happens on first import */ export const buildTestApp = async (env: Record, options: { repository?: TestRepository } = {}) => { for (const [name, value] of Object.entries(env)) { @@ -35,10 +47,60 @@ export const buildTestApp = async (env: Record, options: { repos const repo = typeof options.repository === 'function' ? await options.repository({ opensearchClient, search }) : (options.repository ?? repository); /** Builds another app, as another server process would */ const build = () => buildApp({ repository: repo, search }); - return { app: build(), build, config, prisma, repository: repo }; + return { app: build(), build, config, prisma, repository: repo, search }; } finally { vi.unstubAllEnvs(); } }; export type TestApp = Awaited>['app']; + +/** + * A new OCFL repository for a file's tests, imported from the directory `source` returns (test-data by default), + * which it may create under the scratch directory it is given + */ +export const ocflStorage = ({ source }: { source?: (scratch: string) => Promise } = {}) => { + let storage: string; + const ocflRoot = () => join(storage, 'ocfl'); + + beforeAll(async () => { + storage = await mkdtemp(join(tmpdir(), 'ldacapi-')); + const sources = source ? [await source(storage)] : []; + execFileSync('node', ['scripts/crates-import.ts', ...sources], { env: { ...process.env, OCFL_ROOT: ocflRoot(), LOG_LEVEL: 'silent' }, stdio: 'ignore' }); + }); + + afterAll(async () => { + await rm(storage, { recursive: true, force: true }); + }); + + return { ocflRoot }; +}; + +/** The app over a new OCFL repository (see `ocflStorage`), its indexes deleted after the file's tests */ +export const ocflTestbed = (options: Parameters[0] = {}) => { + const { ocflRoot } = ocflStorage(options); + let started: Awaited>; + + beforeAll(async () => { + started = await buildTestApp({ OCFL_ROOT: ocflRoot(), LOG_LEVEL: 'silent' }, { repository: ocflRepository }); + }); + + afterAll(async () => { + await started?.repository.deleteIndex(); + await started?.app.close(); + await started?.prisma.$disconnect(); + }); + + return { + get started() { + return started; + }, + + /** POSTs `body` to the search endpoint, expecting success */ + async search(body: Record) { + const response = await started.app.inject({ method: 'POST', url: `${started.config.prefix}/search`, payload: body }); + expect(response.statusCode).toBe(200); + return response.json(); + }, + }; +}; diff --git a/test/bootstrap.test.ts b/test/bootstrap.test.ts index d5eef81..c82f780 100644 --- a/test/bootstrap.test.ts +++ b/test/bootstrap.test.ts @@ -3,8 +3,9 @@ import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { Client } from '@opensearch-project/opensearch'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; import { config } from '../src/configuration.ts'; +import { countDocuments, createSearchBackend } from '../src/indexer/search.ts'; import { prisma } from '../src/prisma.ts'; const crateIds = [ @@ -14,13 +15,13 @@ const crateIds = [ 'arcp://name,distributed_metadata_restricted', ]; -describe('seed and index', () => { +describe('import and index', () => { let storage: string; beforeAll(async () => { storage = await mkdtemp(join(tmpdir(), 'ldacapi-')); const env = { ...process.env, OCFL_ROOT: join(storage, 'ocfl'), LOG_LEVEL: 'silent' }; - execFileSync('node', ['scripts/seed.ts'], { env, stdio: 'ignore' }); + execFileSync('node', ['scripts/crates-import.ts'], { env, stdio: 'ignore' }); execFileSync('node', ['scripts/index.ts'], { env, stdio: 'ignore' }); }); @@ -37,14 +38,15 @@ describe('seed and index', () => { it('indexes every crate into OpenSearch', async () => { const opensearch = new Client({ node: config.opensearchUrl }); - const indexed = await Promise.all( - crateIds.map(async (id) => { - const { body } = await opensearch.count({ index: config.search.entityIndex, body: { query: { term: { rootCollection: id } } } }); - return body.count > 0; - }), - ); + const search = createSearchBackend(opensearch); + const rows = await prisma.entity.findMany({ select: { id: true } }); + const ids = rows.map(({ id }) => id); + expect(ids).toEqual(expect.arrayContaining(crateIds)); + + // Documents are searchable after OpenSearch's next refresh + await vi.waitFor(async () => { + expect(await countDocuments(search, ids)).toBe(ids.length); + }); await opensearch.close(); - - expect(indexed).toEqual(crateIds.map(() => true)); }); }); diff --git a/test/logs.ts b/test/logs.ts new file mode 100644 index 0000000..d07d7d7 --- /dev/null +++ b/test/logs.ts @@ -0,0 +1,13 @@ +import { Writable } from 'node:stream'; +import { pino } from 'pino'; + +/** The module `vi.mock('./logger.ts', …)` returns: a logger at `level` that writes each record into `logs` */ +export const capturingLogger = (logs: Record[], level: string) => { + const stream = new Writable({ + write(chunk, _encoding, done) { + logs.push(JSON.parse(chunk.toString())); + done(); + }, + }); + return { logger: pino({ level }, stream) }; +};