Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,6 @@
# OPENSEARCH_URL=http://localhost:9200
# OCFL_ROOT=storage/ocfl

# Working directory for OCFL writes. Defaults to `scratch` beside OCFL_ROOT
# OCFL_SCRATCH=storage/scratch

# LDACAPI_PORT=8080
# LDACAPI_HOST=localhost
# LDACAPI_MAX_PARAM_LENGTH=500
Expand Down
19 changes: 9 additions & 10 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,6 @@ set.
| `DATABASE_URL` | `postgresql://ldacapi:ldacapi@localhost:5432/ldacapi` |
| `OPENSEARCH_URL` | `http://localhost:9200` |
| `OCFL_ROOT` | `storage/ocfl` |
| `OCFL_SCRATCH` | `scratch` beside `OCFL_ROOT` |
| `LDACAPI_PORT` | `8080` |
| `LDACAPI_HOST` | `localhost` |
| `LDACAPI_MAX_PARAM_LENGTH` | `500` |
Expand Down Expand Up @@ -352,11 +351,10 @@ docker run --rm ldacapi pnpm generate-keys >> ldacapi.env
### Storage

The image has no default `OCFL_ROOT`. Mount your storage and set `OCFL_ROOT`
(and `OCFL_SCRATCH`, if you need it elsewhere) to paths inside the container
(above, `OCFL_ROOT=/storage/ocfl`). The API runs as the
image's `node` user, uid 1000, so the OCFL repository must be readable by uid
1000 and the scratch directory writable by it. `pnpm crates:import` writes to the
repository, so give it write access too when you run it.
to a path inside the container (above, `OCFL_ROOT=/storage/ocfl`). The API runs
as the image's `node` user, uid 1000, so the OCFL repository must be readable by
uid 1000. `pnpm crates:import` writes to the repository, so give it write access
too when you run it.

## Production with Docker Compose

Expand Down Expand Up @@ -399,10 +397,11 @@ sudo sysctl --system
```

Storage is a directory on the host, usually an NFS mount, set by `STORAGE_DIR`
in `.env`. The OCFL repository is its `ocfl` directory, and imports are staged
in its `scratch` directory. Both must be in the one mount: OCFL commits an
import by renaming it from scratch into the repository, and a rename can't
cross mounts. The API runs as uid 1000, so the directory must be writable by
in `.env`. The OCFL repository is its `ocfl` directory. Imports are staged
inside the repository, in `extensions/ocfl-js-workspace`, and committed by
renaming them into place, so the repository must be on one filesystem, with no
mount point inside it, and that filesystem must support hard links, as NFS
does. The API runs as uid 1000, so the directory must be writable by
uid 1000: on NFS, set the owner on the server or map the export to uid 1000, as
root in the container is squashed. Otherwise `pnpm ocfl:init` fails with
`EACCES: permission denied, mkdir '/storage/ocfl'`. Serving and indexing only
Expand Down
4 changes: 2 additions & 2 deletions deploy/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ ACME_EMAIL=ops@example.org
# Generate with: openssl rand -hex 32
POSTGRES_PASSWORD=

# The directory holding the OCFL repository (its `ocfl` directory) and scratch (`scratch`),
# e.g. an NFS mount. It must be writable by uid 1000 to import crates.
# The directory holding the OCFL repository (its `ocfl` directory), e.g. an NFS mount.
# It must be writable by uid 1000 to import crates.
STORAGE_DIR=/mnt/ldacapi

# OpenSearch's JVM heap; about half the memory you can give it
Expand Down
1 change: 0 additions & 1 deletion deploy/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ services:
DATABASE_URL: postgresql://ldacapi:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@db:5432/ldacapi
OPENSEARCH_URL: http://opensearch:9200
OCFL_ROOT: /storage/ocfl
OCFL_SCRATCH: /storage/scratch
OIDC_ISSUER: https://${DOMAIN:?Set DOMAIN in .env}
ADMIN_ORIGIN: https://${ADMIN_DOMAIN:?Set ADMIN_DOMAIN in .env}
OIDC_CLIENTS: >-
Expand Down
5 changes: 2 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,11 @@
"@fastify/sensible": "^6.0.7",
"@fastify/static": "^10.1.7",
"@fastify/view": "^12.0.1",
"@ocfl/ocfl-fs": "^0.4.0",
"@ocfl/ocfl-fs": "1.0.0-beta.59",
"@opensearch-project/opensearch": "^3.9.0",
"@prisma/adapter-pg": "^7.10.1",
"@prisma/client": "^7.10.1",
"arocapi": "5.0.0-beta.9",
"arocapi": "5.0.0-beta.10",
"eta": "^4.6.0",
"fastify": "^5.12.7",
"jose": "^6.2.13",
Expand All @@ -73,7 +73,6 @@
"@biomejs/biome": "^2.5.15",
"@commitlint/cli": "^21.2.4",
"@commitlint/config-conventional": "^21.2.4",
"@ocfl/ocfl": "^0.4.0",
"@tailwindcss/cli": "^4.3.3",
"@types/node": "^26.6.6",
"@types/oidc-provider": "^9.12.1",
Expand Down
49 changes: 13 additions & 36 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion pnpm-workspace.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,5 @@ allowBuilds:
# ships prebuilt binaries; its install script only builds from source
"@parcel/watcher": false
minimumReleaseAgeExclude:
- arocapi@5.0.0-beta.9
- arocapi@5.0.0-beta.10
- "@ocfl/ocfl-fs@1.0.0-beta.59"
33 changes: 29 additions & 4 deletions scripts/crates-import.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,44 @@
import { readdir, readFile } from 'node:fs/promises';
import { join, resolve } from 'node:path';
import { OcflError, type StorageRoot, type VersionDetails } from '@ocfl/ocfl-fs';
import { ROCrate } from 'ro-crate';
import { config } from '../src/configuration.ts';
import { logger } from '../src/logger.ts';
import { openOrCreateStorage } from '../src/ocfl.ts';
import { openOrCreateRepository } from '../src/ocfl-repository.ts';

const source = resolve(process.argv[2] ?? 'test-data');

const repository = await openOrCreateStorage();
/** Merges the crate at `path` into its OCFL object, creating the object on first import */
async function importCrate(repository: StorageRoot, crateId: string, path: string) {
const details: VersionDetails = {
message: `Imported from ${path}`,
user: { name: 'ldacapi crates:import', address: 'https://github.com/Language-Research-Technology/ldacapi' },
};
try {
return await repository.create(crateId, details, (draft) => draft.import(path));
} catch (error) {
if (!(error instanceof OcflError && error.code === 'OBJECT_EXISTS')) {
throw error;
}
}
const object = await repository.open(crateId);
return object.update(details, (draft) => draft.import(path));
}

const repository = await openOrCreateRepository();

const directories = await readdir(source, { withFileTypes: true });
for (const directory of directories.filter((entry) => entry.isDirectory())) {
const path = join(source, directory.name);
const crate = await ROCrate.create(JSON.parse(await readFile(join(path, 'ro-crate-metadata.json'), 'utf-8')));
await repository.object(crate.rootId).import(path);
logger.info({ crateId: crate.rootId, path }, 'Imported crate');
try {
const object = await importCrate(repository, crate.rootId, path);
logger.info({ crateId: crate.rootId, path, version: object.head.number }, 'Imported crate');
} catch (error) {
// A crate another writer holds, or one that needs recovery, is left for a person; the rest still import
const lockedAt = error instanceof OcflError ? error.lockedAt : undefined;
logger.error({ err: error, crateId: crate.rootId, path, lockedAt }, 'Cannot import crate');
process.exitCode = 1;
}
}
logger.info({ root: config.ocfl.root }, 'Imported crates into the OCFL repository');
4 changes: 2 additions & 2 deletions scripts/ocfl-init.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { config } from '../src/configuration.ts';
import { logger } from '../src/logger.ts';
import { openOrCreateStorage } from '../src/ocfl.ts';
import { openOrCreateRepository } from '../src/ocfl-repository.ts';

await openOrCreateStorage();
await openOrCreateRepository();
logger.info({ root: config.ocfl.root }, 'The OCFL repository is ready');
7 changes: 1 addition & 6 deletions src/configuration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ describe('loadConfig', () => {
expect(config.databaseUrl).toBe('postgresql://ldacapi:ldacapi@localhost:5432/ldacapi');
expect(config.opensearchUrl).toBe('http://localhost:9200');
expect(config.port).toBe(8080);
expect(config.ocfl).toEqual({ root: resolve('storage/ocfl'), scratch: resolve('storage/scratch') });
expect(config.ocfl).toEqual({ root: resolve('storage/ocfl') });
});

it('uses the test database and index, and logs nothing, in test', () => {
Expand All @@ -49,11 +49,6 @@ describe('loadConfig', () => {
expect(config.search.entityIndex).toBe('other');
});

it('puts OCFL scratch beside the OCFL root unless set', () => {
expect(loadConfig({ OCFL_ROOT: '/data/ocfl' }).ocfl).toEqual({ root: '/data/ocfl', scratch: '/data/scratch' });
expect(loadConfig({ OCFL_ROOT: '/data/ocfl', OCFL_SCRATCH: '/tmp/ocfl' }).ocfl.scratch).toBe('/tmp/ocfl');
});

it('treats empty values as unset', () => {
expect(loadConfig({ OPENSEARCH_URL: '' }).opensearchUrl).toBe('http://localhost:9200');
});
Expand Down
4 changes: 1 addition & 3 deletions src/configuration.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { createPrivateKey, createPublicKey, sign, verify } from 'node:crypto';
import { dirname, join, resolve } from 'node:path';
import { resolve } from 'node:path';
import * as z from 'zod';
import { generateCookieKey, generateSigningKey, type Jwk, SIGNING_ALG } from './oidc/keys.ts';

Expand Down Expand Up @@ -118,7 +118,6 @@ export function loadConfig(processEnv: Env) {
DATABASE_URL: devDefault(`postgresql://ldacapi:ldacapi@localhost:5432/ldacapi${nodeEnv === 'test' ? '_test' : ''}`),
OPENSEARCH_URL: devDefault('http://localhost:9200'),
OCFL_ROOT: devDefault('storage/ocfl'),
OCFL_SCRATCH: z.string().optional(),
LDACAPI_PORT: z.coerce.number().int().default(8080),
LDACAPI_HOST: z.string().default('localhost'),
LDACAPI_MAX_PARAM_LENGTH: z.coerce.number().int().default(500),
Expand Down Expand Up @@ -193,7 +192,6 @@ export function loadConfig(processEnv: Env) {
maxParamLength: env.LDACAPI_MAX_PARAM_LENGTH,
ocfl: {
root: resolve(env.OCFL_ROOT),
scratch: resolve(env.OCFL_SCRATCH ?? join(dirname(env.OCFL_ROOT), 'scratch')),
},
defaultLicense: env.DEFAULT_LICENSE,
defaultMetadataLicense: env.DEFAULT_METADATA_LICENSE,
Expand Down
33 changes: 33 additions & 0 deletions src/damaged-crate.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { beforeAll, describe, expect, it } from 'vitest';
import { ocflTestbed } from '../test/app.ts';

const PEOPLE = 'arcp://name,people';
const BUNDLED = 'arcp://name,bundled';

const testbed = ocflTestbed();

describe('a crate whose OCFL object is damaged', () => {
beforeAll(async () => {
const { config, repository } = testbed.started;
expect(config.ocfl.root.startsWith(tmpdir())).toBe(true);
await writeFile(join(config.ocfl.root, 'arcp_name_people', '__object__', 'inventory.json'), 'not json');
await repository.deleteIndex();
await repository.createIndex();
});

it('is left out when every crate is indexed, and the rest are indexed', async () => {
const ids = (await testbed.started.prisma.entity.findMany({ where: { id: { in: [PEOPLE, BUNDLED] } }, select: { id: true } })).map(({ id }) => id);

expect(ids).toEqual([BUNDLED]);
});

it('is left out of the list of crates', async () => {
const ids = (await Array.fromAsync(testbed.started.repository.objects())).map(({ id }) => id);

expect(ids).toContain(BUNDLED);
expect(ids).not.toContain(PEOPLE);
});
});
3 changes: 1 addition & 2 deletions src/indexer/database.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,13 @@ describe('DatabaseIndexer', () => {
});
const crateObject: CrateObject = {
id: 'database-test',
root: '/nowhere',
text: async () => '',
file: async (path) => {
const size = inventory[path];
if (size === undefined) {
throw new Error(`Not in inventory: ${path}`);
}
return { size, crc32: '' };
return { size };
},
};
await indexer.index({ crateObject, crate, topCollection: ROOT });
Expand Down
2 changes: 1 addition & 1 deletion src/indexer/database.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ export class DatabaseIndexer extends Indexer {
filename: (ocflPath ?? entityId).split('/').pop(),
mediaType: entity.encodingFormat?.find((v: unknown) => typeof v === 'string') || 'application/octet-stream',
size: f?.size ?? statedSize(entity.contentSize?.[0]),
meta: ocflPath ? ({ ocflObject: crateObject.id, ocflPath, crc32: f?.crc32 ?? '' } satisfies OcflStorageMeta & { crc32: string }) : {},
meta: ocflPath ? ({ ocflObject: crateObject.id, ocflPath } satisfies OcflStorageMeta) : {},
};
}
try {
Expand Down
Loading
Loading