diff --git a/.env.example b/.env.example index e5abb2cb..07a0b783 100644 --- a/.env.example +++ b/.env.example @@ -37,6 +37,9 @@ S3_ENDPOINT=http://localhost:${PT_S3_PORT} # Enabled plugins. Defaults to core,site, which are the only plugins that exist so far. # core is always required. PT_PLUGINS=core,site +# The languages in use, chosen from those registered in apps/cms/src/locales.ts: en, fr, pt, ar, +# sw. English is always on. Start-up stops on a code that is not registered. Defaults to en. +# PT_LOCALES=en,fr # Chooses the verification adapter when more than one enabled plugin provides one. With only # core enabled, the built-in `native` adapter is used and this can stay unset. # PT_VERIFICATION=native diff --git a/AGENTS.md b/AGENTS.md index 871875ca..2d6c8380 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,6 +2,7 @@ PromiseTracker v3. The system specification in `docs/spec/promisetracker-v3-system-specification.md` is the source of truth for behaviour. Work is tracked as issues in the "v3: version 1" milestone. +Issue #672 gives the build order and the owner of each issue. ## Layout @@ -43,6 +44,50 @@ is the source of truth for behaviour. Work is tracked as issues in the "v3: vers - Next.js 16 ships its own docs in `apps/site/node_modules/next/dist/docs/`. Read them before relying on remembered APIs. +## Working on an issue + +1. Find the issue. Issue #672 lists every issue in build order, with its owner. Work only on + issues assigned to the person you are working for, and only on ones labelled `status: ready`. +2. Read the issue's "Start here" section. It names the specification sections to read, where the + code goes, what already exists, and any v2 code to reuse. Read v2 code with + `git show v2-final:`. v2 is not in the working tree. +3. Read those specification sections before writing code. If the issue and the specification + disagree, the specification wins. Say so in the pull request. +4. Branch from `main`: `feat/v3-`, or `docs/...`, `fix/...`, `chore/...`. +5. Build only what the acceptance criteria ask for. Each criterion needs a test, or a line in the + pull request saying how it was checked. +6. After changing collections or fields: `pnpm generate`, then `pnpm payload migrate:create `, + then `pnpm migrate:check`. +7. Before opening the pull request: `pnpm lint`, `pnpm typecheck` and `pnpm test`. The last needs + `pnpm services` running. +8. The pull request says `Closes #` and follows the format of #663 and #664: what it does, + decisions made while building, environment and config changes, migrations, how to test. +9. A decision that changes behaviour also goes in the specification's decision register, section + 20.1, with the PDFs rebuilt. +10. After the merge, look at the issue's "Unblocks" section. Move each issue listed there to + `status: ready` if nothing else open blocks it. + +Where things are: + +| What | Where | +| ------------------------------------------------ | ----------------------------- | +| Ports, the plugin helper, start-up checks | `packages/kernel/src` | +| Record types and default adapters | `plugins/core/src` | +| Settings for the default public site | `plugins/site/src` | +| Payload config, migrations, generated types | `apps/cms/src` | +| `/api/v1` handlers | `apps/cms/src/api/v1.ts` | +| Public API response schemas | `packages/contract/src` | +| The typed client the site uses | `packages/sdk/src` | +| Components, props only | `packages/ui/src` | +| Public pages | `apps/site/src/app/(site)` | +| Admin, REST, `/api/v1`, health and plugin routes | `apps/site/src/app/(payload)` | +| Registered languages | `apps/cms/src/locales.ts` | +| Every environment key | `.env.example` | + +A new plugin is a package in `plugins/`. It is also added to `firstPartyPlugins` in +`apps/cms/src/kernel.ts`, to `transpilePackages` in `apps/site/next.config.ts`, and to the `COPY` +lines for `package.json` files in `docker/Dockerfile`. + ## Style - Biome formats and lints: two spaces, single quotes, no semicolons, 100 columns. diff --git a/apps/cms/src/kernel.ts b/apps/cms/src/kernel.ts index 7f43d752..8b768d1b 100644 --- a/apps/cms/src/kernel.ts +++ b/apps/cms/src/kernel.ts @@ -2,6 +2,8 @@ import { createKernel } from '@promisetracker/kernel' import { corePlugin } from '@promisetracker/plugin-core' import { sitePlugin } from '@promisetracker/plugin-site' +import { defaultLocale, registeredLocales } from './locales' + /** * Every first-party plugin, enabled or not. Their record types and fields are * always in the schema (D29). PT_PLUGINS decides which ones act. @@ -9,4 +11,8 @@ import { sitePlugin } from '@promisetracker/plugin-site' export const firstPartyPlugins = [corePlugin, sitePlugin] as const /** Built once per process. A configuration problem stops start-up here. */ -export const kernel = createKernel({ plugins: firstPartyPlugins }) +export const kernel = createKernel({ + plugins: firstPartyPlugins, + locales: registeredLocales, + defaultLocale, +}) diff --git a/apps/cms/src/locales.ts b/apps/cms/src/locales.ts new file mode 100644 index 00000000..02f38d72 --- /dev/null +++ b/apps/cms/src/locales.ts @@ -0,0 +1,16 @@ +import type { LocaleSpec } from '@promisetracker/kernel' + +/** + * Every language PromiseTracker can run in (section 7.1, D45). The database + * stores this list, so adding one here needs a migration. PT_LOCALES chooses + * which of them a deployment uses, and never changes the schema. + */ +export const registeredLocales = [ + { code: 'en', label: 'English' }, + { code: 'fr', label: 'Français' }, + { code: 'pt', label: 'Português' }, + { code: 'ar', label: 'العربية', rtl: true }, + { code: 'sw', label: 'Kiswahili' }, +] as const satisfies readonly LocaleSpec[] + +export const defaultLocale = 'en' diff --git a/docker/compose.yaml b/docker/compose.yaml index 7bc7b20d..633df9aa 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -25,6 +25,7 @@ x-app: &app S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-promisetracker} S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-promisetracker} PT_PLUGINS: ${PT_PLUGINS:-core,site} + PT_LOCALES: ${PT_LOCALES:-} PT_VERIFICATION: ${PT_VERIFICATION:-} PT_LLM_MODEL: ${PT_LLM_MODEL:-} PT_STARTUP_LOCK_TIMEOUT_SECONDS: ${PT_STARTUP_LOCK_TIMEOUT_SECONDS:-} diff --git a/docs/spec/promisetracker-v3-system-specification.md b/docs/spec/promisetracker-v3-system-specification.md index 8ce05a47..a599a63a 100644 --- a/docs/spec/promisetracker-v3-system-specification.md +++ b/docs/spec/promisetracker-v3-system-specification.md @@ -812,6 +812,9 @@ so the same file is stored once. default), and each plugin's credentials. Only super-admins can read it, and every credential is encrypted. Environment variables choose which plugins run. The database holds what they need. +The kernel reads the settings once and reuses them, so an adapter call costs no database read. A +change takes effect at once in the process that saved it, and within 30 seconds in the others. + ### 7.11 Indexes and search | Record type | Indexes | diff --git a/docs/spec/promisetracker-v3-system-specification.pdf b/docs/spec/promisetracker-v3-system-specification.pdf index f4ab6ef2..0d4fcbf6 100644 Binary files a/docs/spec/promisetracker-v3-system-specification.pdf and b/docs/spec/promisetracker-v3-system-specification.pdf differ diff --git a/packages/kernel/src/env.ts b/packages/kernel/src/env.ts index 7969308b..50954706 100644 --- a/packages/kernel/src/env.ts +++ b/packages/kernel/src/env.ts @@ -1,7 +1,7 @@ /** Environment variables, as process.env or a plain object in tests. */ export type Env = Readonly> -/** The three variables that select the set-up (section 4.3). */ +/** The four variables that select the set-up (section 4.3). */ export interface KernelEnv { /** Enabled plugin slugs. `core` and `site` when PT_PLUGINS is unset. */ readonly plugins: readonly string[] @@ -9,20 +9,31 @@ export interface KernelEnv { readonly verification: string | undefined /** `provider:model`, for example `google:gemini-2.5-pro`. */ readonly llmModel: string | undefined + /** Locale codes from PT_LOCALES, as written. The kernel checks them (section 7.1). */ + readonly locales: readonly string[] } export const DEFAULT_PLUGINS = ['core', 'site'] as const const clean = (value: string | undefined): string | undefined => value?.trim() || undefined -export function readEnabledPlugins(env: Env = process.env): string[] { - const raw = clean(env.PT_PLUGINS) - if (!raw) return [...DEFAULT_PLUGINS] - const plugins = raw +const readList = (raw: string | undefined): string[] | undefined => { + const value = clean(raw) + if (!value) return undefined + const items = value .split(',') - .map((p) => p.trim()) + .map((item) => item.trim()) .filter(Boolean) - return [...new Set(plugins)] + return [...new Set(items)] +} + +export function readEnabledPlugins(env: Env = process.env): string[] { + return readList(env.PT_PLUGINS) ?? [...DEFAULT_PLUGINS] +} + +/** The locale codes in PT_LOCALES, in lower case. Empty when it is unset. */ +export function readLocales(env: Env = process.env): string[] { + return [...new Set((readList(env.PT_LOCALES) ?? []).map((code) => code.toLowerCase()))] } /** Reads the set-up once. The kernel keeps the result for the life of the process. */ @@ -31,5 +42,6 @@ export function readKernelEnv(env: Env = process.env): KernelEnv { plugins: Object.freeze(readEnabledPlugins(env)), verification: clean(env.PT_VERIFICATION), llmModel: clean(env.PT_LLM_MODEL), + locales: Object.freeze(readLocales(env)), }) } diff --git a/packages/kernel/src/index.ts b/packages/kernel/src/index.ts index b4b16c69..66d9fb43 100644 --- a/packages/kernel/src/index.ts +++ b/packages/kernel/src/index.ts @@ -3,11 +3,12 @@ * the system specification). */ export type { Env, KernelEnv } from './env' -export { DEFAULT_PLUGINS, readEnabledPlugins, readKernelEnv } from './env' +export { DEFAULT_PLUGINS, readEnabledPlugins, readKernelEnv, readLocales } from './env' export { KernelError } from './errors' -export type { Kernel, KernelOptions, ResolvedAdapters } from './kernel' +export type { Kernel, KernelOptions, LocaleSpec, ResolvedAdapters } from './kernel' export { createKernel } from './kernel' export type { HttpMethod, PluginManifest, PluginRoute, PromiseTrackerPlugin } from './plugin' -export { applyManifest, definePromiseTrackerPlugin, isSuperAdmin, SETTINGS_GLOBAL } from './plugin' +export { applyManifest, definePromiseTrackerPlugin, isSuperAdmin } from './plugin' export * from './ports' export { pluginRoute } from './routes' +export { forgetSettings, readSettings, SETTINGS_GLOBAL, SETTINGS_TTL_MS } from './settings' diff --git a/packages/kernel/src/kernel.ts b/packages/kernel/src/kernel.ts index bec9e08b..d52ffec3 100644 --- a/packages/kernel/src/kernel.ts +++ b/packages/kernel/src/kernel.ts @@ -4,7 +4,6 @@ import type { Env, KernelEnv } from './env' import { readKernelEnv } from './env' import { KernelError } from './errors' import type { PluginManifest, PromiseTrackerPlugin } from './plugin' -import { SETTINGS_GLOBAL } from './plugin' import type { AdapterContext, AnalyticsProvider, @@ -20,6 +19,7 @@ import type { VerificationProvider, } from './ports' import { PORTS } from './ports' +import { readSettings } from './settings' /** The adapters that run, one entry per port (section 4.1). */ export interface ResolvedAdapters { @@ -34,8 +34,20 @@ export interface ResolvedAdapters { captcha: CaptchaProvider } +/** A language the Payload config registers (section 7.1). */ +export interface LocaleSpec { + /** Lower case, such as `en` or `fr`. */ + code: string + label: string + /** Written right to left, as Arabic is. */ + rtl?: boolean +} + export interface Kernel { readonly env: KernelEnv + /** The locales in use: the default, then those PT_LOCALES names, in registered order. */ + readonly locales: readonly LocaleSpec[] + readonly defaultLocale: string /** Enabled plugin slugs, in load order. */ readonly enabled: readonly string[] readonly adapters: ResolvedAdapters @@ -60,10 +72,15 @@ export interface KernelOptions { env?: Env /** The plugin whose adapters are defaults. */ defaultsFrom?: string + /** + * Every locale the Payload config registers. The database stores this list, + * so PT_LOCALES can only choose from it (D45). Defaults to English alone. + */ + locales?: readonly LocaleSpec[] + /** Always in use, whatever PT_LOCALES says. Defaults to `en`. */ + defaultLocale?: string } -type GlobalSlugParam = Parameters[0]['slug'] - type Candidate

= { plugin: string; adapter: PortAdapters[P] } const list = (values: readonly string[]): string => values.map((v) => `"${v}"`).join(', ') @@ -115,6 +132,36 @@ function checkDependencies( } } + for (const m of manifests) { + for (const { collection } of m.takesOver ?? []) { + const owner = owners.get(collection) + if (owner && owner !== m.slug && !(m.dependsOn ?? []).includes(owner)) { + throw new KernelError( + `Plugin "${m.slug}" takes over fields on "${collection}", which plugin "${owner}" owns. ` + + `Add "${owner}" to its dependsOn.`, + ) + } + } + } + + // Two enabled plugins cannot both write one field. + const takenBy = new Map() + for (const m of manifests) { + if (!enabled.has(m.slug)) continue + for (const { collection, fields } of m.takesOver ?? []) { + for (const path of fields) { + const key = `${collection}.${path}` + const other = takenBy.get(key) + if (other) { + throw new KernelError( + `Plugins "${other}" and "${m.slug}" both take over "${key}". Remove one from PT_PLUGINS.`, + ) + } + takenBy.set(key, m.slug) + } + } + } + for (const m of manifests) { for (const dep of m.dependsOn ?? []) { const target = bySlug.get(dep) @@ -210,6 +257,34 @@ function pickLlm( return { adapter: match.adapter, model: modelName } } +/** The locales in use: the default and those PT_LOCALES names, in registered order. */ +function pickLocales( + registered: readonly LocaleSpec[], + defaultLocale: string, + chosen: readonly string[], +): { locales: readonly LocaleSpec[]; defaultLocale: string } { + const codes = registered.map((l) => l.code) + if (!codes.includes(defaultLocale)) { + throw new KernelError( + `The default locale "${defaultLocale}" is not registered. Registered locales: ${list(codes)}.`, + ) + } + const unknown = chosen.filter((code) => !codes.includes(code)) + if (unknown.length > 0) { + throw new KernelError( + `PT_LOCALES names locales that are not registered: ${list(unknown)}. ` + + `Registered locales: ${list(codes)}. Registering another needs a code change and a migration.`, + ) + } + const inUse = registered.filter((l) => l.code === defaultLocale || chosen.includes(l.code)) + // The default comes first, as the site and the admin expect. + const ordered = [ + ...inUse.filter((l) => l.code === defaultLocale), + ...inUse.filter((l) => l.code !== defaultLocale), + ] + return { locales: Object.freeze(ordered), defaultLocale } +} + export function createKernel(options: KernelOptions): Kernel { const env = readKernelEnv(options.env) const defaultsFrom = options.defaultsFrom ?? 'core' @@ -230,6 +305,11 @@ export function createKernel(options: KernelOptions): Kernel { } const enabledSet = new Set(env.plugins) checkDependencies(manifests, enabledSet) + const { locales, defaultLocale } = pickLocales( + options.locales ?? [{ code: 'en', label: 'English' }], + options.defaultLocale ?? 'en', + env.locales, + ) const adapters: ResolvedAdapters = Object.freeze({ verification: pickOne( @@ -301,23 +381,16 @@ export function createKernel(options: KernelOptions): Kernel { const portList = Object.entries(ports()) .map(([port, ids]) => `${port}=${ids.length ? ids.join('+') : 'none'}`) .join(' ') - return `PromiseTracker plugins=${enabled.join(',')} ${portList}` + const localeList = locales.map((l) => l.code).join(',') + return `PromiseTracker plugins=${enabled.join(',')} locales=${localeList} ${portList}` } const settingsFor = async ( payload: Payload, slug: string, ): Promise>> => { - // The settings global exists only once a plugin declares settings. - const globalSlugs: readonly string[] = payload.config.globals.map((g) => g.slug) - if (!globalSlugs.includes(SETTINGS_GLOBAL)) return {} - // The kernel cannot see the app's generated global names, so it names this - // one through Payload's own parameter type. Settings hold credentials, and - // the kernel reads them as the system. - const settings = await payload.findGlobal({ - slug: SETTINGS_GLOBAL as GlobalSlugParam, - overrideAccess: true, - }) + // Read once and reused, so an adapter call does not cost a database read. + const settings = await readSettings(payload) const own: unknown = Reflect.get(settings, slug) return own && typeof own === 'object' ? (own as Record) : {} } @@ -357,6 +430,8 @@ export function createKernel(options: KernelOptions): Kernel { return Object.freeze({ env, + locales, + defaultLocale, enabled: Object.freeze(enabled), adapters, payloadPlugins: Object.freeze(payloadPlugins), diff --git a/packages/kernel/src/plugin.ts b/packages/kernel/src/plugin.ts index a650874b..f1603523 100644 --- a/packages/kernel/src/plugin.ts +++ b/packages/kernel/src/plugin.ts @@ -14,6 +14,7 @@ import { definePlugin } from 'payload' import { KernelError } from './errors' import type { AdapterContext, PortAdapters, PortName } from './ports' +import { forgetSettings, SETTINGS_GLOBAL } from './settings' type JobsConfig = NonNullable type TaskConfig = NonNullable[number] @@ -45,6 +46,13 @@ export interface PluginManifest { * after this plugin, must be optional, and are read-only in the admin. */ extend?: readonly { collection: string; fields: readonly Field[] }[] + /** + * Fields on another plugin's record type that this plugin writes while it is + * enabled, as dotted paths such as `verification.verdict` (section 4.5). + * They become read-only in the admin to everyone but a super-admin. The + * fields stay where they are, so disabling the plugin hands them back. + */ + takesOver?: readonly { collection: string; fields: readonly string[] }[] /** Adapters, by port. */ provides?: { readonly [P in PortName]?: readonly PortAdapters[P][] } tasks?: readonly TaskConfig[] @@ -65,8 +73,6 @@ export interface PromiseTrackerPlugin { toPayloadPlugin(enabled: boolean): Plugin } -export const SETTINGS_GLOBAL = 'settings' - /** Super-admins arrive with roles in V1-05. Until then nobody passes. */ export const isSuperAdmin = (user: PayloadRequest['user']): boolean => { const roles: unknown = user ? Reflect.get(user, 'roles') : undefined @@ -253,6 +259,129 @@ function extendCollections( return result } +/** Logs a super-admin's change to one field a plugin has taken over. */ +const logTakenOverChange = + (slug: string, path: string): FieldHook => + ({ req, value, previousValue, collection, originalDoc }) => { + if (!isSuperAdmin(req.user)) return value + if (JSON.stringify(value) !== JSON.stringify(previousValue)) { + req.payload.logger.warn({ + msg: `Super-admin override of a field taken over by plugin "${slug}"`, + collection: collection?.slug, + id: originalDoc?.id, + user: req.user?.id, + fields: [path], + }) + } + return value + } + +type Guarded = { + access?: Record + admin?: Record + hooks?: { beforeChange?: FieldHook[] } +} + +/** A core field as it is while a plugin has taken it over. Its other rules stay. */ +function takenOver(slug: string, path: string, field: Field): Field { + const current = field as Field & Guarded + return { + ...current, + access: { ...current.access, update: superAdminOnly }, + admin: { ...current.admin, readOnly: true }, + hooks: { + ...current.hooks, + beforeChange: [...(current.hooks?.beforeChange ?? []), logTakenOverChange(slug, path)], + }, + } as Field +} + +/** + * Replaces the field at a dotted path, or returns `undefined` when there is + * none. A group or a named tab is one step of the path. Rows, collapsibles and + * unnamed tabs hold fields without adding a step. Fields inside arrays and + * blocks cannot be named this way. + */ +function replaceAtPath( + fields: readonly Field[], + path: readonly string[], + replace: (field: Field) => Field, +): Field[] | undefined { + const [head, ...rest] = path + if (head === undefined) return undefined + let found = false + const next = fields.map((field): Field => { + if (found) return field + if (field.type === 'tabs') { + const tabs = field.tabs.map((tab) => { + if (found) return tab + const named = 'name' in tab && typeof tab.name === 'string' + if (named && tab.name !== head) return tab + const replaced = replaceAtPath(tab.fields, named ? rest : path, replace) + if (!replaced) return tab + found = true + return { ...tab, fields: replaced } + }) + return found ? ({ ...field, tabs } as Field) : field + } + if ('name' in field && typeof field.name === 'string') { + if (field.name !== head) return field + if (rest.length === 0) { + found = true + return replace(field) + } + if (field.type !== 'group') return field + const replaced = replaceAtPath(field.fields, rest, replace) + if (!replaced) return field + found = true + return { ...field, fields: replaced } as Field + } + if ('fields' in field && Array.isArray(field.fields)) { + const replaced = replaceAtPath(field.fields, path, replace) + if (!replaced) return field + found = true + return { ...field, fields: replaced } as Field + } + return field + }) + return found ? next : undefined +} + +/** + * Applies `takesOver`. A path that names nothing stops start-up whether or not + * the plugin is enabled, so a typo shows up before the plugin is switched on. + */ +function takeOverFields( + collections: readonly CollectionConfig[], + manifest: PluginManifest, + enabled: boolean, +): CollectionConfig[] { + const result = [...collections] + for (const { collection, fields } of manifest.takesOver ?? []) { + const index = result.findIndex((c) => c.slug === collection) + let target = result[index] + if (!target) { + throw new KernelError( + `Plugin "${manifest.slug}" takes over fields on "${collection}", which no earlier plugin defines. ` + + 'Load the plugin that owns it first.', + ) + } + for (const path of fields) { + const replaced = replaceAtPath(target.fields, path.split('.'), (field) => + takenOver(manifest.slug, path, field), + ) + if (!replaced) { + throw new KernelError( + `Plugin "${manifest.slug}" takes over "${collection}.${path}", which does not exist.`, + ) + } + target = { ...target, fields: replaced } + } + if (enabled) result[index] = target + } + return result +} + /** * A disabled plugin's tasks stay registered, because Payload builds the jobs * table from task names, but they refuse to run, have no schedule, and fail @@ -301,6 +430,15 @@ function addSettings(globals: readonly GlobalConfig[], manifest: PluginManifest, read: ({ req }) => isSuperAdmin(req.user), update: ({ req }) => isSuperAdmin(req.user), }, + // The kernel reuses the settings it read. A save here makes it read again. + hooks: { + afterChange: [ + ({ doc, req }) => { + forgetSettings(req.payload) + return doc + }, + ], + }, fields: [group], } satisfies GlobalConfig, ] @@ -322,8 +460,8 @@ export function applyManifest(config: Config, manifest: PluginManifest, enabled: const next: Config = { ...config, - collections: extendCollections( - [...(config.collections ?? []), ...ownCollections], + collections: takeOverFields( + extendCollections([...(config.collections ?? []), ...ownCollections], manifest, enabled), manifest, enabled, ), diff --git a/packages/kernel/src/ports.ts b/packages/kernel/src/ports.ts index f830e201..33c5ad4c 100644 --- a/packages/kernel/src/ports.ts +++ b/packages/kernel/src/ports.ts @@ -41,7 +41,8 @@ export interface SourceContent { // Verification -export type Verdict = 'verified' | 'false' | 'unverified' +/** The fact-checker's finding (section 7.5). Only `verified` claims are published. */ +export type Verdict = 'verified' | 'false' | 'misleading' | 'unverified' export interface ClaimForVerification { id: string @@ -57,6 +58,14 @@ export interface VerdictUpdate { verdict: Verdict rationale?: string verifiedAt?: string + /** The published report, when the outside system has one. */ + reportUrl?: string + /** + * The outside system took back a verdict it had published, as when a Check + * report is paused. The core then holds the claim and tells an editor, which + * a claim that is simply not verified yet does not need. + */ + withdrawn?: boolean } export interface VerificationProvider extends Adapter { @@ -100,19 +109,27 @@ export interface ContentExtractor extends Adapter { ): Promise<{ text: string; pageCount: number | null; metadata: Record }> } +/** One row of a spreadsheet, with its number in the file for error messages. */ +export interface ImportRow { + /** Counted from 1, where 1 is the first row after the headings. */ + row: number + /** One value per heading, in the headings' order. */ + values: readonly string[] +} + +/** + * Reads a spreadsheet. It never saves anything: the core maps rows to records, + * previews and commits, once for every file format (section 9.6, D49). + */ export interface StructuredImporter extends Adapter { readonly accepts: readonly string[] - /** Reads headings and sample rows without saving anything. */ + /** Reads headings and sample rows. */ inspect( content: SourceContent, ctx: AdapterContext, ): Promise<{ headings: string[]; sampleRows: string[][] }> - /** Creates or updates records, mapping each record field to a column heading. */ - import( - content: SourceContent, - mapping: Readonly>, - ctx: AdapterContext, - ): Promise<{ created: number; updated: number; errors: { row: number; message: string }[] }> + /** Every row after the headings, in order. */ + rows(content: SourceContent, ctx: AdapterContext): AsyncIterable } // AI @@ -125,9 +142,16 @@ export interface LLMRequest { outputSchema: Readonly> } +export interface LLMResult { + /** The structured output. The caller validates it against the schema it sent. */ + output: unknown + /** Tokens used, which extractions record (section 7.8). */ + usage: { inputTokens: number; outputTokens: number } +} + export interface LLMProvider extends Adapter { readonly models: readonly string[] - generate(request: LLMRequest, ctx: AdapterContext): Promise + generate(request: LLMRequest, ctx: AdapterContext): Promise } // Public site diff --git a/packages/kernel/src/settings.ts b/packages/kernel/src/settings.ts new file mode 100644 index 00000000..6a5c8679 --- /dev/null +++ b/packages/kernel/src/settings.ts @@ -0,0 +1,46 @@ +import type { Payload } from 'payload' + +export const SETTINGS_GLOBAL = 'settings' + +/** + * How long a process reuses the settings it read. A save in this process + * clears them at once (see `forgetSettings`). Another process, such as the + * worker, cannot see that save, so it reads again after this long. + */ +export const SETTINGS_TTL_MS = 30_000 + +type Settings = Readonly> +type GlobalSlugParam = Parameters[0]['slug'] + +const cache = new WeakMap }>() + +/** + * Every plugin's settings, read once and reused, so an adapter call does not + * cost a database read (section 7.10). + */ +export function readSettings(payload: Payload, now: number = Date.now()): Promise { + // The settings global exists only once a plugin declares settings. + const globalSlugs: readonly string[] = payload.config.globals.map((g) => g.slug) + if (!globalSlugs.includes(SETTINGS_GLOBAL)) return Promise.resolve({}) + + const cached = cache.get(payload) + if (cached && now - cached.at < SETTINGS_TTL_MS) return cached.value + + // The kernel cannot see the app's generated global names, so it names this + // one through Payload's own parameter type. Settings hold credentials, and + // the kernel reads them as the system. + const value = payload + .findGlobal({ slug: SETTINGS_GLOBAL as GlobalSlugParam, overrideAccess: true }) + .then((doc: unknown): Settings => (doc && typeof doc === 'object' ? (doc as Settings) : {})) + cache.set(payload, { at: now, value }) + // A failed read is not kept, so the next call tries again. + value.catch(() => { + if (cache.get(payload)?.value === value) cache.delete(payload) + }) + return value +} + +/** Drops the settings this process holds, so the next read goes to the database. */ +export function forgetSettings(payload: Payload): void { + cache.delete(payload) +} diff --git a/tests/unit/kernel.unit.spec.ts b/tests/unit/kernel.unit.spec.ts index fac1a58c..ec9bee0b 100644 --- a/tests/unit/kernel.unit.spec.ts +++ b/tests/unit/kernel.unit.spec.ts @@ -1,17 +1,25 @@ import type { + ImportRow, LLMProvider, + LLMResult, PluginManifest, SearchProvider, + StructuredImporter, + VerdictUpdate, VerificationProvider, } from '@promisetracker/kernel' import { applyManifest, createKernel, definePromiseTrackerPlugin, + forgetSettings, KernelError, pluginRoute, readEnabledPlugins, readKernelEnv, + readLocales, + readSettings, + SETTINGS_TTL_MS, } from '@promisetracker/kernel' import { corePlugin } from '@promisetracker/plugin-core' import { sitePlugin } from '@promisetracker/plugin-site' @@ -64,12 +72,17 @@ describe('reading the environment', () => { ]) }) - it('reads the three selecting variables once, into a frozen object', () => { - const env = readKernelEnv({ PT_VERIFICATION: ' check ', PT_LLM_MODEL: 'google:gemini' }) + it('reads the four selecting variables once, into a frozen object', () => { + const env = readKernelEnv({ + PT_VERIFICATION: ' check ', + PT_LLM_MODEL: 'google:gemini', + PT_LOCALES: 'en, FR,fr', + }) expect(env).toEqual({ plugins: ['core', 'site'], verification: 'check', llmModel: 'google:gemini', + locales: ['en', 'fr'], }) expect(Object.isFrozen(env)).toBe(true) }) @@ -591,3 +604,285 @@ describe('second review follow-ups', () => { expect(Object.isFrozen(kernelWith([]).adapters.contentExtractor)).toBe(true) }) }) + +describe('November review changes (#665)', () => { + describe('locales', () => { + const registered = [ + { code: 'en', label: 'English' }, + { code: 'fr', label: 'Français' }, + { code: 'ar', label: 'العربية', rtl: true }, + ] + const withLocales = (env: Record = {}) => + createKernel({ plugins: [corePlugin, sitePlugin], locales: registered, env }) + + it('reads PT_LOCALES in lower case, without repeats', () => { + expect(readLocales({ PT_LOCALES: ' FR , ar,fr ' })).toEqual(['fr', 'ar']) + expect(readLocales({})).toEqual([]) + }) + + it('uses only English when PT_LOCALES is unset', () => { + const kernel = withLocales() + expect(kernel.locales.map((l) => l.code)).toEqual(['en']) + expect(kernel.defaultLocale).toBe('en') + }) + + it('keeps the default locale on and first, whatever PT_LOCALES says', () => { + expect(withLocales({ PT_LOCALES: 'ar,fr' }).locales.map((l) => l.code)).toEqual([ + 'en', + 'fr', + 'ar', + ]) + }) + + it('stops on a locale that is not registered and lists the registered ones', () => { + expect(() => withLocales({ PT_LOCALES: 'en,sw' })).toThrow( + /PT_LOCALES names locales that are not registered: "sw"\. Registered locales: "en", "fr", "ar"/, + ) + }) + + it('stops when the default locale is not registered', () => { + expect(() => + createKernel({ + plugins: [corePlugin, sitePlugin], + locales: registered, + defaultLocale: 'pt', + }), + ).toThrow('The default locale "pt" is not registered') + }) + + it('names the locales in the summary line', () => { + expect(withLocales({ PT_LOCALES: 'fr' }).summary()).toContain('locales=en,fr') + }) + }) + + describe('taking over core fields', () => { + const claims: CollectionConfig = { + slug: 'claims', + fields: [ + { name: 'title', type: 'text' }, + { + type: 'row', + fields: [ + { + name: 'verification', + type: 'group', + fields: [ + { name: 'verdict', type: 'text', access: { read: () => true } }, + { name: 'rationale', type: 'textarea' }, + ], + }, + ], + }, + ], + } + const config = (): Config => ({ ...baseConfig(), collections: [claims] }) + const checker: PluginManifest = { + slug: 'checker', + version: '0.0.0', + order: 50, + takesOver: [{ collection: 'claims', fields: ['verification.verdict'] }], + } + const verdictIn = (c: Config): Field => { + const row = collection(c, 'claims').fields.find((f) => f.type === 'row') + const group = fieldNamed(row && 'fields' in row ? row.fields : [], 'verification') + return fieldNamed('fields' in group ? group.fields : [], 'verdict') + } + const superAdmin = { id: 'u1', roles: ['super-admin'] } + + it('makes the field read-only to everyone but a super-admin while enabled', () => { + const verdict = verdictIn(applyManifest(config(), checker, true)) + expect(verdict).toMatchObject({ admin: { readOnly: true } }) + const access = 'access' in verdict ? verdict.access : undefined + expect(access?.update?.({ req: { user: { roles: ['reviewer'] } } } as never)).toBe(false) + expect(access?.update?.({ req: { user: superAdmin } } as never)).toBe(true) + // The field's other rules stay. + expect(access?.read).toBeTypeOf('function') + }) + + it('leaves other fields and a disabled plugin alone', () => { + const on = applyManifest(config(), checker, true) + const row = collection(on, 'claims').fields.find((f) => f.type === 'row') + const group = fieldNamed(row && 'fields' in row ? row.fields : [], 'verification') + expect(fieldNamed('fields' in group ? group.fields : [], 'rationale')).not.toHaveProperty( + 'admin', + ) + expect(verdictIn(applyManifest(config(), checker, false))).not.toHaveProperty('admin') + }) + + it("logs a super-admin's change by field name, without the values", () => { + const verdict = verdictIn(applyManifest(config(), checker, true)) + const hook = 'hooks' in verdict ? verdict.hooks?.beforeChange?.[0] : undefined + const run = (user: unknown) => { + const warn = vi.fn() + hook?.({ + req: { user, payload: { logger: { warn } } }, + value: 'new-value', + previousValue: 'old-value', + collection: { slug: 'claims' }, + originalDoc: { id: 'c1' }, + } as never) + return warn + } + const warn = run(superAdmin) + expect(warn).toHaveBeenCalledOnce() + const entry = JSON.stringify(warn.mock.calls[0]) + expect(entry).toContain('verification.verdict') + expect(entry).not.toContain('value') + expect(run({ roles: ['editor'] })).not.toHaveBeenCalled() + }) + + it('stops on a path that does not exist, even while the plugin is disabled', () => { + const typo = { ...checker, takesOver: [{ collection: 'claims', fields: ['verification.x'] }] } + for (const enabled of [true, false]) { + expect(() => applyManifest(config(), typo, enabled)).toThrow( + 'takes over "claims.verification.x", which does not exist', + ) + } + }) + + it('stops when the record type is not defined yet', () => { + expect(() => applyManifest(baseConfig(), checker, true)).toThrow( + 'takes over fields on "claims", which no earlier plugin defines', + ) + }) + + const owner = definePromiseTrackerPlugin({ + slug: 'owner', + version: '0.0.0', + order: 20, + dependsOn: ['core'], + collections: [claims], + }) + const taker = (slug: string, dependsOn: string[] = ['core', 'owner']) => + plugin({ + slug, + dependsOn, + takesOver: [{ collection: 'claims', fields: ['verification.verdict'] }], + }) + + it("stops when the record type's owner is not in dependsOn", () => { + expect(() => kernelWith([owner, taker('check', ['core'])])).toThrow( + 'Plugin "check" takes over fields on "claims", which plugin "owner" owns. Add "owner" to its dependsOn.', + ) + }) + + it('stops when two enabled plugins take over one field', () => { + const both = [owner, taker('check'), taker('superdesk')] + expect(() => kernelWith(both, enabling('owner', 'check', 'superdesk'))).toThrow( + 'Plugins "check" and "superdesk" both take over "claims.verification.verdict"', + ) + expect(() => kernelWith(both, enabling('owner', 'check'))).not.toThrow() + }) + }) + + describe('plugin settings', () => { + const fake = (doc: unknown = { check: { apiKey: 'k' } }) => { + const findGlobal = vi.fn(async () => doc) + const payload = { config: { globals: [{ slug: 'settings' }] }, findGlobal } + return { payload: payload as unknown as Payload, findGlobal } + } + + it('are read once and reused', async () => { + const { payload, findGlobal } = fake() + await readSettings(payload, 1_000) + await expect(readSettings(payload, 2_000)).resolves.toEqual({ check: { apiKey: 'k' } }) + expect(findGlobal).toHaveBeenCalledOnce() + }) + + it('are read again after a save in this process', async () => { + const { payload, findGlobal } = fake() + await readSettings(payload, 1_000) + forgetSettings(payload) + await readSettings(payload, 1_001) + expect(findGlobal).toHaveBeenCalledTimes(2) + }) + + it('are read again once they are old, for a save made in another process', async () => { + const { payload, findGlobal } = fake() + await readSettings(payload, 1_000) + await readSettings(payload, 1_000 + SETTINGS_TTL_MS) + expect(findGlobal).toHaveBeenCalledTimes(2) + }) + + it('do not keep a failed read', async () => { + const findGlobal = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error('database down')) + .mockResolvedValueOnce({ check: {} }) + const payload = { config: { globals: [{ slug: 'settings' }] }, findGlobal } + await expect(readSettings(payload as unknown as Payload, 1_000)).rejects.toThrow('down') + await expect(readSettings(payload as unknown as Payload, 1_001)).resolves.toEqual({ + check: {}, + }) + }) + + it('are empty, with no read, when no plugin declares settings', async () => { + const findGlobal = vi.fn() + const payload = { config: { globals: [] }, findGlobal } + await expect(readSettings(payload as unknown as Payload)).resolves.toEqual({}) + expect(findGlobal).not.toHaveBeenCalled() + }) + + it('are forgotten when the settings global is saved', () => { + const on = applyManifest(baseConfig(), fullManifest, true) + const settings = on.globals?.find((g) => g.slug === 'settings') + expect(settings?.hooks?.afterChange).toHaveLength(1) + }) + + it("reach an adapter's context as that plugin's own group", async () => { + const { payload, findGlobal } = fake({ check: { apiKey: 'k' }, ai: { apiKey: 'other' } }) + Object.assign(payload, { logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } }) + const kernel = kernelWith([]) + await expect(kernel.context(payload, 'check')).resolves.toMatchObject({ + settings: { apiKey: 'k' }, + }) + await kernel.context(payload, 'check') + expect(findGlobal).toHaveBeenCalledOnce() + }) + }) + + describe('port shapes', () => { + it('carries a misleading verdict, a report link and a withdrawn report', () => { + const update: VerdictUpdate = { + externalId: 'item-1', + verdict: 'misleading', + reportUrl: 'https://example.org/report', + withdrawn: true, + } + expect(update.verdict).toBe('misleading') + }) + + it('returns the tokens an AI call used', async () => { + const result: LLMResult = { output: {}, usage: { inputTokens: 10, outputTokens: 4 } } + const provider: LLMProvider = { ...llm('stub'), generate: async () => result } + const ctx = await kernelWith([]).context( + { config: { globals: [] }, logger: {} } as unknown as Payload, + 'ai', + ) + const request = { model: 'm', system: '', prompt: '', outputSchema: {} } + await expect(provider.generate(request, ctx)).resolves.toEqual(result) + }) + + it('lets an importer read rows, and nothing else', async () => { + const importer: StructuredImporter = { + id: 'csv', + accepts: ['text/csv'], + health: healthy, + inspect: async () => ({ headings: ['name'], sampleRows: [['Nairobi']] }), + rows: async function* () { + yield { row: 1, values: ['Nairobi'] } + yield { row: 2, values: ['Mombasa'] } + }, + } + const content = { filename: 'a.csv', mimeType: 'text/csv', data: new Uint8Array() } + const ctx = await kernelWith([]).context( + { config: { globals: [] }, logger: {} } as unknown as Payload, + 'core', + ) + const read: ImportRow[] = [] + for await (const row of importer.rows(content, ctx)) read.push(row) + expect(read.map((r) => r.row)).toEqual([1, 2]) + expect(importer).not.toHaveProperty('import') + }) + }) +})