diff --git a/README.md b/README.md index afa8230..2751e26 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,9 @@ epg try # one channel-day: the request, the payload, the programmes epg validate # read the guide and report what is wrong with it epg filter guide.xml --channels my-60.m3u -o small.xml # keep 60 of 900 epg channels --against playlist.m3u # which wanted channels will get no guide -epg prune # drop cached days older than today +epg cache stats # how much is cached, per site +epg cache show # what is cached for a site, as XMLTV +epg cache prune # drop cached days older than today epg init-grabber tv_grab_sk_example # write an XMLTV grabber for this config epg build -d 14 -o public/epg.xml epg build --offset 1 # start the window tomorrow instead of today @@ -186,7 +188,8 @@ into place, so a reader never sees half a guide. | `--allow-missing ` | exit 0 with up to this much of the guide missing — a number of channel-days, or a share like `5%`; [a site that answered nothing is never forgiven](./docs/configuration.md#allowing-some-of-the-guide-to-be-missing) | | `--dry-run` | `grab`/`build` only: [say what a run would fetch](./docs/configuration.md#what-a-run-would-do), and stop — `--format json` for a machine-readable one | | `--threads[=n]` | `build`/`grab`/`serve`: run the sites in [worker threads](./docs/configuration.md#sites-in-threads-of-their-own), one per core or at most `n`; `--no-threads` overrides a config that asks for them | -| `--before ` | `prune` only: remove days before `YYYY-MM-DD`, default today | +| `--before ` | `cache prune` only: remove days before `YYYY-MM-DD`, default today | +| `--keys` | `cache state` with a group only: the keys it holds, not what it holds | | `--log-level ` | how much to report: `error`, `warn`, `info` (default) or `debug` | | `-v, --verbose` | same as `--log-level debug`; `-q, --quiet` is `--log-level error` | | `--reporter ` | `progress` (default — a live line on a terminal, `text` anywhere else), `text`, or `json` for one object per line | diff --git a/docs/api.md b/docs/api.md index 5dd45ff..f04548a 100644 --- a/docs/api.md +++ b/docs/api.md @@ -614,10 +614,10 @@ M3U one above. `CacheManager`, the drivers `FsNdjsonCacheDriver`, `FsXmltvCacheDriver`, `MemoryCacheDriver`* and `NoCacheDriver`* with the abstract `FsCacheDriver` the file ones build on, the codecs a driver names in `CODECS` and the `toRecord` / -`fromRecord` pair they build on, `CACHE_SCHEMA`*, `STATE_SCHEMA`*, `isStale`, +`fromRecord` pair they build on, `CorruptEntryError`, `CACHE_SCHEMA`*, `STATE_SCHEMA`*, `isStale`, `DEFAULT_STALENESS`, `CACHE_DRIVER_NAMES`, and the `CacheStore` / `CacheDriver` / `CacheManagerOptions` / `FsCacheDriverOptions` / `CacheCodec` / `CodecName` / -`ChannelDayKey` / `CacheEntryMeta` / `StoredEntryMeta` / `StoredProgramme` / +`ChannelDayKey` / `CachedKey` / `CacheListOptions` / `Inspected` / `Unusable` / `CacheEntryMeta` / `StoredEntryMeta` / `StoredProgramme` / `FoundMeta` / `FoundEntry` / `StateEntry` / `StoredStateMeta` / `FoundState` / `StalenessPolicy` / `CacheDriverName` types. @@ -629,16 +629,44 @@ files, a database, a bucket — and holds no policy of its own. The which is what `grab` and `generateGuide` take), and it owns everything that must be true of every driver alike: the meta an entry carries, stamped on the way in and judged on the way out, and what to do with an entry that cannot answer for -itself — it goes, so the day reads as never grabbed and the next run fetches it. +itself, or whose programmes do not read back as what it says it holds — it +goes, so the day reads as never grabbed and the next run fetches it. ```ts const cache = new CacheManager({ driver: new FsNdjsonCacheDriver({ dir }) }); ``` +It also answers what [`epg cache`](./configuration.md#looking-inside-the-cache) +asks, and changes nothing doing so. `list(options?)` is the driver's own. +`inspectMetas(keys)`, `inspect(key)` and `inspectState(site, key)` judge an +entry or a group exactly as `getMetas`, `read` and `getState` do — the config's +`invalidate` included — and one a run would remove stays where it is. What +comes back says which by its `status`, and a key with nothing under it is +`undefined`: + +| `status` | is | beside it | +|---|---|---| +| `'readable'` | what a run would use | what was read: `meta`, and `programmes` or `data` | +| `'unreadable'` | something that says nothing about itself this version can read | `meta`, whatever was there, or `undefined` | +| `'incompatible'` | written in a shape this version does not read, older or newer | `meta`, what it says about itself — its `schema` the shape it was written in | +| `'invalidated'` | an entry `invalidate` voids | `meta`, what it says about itself | +| `'corrupt'` | an entry whose meta reads and whose programmes do not: its codec throws `CorruptEntryError` on them — as those of `CODECS` do for more or fewer than the meta says it holds | `meta`, and `error`, what the codec threw | + +```ts +const looked = await cache.inspect({ site: 'example.tv', channelId: 'one', day: '2026-07-17' }); +// { status: 'readable', meta, programmes }, or { status: 'incompatible', meta: { schema: 0, … } }, … +``` + +What the driver throws — a file this user may not read — is thrown, as it is to +a run: there is no entry to judge. `epg cache` says so of that one and goes on +to the rest. + +#### A driver of your own + A driver is a small thing to write: `codec`, `readMeta`, `read`, `write`, `delete`, `prune`, the three that keep [a site's state](#what-a-site-remembers-between-runs) — `readState`, `writeState`, -`deleteState` — and, if it has them to offer, `readMetas` and `close`. It keeps +`deleteState` — `list`, and, if it has them to offer, `readMetas` and `close`. It keeps an entry as **bytes** beside its meta and never sees a programme: `codec` says what the bytes are, and the manager encodes on the way in and decodes on the way out, so no driver has to remember to. `read` and `readMeta` return @@ -646,6 +674,19 @@ way out, so no driver has to remember to. `read` and `readMeta` return when there is one that says nothing readable; the manager tells those apart and only removes the second. +`list(options?)` names everything the store holds, as an async iterable: +`{ kind: 'entry', key }` for a channel-day and `{ kind: 'state', site, key }` +for a group of a site's state. `{ site, channelId, kind }` narrow it — to one +site, to one channel's entries, to entries or to groups — so a store kept by +site and channel need not look at the rest; one that cannot narrow by something +may answer with more, and `CacheManager.list` leaves out what was not asked +for. Keys as they were written, not as the store had to escape them, and only keys: +what is in them is what `read` and `readState` are for, so a store that keeps an +entry's meta inside the entry need not open every one to answer. In whatever +order suits the store. It is what [`epg cache`](./configuration.md#looking-inside-the-cache) +looks inside a store with, and it is required — a store that holds nothing +yields nothing, as `NoCacheDriver` does. + `readMetas(keys)` is the optional one worth knowing about. A grab asks about every channel-day of its window before fetching anything, a channel's window at a time, so a store that can settle fourteen days in one question says so here — @@ -719,10 +760,59 @@ and the main thread only stamps and saves them. A driver that keeps something none of these make — fewer fields, a format of its own — brings a **`CacheCodec`** instead, an object with `encode(programmes)` -and `decode(body)`; `toRecord` and `fromRecord` are there to build one on. It +and `decode(body, meta)`; `toRecord` and `fromRecord` are there to build one on. It runs on the main thread only, since a thread cannot be handed code: a site in a thread sends its programmes, and the manager encodes them there. +A codec says an entry is damaged — bytes that are not what it wrote: a line +cut short, an edit, a disk that filled — by throwing **`CorruptEntryError`** +from `decode`. A run then removes the entry, says so (see `warn` below), and +the next run grabs the day again. Anything else `decode` throws is taken for +the codec's own failing — a bug, a format changed without a schema to say so — +and thrown out of `read` as it is, the entry kept: a removal, for every entry at +once, would empty the cache and write an empty guide over the last good one. + +`decode(body, meta)` is handed what the entry says about itself beside its +bytes, which is how a codec tells a cut that still reads — an ndjson entry cut +between lines, an xmltv document that ends early — from a whole one: by +`meta.programmeCount`. + +```ts +import { gunzipSync, gzipSync } from 'node:zlib'; +import { CODECS, CorruptEntryError, type CacheCodec } from 'epg-tools/cache'; + +const gzipped: CacheCodec = { + encode: (programmes) => gzipSync(CODECS.json.encode(programmes)), + decode(body, meta) { + let json: Uint8Array; + + try { + json = gunzipSync(body); + } catch (error) { + // Bytes that are not gzip are an entry damaged, not a codec. + throw new CorruptEntryError('An entry of the gzipped codec is not gzip', { cause: error }); + } + + // Which says so itself of records it cannot read, or fewer than `meta` says. + return CODECS.json.decode(json, meta); + }, +}; +``` + +The codecs of `CODECS` throw it for what their parser cannot read — JSON that +is not JSON, a date that is no date, a document that is not XMLTV — with the +parser's error as its `cause`, and for a programme count other than the meta's: +they give back exactly what they were given. Anything else they throw is a +fault of this package's, and thrown as it is. Each takes the meta now, and +throws where before the `json` one answered `[]` and the other two threw the +parser's error itself. A codec of a driver's own may give back other than it +was given — one that leaves out, or folds together — and holds an entry to its +count or not, as it knows it should. + +`instanceof CorruptEntryError` knows one by its name as well, so a driver +whose package brings a copy of `epg-tools` of its own throws one the manager +knows. + What `write` is handed may be a view into a larger buffer — a codec allocates for the worst case rather than measuring first — so a driver that holds on to the bytes, rather than writing them out, keeps a copy of the view: @@ -764,6 +854,14 @@ It is asked about every entry a run looks at — thousands of times — so it sh decide from the meta it is given and nothing further. An entry the shape check already refused never reaches it. +`warn(message, key)` on `CacheManager` is where it says it removed an entry +whose codec said it was damaged — a line cut short, an edit — so the day is +missing from anything built from the cache until the next run grabs it again. +Only that: an entry removed for a schema this version does not read, or by +`invalidate`, is what an upgrade or a config asked for. A run passes it on as a +`site:warning` of the site whose day it was, whether a grab came across it or a +merge. + ### Drivers that need no store Two drivers need no store at all. **`MemoryCacheDriver`** keeps entries for the @@ -864,6 +962,16 @@ row can be read past, so the first run of 0.7.0 empties a database an older version made — its listings and what its sites remember — and the run fills it again. +Nothing is made or replaced until something is written. A database another +version made reads as empty — as does one not there yet — and the first write +replaces its tables, or makes the file and its directory. So a look — `epg +cache show`, `stats`, `state` — changes nothing, pointed at a directory with no +database in it or at a cache another version wrote. A place the file could +never be made — a read-only mount, a directory without permission, a path +through a file — is refused when the driver is made, though, for a look as for +a run: a run would otherwise fetch everything before finding out, and the driver +cannot tell which it is serving. + ### `epg-tools/grabber` `grab`, `defineSiteConfig`, `defineStreamSiteConfig`, `defineXmltvSite`, diff --git a/docs/configuration.md b/docs/configuration.md index 2d005e8..6471b19 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -190,7 +190,7 @@ that follows the lineup still moves the tag, because the lineup does. | field | type | default | what it is | |---|---|---|---| | `dir` | `string` | `.epg-cache` in the working directory | Cache root. **Make it absolute** if the config is also used by a [grabber](./tv-grab.md), which runs from wherever it is called. | -| `driver` | `'ndjson' \| 'xmltv' \| 'sqlite' \| 'memory' \| CacheDriverFactory` | `'ndjson'` | Where and how cached days are kept. Both names are **one file** per channel-day under `dir`, its own meta included. `ndjson` writes `.ndjson`: a meta line (`grabbedAt`, `programmeCount`, and the two versions [an entry records about itself](./api.md#what-an-entry-says-about-itself)), then one JSON programme per line, with dates in XMLTV form (`20260807203000 +0200`) so the offset the source wrote them in and how precise it was both survive. `xmltv` writes `.xml`, one small indented document — its root element carrying `date` as XMLTV means it, and its meta in a [processing instruction](./xmltv.md#processing-instructions) rather than in an attribute no DTD defines. Whether an entry then validates is down to the listing in it: a cache keeps whatever the site produced, [provider extensions](#provider-extensions) included, and the merge reads them back — so leaving them out is a decision for the guide, not for the cache. Pick it when something other than this package reads the cache. A driver reads **its own** entries only, so switching means starting the cache again. `sqlite` keeps the whole cache in one `cache.sqlite` under `dir` instead of a file per channel-day — worth it at thousands of channels, where a directory tree is tens of thousands of inodes to walk and back up; it needs Node 24 (or 22.5 with `--experimental-sqlite`), and is [loaded only when named](./api.md#epg-toolscachesqlite). `memory` keeps entries only as long as the process, which is enough for a `build` — its grab and its merge share one cache — and nothing for two commands, since `epg grab` and then `epg merge` would find the second one empty. Or pass a function returning a [driver of your own](./api.md#epg-toolscache) — `({ dir, signal }) => driver`, or a [builder](./api.md#epg-toolscache) that takes your options and returns one — which is how a cache ends up in a database, a bucket, or nowhere at all. It may await, so opening a connection belongs there. | +| `driver` | `'ndjson' \| 'xmltv' \| 'sqlite' \| 'memory' \| CacheDriverFactory` | `'ndjson'` | Where and how cached days are kept. Both names are **one file** per channel-day under `dir`, its own meta included. `ndjson` writes `.ndjson`: a meta line (`grabbedAt`, `programmeCount`, and the two versions [an entry records about itself](./api.md#what-an-entry-says-about-itself)), then one JSON programme per line, with dates in XMLTV form (`20260807203000 +0200`) so the offset the source wrote them in and how precise it was both survive. `xmltv` writes `.xml`, one small indented document — its root element carrying `date` as XMLTV means it, and its meta in a [processing instruction](./xmltv.md#processing-instructions) rather than in an attribute no DTD defines. Whether an entry then validates is down to the listing in it: a cache keeps whatever the site produced, [provider extensions](#provider-extensions) included, and the merge reads them back — so leaving them out is a decision for the guide, not for the cache. Pick it when something other than this package reads the cache. A driver reads **its own** entries only, so switching means starting the cache again. `sqlite` keeps the whole cache in one `cache.sqlite` under `dir` instead of a file per channel-day — worth it at thousands of channels, where a directory tree is tens of thousands of inodes to walk and back up; it needs Node 24 (or 22.5 with `--experimental-sqlite`), and is [loaded only when named](./api.md#epg-toolscachesqlite). `memory` keeps entries only as long as the process, which is enough for a `build` — its grab and its merge share one cache — and nothing for two commands, since `epg grab` and then `epg merge` would find the second one empty. Or pass a function returning a [driver of your own](./api.md#a-driver-of-your-own) — `({ dir, signal }) => driver`, or a [builder](./api.md#epg-toolscache) that takes your options and returns one — which is how a cache ends up in a database, a bucket, or nowhere at all. It may await, so opening a connection belongs there. | | `staleness` | `Partial` | `{ refetchAll: false, alwaysRefetchDays: 1, maxAgeDays: 7, emptyMaxAgeDays: 1 }` | When a cached day is refetched. `refetchAll` is what `--refresh` sets: every day in the window is fetched whatever the cache holds, and it reaches days behind today, which `alwaysRefetchDays` never does. `alwaysRefetchDays: 1` means today only, `2` today and tomorrow, `0` never force-refetch. `maxAgeDays` busts anything grabbed longer ago than that, and `emptyMaxAgeDays` does the same for an entry that came back with **no programmes** — a source that was briefly broken is asked again the next day instead of leaving a hole for a week, while a channel that genuinely has nothing on costs one request a day rather than one per run. `0` refetches an empty day on any later run; a value as large as `maxAgeDays` turns the distinction off. | | `prune` | `boolean` | `true` | Remove cached days older than today after a successful grab. | | `invalidate` | `(meta, key) => boolean` | — | One more reason a cached entry is void. The stored shape is [already checked](./api.md#what-an-entry-says-about-itself), so this is for what that cannot describe: a release whose *grabbing* changed rather than its storing, a site whose channel ids were renamed, a cache to be emptied gradually. Return `true` and the entry goes, so the day reads as never grabbed. | @@ -206,7 +206,9 @@ epg try example.tv one.example.tv # one channel-day, with the working shown epg validate # read the guide and report what is wrong with it epg channels --against playlist.m3u # which wanted channels will get no guide epg filter guide.xml --channels my-60.m3u -o small.xml # subset a guide you have -epg prune # drop cached days older than today +epg cache stats # how much is cached, per site +epg cache show example.tv # what is cached for a site, as XMLTV +epg cache prune # drop cached days older than today epg init-grabber tv_grab_sk_example # write an XMLTV grabber for this config epg build -d 14 -o public/epg.xml epg build --offset 1 # start the window tomorrow instead of today @@ -238,15 +240,16 @@ epg build -o /home/hts/.hts/tvheadend/epggrab/xmltv.sock # write into a socket | `--grab-every ` | `serve` only: also grab on this interval — `6h`, `30m`, `1d`. Off unless said — see [grabbing on a schedule](#grabbing-on-a-schedule) | | `--grab-at ` | `serve` only: a time of day to line `--grab-every` up with, as `04:00`, on the clock of [`timeZone`](#which-day-is-today). On its own it means once a day, at that time | | `--raw` | `try` only: print the whole payload, not the first 2000 characters | -| `--format ` | `validate`, `channels` and `--dry-run`: `text` (default) or `json` | +| `--format ` | `validate`, `channels`, `--dry-run` and `cache show`/`stats`/`state`: `text` (default) or `json` | | `--strict` | `validate` only: count warnings as failures too | | `--channels ` | `build`/`grab`/`merge`/`serve`/`filter`: keep only these channels and fetch nothing for the rest — ids, or a file naming them. Repeatable — see [subsetting a guide](#keeping-only-some-channels). `filter` needs it or one of `--profile`, `--extensions`, `--no-extensions` and `--indent` — see [subsetting a guide you already have](#subsetting-a-guide-you-already-have) | -| `--indent ` | `build`/`merge`/`serve`/`filter`: pretty-print with this indentation, mirroring `JSON.stringify` | +| `--indent ` | `build`/`merge`/`serve`/`filter`/`cache show`: pretty-print with this indentation, mirroring `JSON.stringify` | | `--profile ` | `build`/`merge`/`serve`/`filter`: shape the guide for one consumer — `tvheadend` or `jellyfin`. See [Output profiles](#output-profiles) | | `--against ` | `channels` only: what you want a guide for — an M3U playlist, a `*.channels.xml` or an XMLTV guide | | `--write` | `channels` only: write the ids the report suggested back into `--against`, in place — or to `-o` | | `--check` | `channels` only: exit 1 unless every wanted channel matched by id | -| `--before ` | `prune` only: remove days before `YYYY-MM-DD`, default today | +| `--before ` | `cache prune` only: remove days before `YYYY-MM-DD`, default today | +| `--keys` | `cache state` with a group only: the keys it holds, not what it holds — see [looking inside the cache](#looking-inside-the-cache) | | `--log-level ` | how much to report: `error`, `warn`, `info` (default) or `debug` | | `-v, --verbose` | same as `--log-level debug` — every channel-day | | `-q, --quiet` | same as `--log-level error` — failures only. Beats `--verbose` if both are given | @@ -356,7 +359,7 @@ Node's own `fs` and stream APIs take one: | requests | in flight through the site's client, queued ones dropped | | the rate-limit hold after a `429` | the wait is abandoned, nothing is resent | | cache reads and writes | a write stops **before** its rename, so an entry is either there in full or not at all | -| `epg prune` | between days — whole days removed, never half of one | +| `epg cache prune` | between days — whole days removed, never half of one | | parsing a guide | between chunks, and the file descriptor closes with it | | writing a guide | between elements, and the part-written file is discarded | | a socket output | the connection is given up on, rather than waiting to be read from | @@ -1528,6 +1531,51 @@ it let one site that fetches a detail per programme fill every slot in the run with parses waiting on its own pacing: every other site's parses, however cheap, then queued behind it. +### Looking inside the cache + +An entry is kept as its codec's bytes, so a `cache.sqlite` row holds a blob and +`sqlite3` shows nothing worth reading. `epg cache` reads a cache the way a run +does, through its driver, whichever driver that is: + +```sh +epg cache stats # how much is cached, per site +epg cache show example.tv # every programme cached for a site, as XMLTV +epg cache show example.tv one.example.tv 2026-07-17 --format json # one day, a programme per line +epg cache state example.tv # the groups a site remembers between runs +epg cache state example.tv validators --keys # only the keys one of them holds +epg cache prune --before 2026-07-01 # drop the days before that one +``` + +Looking changes nothing — a sqlite cache another version wrote included, whose +tables only a run replaces. An entry a run could not vouch for — written in a +shape this version does not read, or with a meta nobody can read — is counted +as unreadable by `stats`, and one the config's `invalidate` voids as voided; +`show` leaves either out, with a line on stderr saying why, and one whose +programmes do not read as well. It stays where it is. A run removes it — one +with a bad meta when it first asks about the day, one with bad programmes when +it reads them, saying so as a warning of the site's — and a run grabs the day +again, as long as the site is still in the config. + +An entry the store could not open at all — a file this user may not read — is +said on stderr, and counted as one that could not be read, by `stats`, or left +out by `show`; the look goes on to the rest. A run fails on one, so it is worth +the line. A channel's directory this user may not read is an error of the +look's own, since what is in it cannot even be counted; only a directory beside +the sites — a mount's `.Trash-0` — is passed over. + +`show` writes to stdout, as XMLTV indented by two unless `--indent` says +otherwise, and takes no `-o`, `--profile` or `--extensions`: it shows what is +cached, not a guide. + +`stats` reads only what each entry says about itself — when it was grabbed and +how many programmes it holds — so for a cache of files it opens every entry but +reads none of the listings. `show` and `state` exit 1 when there is nothing for +what was asked, or nothing of it this version can read; `stats` reports an +empty cache and exits 0. + +A [driver of your own](./api.md#a-driver-of-your-own) answers all of this through its +`list()`. + ### Sites in threads of their own Every site's code runs on one thread by default, and so does every site's diff --git a/docs/site-config.md b/docs/site-config.md index 6571944..8ef5ffe 100644 --- a/docs/site-config.md +++ b/docs/site-config.md @@ -372,7 +372,10 @@ programme(zonedXmltvDate(item.start, dayZone), item.title); ``` A failed `request` fails every channel-day it covered; one channel-day's -`parseDay` error only drops that channel-day. +`parseDay` error only drops that channel-day. A programme handed back without a +`start` that is a real time — `new Date('')`, a field the source left empty — +is left out of its channel-day, with a warning saying how many: no guide can +place it, and the cache could not keep it. ### Remembering something between runs diff --git a/src/build.ts b/src/build.ts index 95ed693..26495a9 100644 --- a/src/build.ts +++ b/src/build.ts @@ -5,7 +5,7 @@ import { FsXmltvCacheDriver, MemoryCacheDriver, } from './cache/main.js'; -import type { CacheDriver, CacheStore } from './cache/main.js'; +import type { CacheDriver, CacheManagerOptions, CacheStore } from './cache/main.js'; import { resolveSessions } from './grabber/channels.js'; import { grabWith } from './grabber/grab.js'; import { covered, coverageOf } from './grabber/main.js'; @@ -90,6 +90,7 @@ export interface RunOptions { export async function createCacheStore( config: EpgConfig, signal?: AbortSignal, + says?: Pick, ): Promise { const options = { dir: config.cache?.dir ?? path.join(process.cwd(), '.epg-cache'), signal }; const driver = config.cache?.driver; @@ -97,6 +98,7 @@ export async function createCacheStore( return new CacheManager({ driver: await driverFor(options, driver), ...(config.cache?.invalidate ? { invalidate: config.cache.invalidate } : {}), + ...(says?.warn ? { warn: says.warn } : {}), }); } @@ -126,10 +128,10 @@ async function driverFor( // fails, on a Node that has never heard of it. Which is worth saying in // terms of the choice that led here rather than leaving Node to say "No // such built-in module" about something the config never mentioned. - try { - const { SqliteCacheDriver } = await import('./cache/sqlite-driver.js'); + let sqlite: typeof import('./cache/sqlite-driver.js'); - return new SqliteCacheDriver(options); + try { + sqlite = await import('./cache/sqlite-driver.js'); } catch (error) { throw new GrabberError( `The sqlite cache driver needs Node 24 or newer (22.5 with --experimental-sqlite): ${ @@ -137,6 +139,10 @@ async function driverFor( }`, ); } + + // Outside the catch: a place the database could never be made is a + // mistake in the config, not in the runtime, and says so itself. + return new sqlite.SqliteCacheDriver(options); } default: { // Unreachable from TypeScript, which is what the `never` says: every name @@ -168,7 +174,12 @@ async function cacheFor( return { cache: options.cache, release: async () => {} }; } - const cache = await createCacheStore(config, options.signal); + // An entry removed for programmes that would not read is its site's to + // say, whether a grab came across it or a merge. + const emit = emitter(options); + const cache = await createCacheStore(config, options.signal, { + warn: (message, key) => emit({ type: 'site:warning', site: key.site, message }), + }); return { cache, release: () => cache.close() }; } diff --git a/src/cache/codec.ts b/src/cache/codec.ts index 2ee3495..42f1780 100644 --- a/src/cache/codec.ts +++ b/src/cache/codec.ts @@ -22,9 +22,11 @@ import { parseXmltvString, serializeProgramme } from '../xmltv/main.js'; import type { SerializeOptions } from '../xmltv/main.js'; -import { formatXmltvDate, getXmltvOffset, parseXmltvDate } from '../xmltv/date.js'; +import { formatXmltvDate, getXmltvOffset, parseXmltvDate, XmltvDateError } from '../xmltv/date.js'; +import { XmltvInputError } from '../xmltv/scan.js'; import type { XmltvProgramme } from '../xmltv/types.js'; -import type { StoredProgramme } from './types.js'; +import { errorMessage } from '../core/error.js'; +import type { StoredEntryMeta, StoredProgramme } from './types.js'; /** * A channel-day's programmes as bytes, and back — what a driver of your own @@ -33,8 +35,46 @@ import type { StoredProgramme } from './types.js'; export interface CacheCodec { /** The programmes as the bytes a store keeps. */ encode(programmes: XmltvProgramme[]): Uint8Array; - /** The programmes those bytes stand for — what a driver's `read` handed back. */ - decode(body: Uint8Array): XmltvProgramme[]; + /** + * The programmes those bytes stand for — what a driver's `read` handed back, + * with `meta`, what the entry says about itself. Throws + * {@link CorruptEntryError} for bytes that are not what it wrote: a codec + * that gives back exactly what it was given can tell a cut that still reads + * by `meta.programmeCount`. + */ + decode(body: Uint8Array, meta: StoredEntryMeta): XmltvProgramme[]; +} + +/** + * Bytes a codec cannot read as the programmes it wrote — a line cut short, an + * edit, a disk that filled — thrown by its `decode` to say so. + * + * What it tells the {@link CacheManager} is that the entry is damaged, not the + * codec: a run removes it, says so as a warning of its site's, and the next run + * grabs the day again. Anything else a codec throws is taken for the codec's + * own failing — a bug, a format changed without a schema to say so — and thrown + * as it is, since for every entry at once a removal would empty the cache and + * write an empty guide over the last good one. The codecs of {@link CODECS} + * throw it for whatever they cannot read, the cause beside it, and for a count + * other than the one the entry was written with. + * + * `instanceof` knows one by its name as well, so a driver whose package brings + * a copy of this one of its own throws one the manager knows. + */ +export class CorruptEntryError extends Error { + override readonly name = 'CorruptEntryError'; + + /** + * An instance, or an error of the same name — another copy's. Asked of a + * subclass, only an instance of it is one: what a subclass adds, another + * copy's error does not have. + */ + static [Symbol.hasInstance](value: unknown): value is CorruptEntryError { + return ( + Function.prototype[Symbol.hasInstance].call(this, value) || + (this === CorruptEntryError && value instanceof Error && value.name === 'CorruptEntryError') + ); + } } /** @@ -136,28 +176,30 @@ export const CODECS = { ndjson: { encode: (programmes) => utf8(programmes.map(toLine).join('\n')), // A blank line is what the end of a file looks like. - decode: (body) => - text(body) - .split('\n') - .filter((line) => line.trim() !== '') - .map(fromLine), + decode: (body, meta) => + readAs('ndjson', meta, () => + text(body) + .split('\n') + .filter((line) => line.trim() !== '') + .map(fromLine), + ), }, /** The records' array, as JSON: what sqlite and memory keep. */ json: { encode: (programmes) => utf8(JSON.stringify(programmes.map(toRecord))), - decode: (body) => { - try { + decode: (body, meta) => + readAs('json', meta, () => { + // Bytes that are not the records' array say nothing about their + // programmes, and are as damaged as a line cut short: the day is + // grabbed again rather than served as empty. const parsed = JSON.parse(text(body)) as unknown; - return Array.isArray(parsed) ? (parsed as StoredProgramme[]).map(fromRecord) : []; - } catch { - // Bytes that are not JSON say nothing about their programmes, and what - // that means for the entry is not a codec's to decide: the meta beside - // them is what the manager judges, and a count disagreeing with what - // came back is a day worth grabbing again. - return []; - } - }, + if (!Array.isArray(parsed)) { + throw new CorruptEntryError('An entry of the json codec holds no array of records'); + } + + return (parsed as StoredProgramme[]).map(fromRecord); + }), }, /** * `` elements, as XMLTV writes them: what the xmltv driver puts in @@ -168,13 +210,17 @@ export const CODECS = { xmltv: { encode: (programmes) => utf8(programmes.map((programme) => serializeProgramme(programme, XMLTV_ENTRY)).join('')), - decode: (body) => { - const document = text(body); + decode: (body, meta) => + readAs('xmltv', meta, () => { + const document = text(body); - return parseXmltvString( - /^\s*(]|$)/.test(document) ? `${document}` : document, - ).programmes; - }, + // A programme of no channel is one this codec wrote — an entry is a + // channel's whichever way — so it is read back rather than dropped. + return parseXmltvString( + /^\s*(]|$)/.test(document) ? `${document}` : document, + { tolerateMissingId: true }, + ).programmes; + }), }, } satisfies Record; @@ -232,6 +278,58 @@ function text(body: Uint8Array): string { return DECODER.decode(body); } +/** + * What a codec of {@link CODECS} reads — or, for bytes it cannot, a + * {@link CorruptEntryError}: what a parser throws for its input — JSON that is + * not JSON, a date that is no date, a document that is not XMLTV — is the + * entry's doing. Anything else is thrown as it is: a fault of this code taken + * for a damaged entry would remove every entry it read. + * + * And for fewer or more than `meta` says the entry holds, since these give back + * exactly what they were given and the count was taken from the very + * programmes encoded: a cut that still reads — an ndjson entry cut between + * lines, an xmltv document that ends early — reads as fewer. + */ +function readAs( + codec: string, + meta: StoredEntryMeta, + decode: () => XmltvProgramme[], +): XmltvProgramme[] { + let programmes: XmltvProgramme[]; + + try { + programmes = decode(); + } catch (error) { + // The json codec's own CorruptEntryError among them, said as it is. + if (!isInputError(error)) { + throw error; + } + + throw new CorruptEntryError( + `An entry of the ${codec} codec does not read: ${errorMessage(error)}`, + { cause: error }, + ); + } + + if (programmes.length !== meta.programmeCount) { + throw new CorruptEntryError( + `An entry of the ${codec} codec reads as ${programmes.length} programmes, ` + + `of the ${meta.programmeCount} it says it holds`, + ); + } + + return programmes; +} + +/** What a parser throws for input it cannot read, rather than for a fault of its own. */ +function isInputError(error: unknown): boolean { + return ( + error instanceof SyntaxError || + error instanceof XmltvDateError || + error instanceof XmltvInputError + ); +} + /** * One date as it is stored: the XMLTV form, and its trailing offset only when * there is one to keep. diff --git a/src/cache/fs-driver.ts b/src/cache/fs-driver.ts index 0b61a6e..4140e8f 100644 --- a/src/cache/fs-driver.ts +++ b/src/cache/fs-driver.ts @@ -24,12 +24,16 @@ * {@link CacheManager}'s judgement, made the same way for every driver there is. */ +import type { Dirent } from 'node:fs'; import * as fs from 'node:fs/promises'; import * as path from 'node:path'; import { writeFileAtomic } from '../core/atomic.js'; +import { inParallel } from '../core/parallel.js'; import type { CacheCodec, CodecName } from './codec.js'; import type { CacheDriver, + CachedKey, + CacheListOptions, ChannelDayKey, FoundEntry, FoundMeta, @@ -39,8 +43,8 @@ import type { StoredStateMeta, } from './types.js'; -/** How much of a prune's directory walk is in flight at once. */ -const PRUNE_CONCURRENCY = 8; +/** How much of a walk of the cache directory — a prune's, a listing's — is in flight at once. */ +const WALK_CONCURRENCY = 8; /** * How much of an entry is read at a time while looking for its meta. A @@ -149,27 +153,6 @@ export abstract class FsCacheDriver implements CacheDriver { } } - /** - * Run `work` over `items`, `limit` of them at a time. - * - * A prune is thousands of small operations that only wait on the disk, so - * doing them one after another spends the whole time waiting — while doing - * them all at once is the file descriptor storm the rest of this package is - * careful to avoid. Node's own file operations run on a threadpool of four by - * default, so there is little to gain past a handful in flight. - */ - async #inParallel(items: T[], limit: number, work: (item: T) => Promise): Promise { - let next = 0; - - const worker = async (): Promise => { - while (next < items.length) { - await work(items[next++]!); - } - }; - - await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); - } - /** * An open file, a chunk at a time and none of it before it is asked for. * @@ -222,6 +205,34 @@ export abstract class FsCacheDriver implements CacheDriver { return /^\.+$/.test(encoded) ? encoded.replaceAll('.', '%2E') : encoded; } + /** + * A path segment as the name it was made from, or nothing for one this + * driver could not have made: a name it would have made as another segment — + * a mount's `lost+found`, whose `+` it would have escaped, or a `%2f` it + * would have written `%2F` — or one whose `%` is not an escape at all. + * Somebody's own directory, which no key this driver writes is in. + */ + #named(segment: string): string | undefined { + let name: string; + + try { + name = decodeURIComponent(segment); + } catch { + return undefined; + } + + return this.#segment(name) === segment ? name : undefined; + } + + /** + * What this driver's own entry files are called, the day captured. Built when + * asked rather than in the constructor, where a subclass has yet to say what + * its extension is. + */ + #entryFile(): RegExp { + return new RegExp(`^(\\d{4}-\\d{2}-\\d{2})\\.${this.extension}$`); + } + #channelDir(key: ChannelDayKey): string { return path.join(this.dir, this.#segment(key.site), this.#segment(key.channelId)); } @@ -360,63 +371,216 @@ export abstract class FsCacheDriver implements CacheDriver { async prune(options: { before: string }): Promise { let removed = 0; - let sites; - - try { - sites = await fs.readdir(this.dir, { withFileTypes: true }); - } catch (error) { - if (this.#isNotFound(error)) { - return 0; - } - - throw error; - } - - // This driver's own entry files. Built here rather than in the constructor, - // where a subclass has yet to say what its extension is, and a prune is - // once a run either way. - const entryFile = new RegExp(`^(\\d{4}-\\d{2}-\\d{2})\\.${this.extension}$`); + const entryFile = this.#entryFile(); + // Walked as a listing walks it, so what is listed is what is pruned: a site + // moved to another disk and linked back included, and beside the sites a + // directory this process may not read — a mount's `.Trash-0` — or could + // not have named — its `lost+found` — passed over. + const sites = (await this.#children(this.dir, 'named')).filter( + (entry) => entry.directory && this.#named(entry.name) !== undefined, + ); - for (const site of sites.filter((entry) => entry.isDirectory())) { + for (const site of sites) { // Between sites, and between channels below: `readdir` and `rm` take no // signal of their own, and a prune is a walk rather than one long // operation — so this is where it stops, having removed whole days. this.signal?.throwIfAborted(); const sitePath = path.join(this.dir, site.name); - const channels = await fs.readdir(sitePath, { withFileTypes: true }); + const channels = (await this.#children(sitePath, 'found')).filter( + (entry) => entry.directory && this.#named(entry.name) !== undefined, + ); - await this.#inParallel( - channels.filter((entry) => entry.isDirectory()), - PRUNE_CONCURRENCY, - async (channel) => { - this.signal?.throwIfAborted(); + await inParallel(channels, WALK_CONCURRENCY, async (channel) => { + this.signal?.throwIfAborted(); - const channelPath = path.join(sitePath, channel.name); - // Counted after the await, not `+= await`: that reads the total - // before waiting and writes it back after, so two of these running - // together would each add to the same stale number. - const pruned = await this.#pruneChannelDir(channelPath, options.before, entryFile); + const channelPath = path.join(sitePath, channel.name); + // Counted after the await, not `+= await`: that reads the total + // before waiting and writes it back after, so two of these running + // together would each add to the same stale number. + const pruned = await this.#pruneChannelDir(channelPath, options.before, entryFile); - removed += pruned; + removed += pruned; + // A link is left in place, emptied or not: `rmdir` does not follow + // one, and where it points was somebody's choice. + if (!channel.link) { await this.#removeIfEmpty(channelPath); - }, - ); + } + }); - await this.#removeIfEmpty(sitePath); + if (!site.link) { + await this.#removeIfEmpty(sitePath); + } } return removed; } + /** + * A walk of the cache directory, a site at a time: each channel directory + * holds a site's entries, and each `.json` beside them a group of its + * state. A file of anything else — one being written, one somebody put there + * — is passed over, as is a name this driver could not have made. + * + * Only a directory found beside the sites is passed over when it cannot be + * read — a mount's `.Trash-0` holds nothing this process wrote. Inside a site + * everything is this driver's, so a channel that cannot be read says so, as + * a run reading it would, rather than be listed as holding nothing. + */ + async *list(options: CacheListOptions = {}): AsyncIterable { + const entryFile = this.#entryFile(); + const sites = + options.site === undefined + ? (await this.#children(this.dir, 'named')) + .filter((entry) => entry.directory) + .map((entry) => entry.name) + : [this.#segment(options.site)]; + + for (const segment of sites) { + // Between sites and between channels, as a prune is: `readdir` takes no + // signal of its own. + this.signal?.throwIfAborted(); + + const site = this.#named(segment); + + if (site === undefined) { + continue; + } + + const sitePath = path.join(this.dir, segment); + + // One channel's directory, where its site was named too, and none of the + // rest of the site. In a walk of every site it is looked for among what + // each holds below: a site found unreadable has none to look in. + if (options.channelId !== undefined && options.site !== undefined) { + if (options.kind !== 'state') { + yield* this.#entriesIn( + path.join(sitePath, this.#segment(options.channelId)), + site, + options.channelId, + entryFile, + ); + } + + continue; + } + + for (const child of await this.#children( + sitePath, + options.site === undefined ? 'found' : 'named', + )) { + if (child.file && child.name.endsWith('.json')) { + // A group is a site's, and no channel's. + const key = + options.kind === 'entry' || options.channelId !== undefined + ? undefined + : this.#named(child.name.slice(0, -'.json'.length)); + + if (key !== undefined) { + yield { kind: 'state', site, key }; + } + + continue; + } + + // Only groups asked for, and the channel directories are not read. + const channelId = + child.directory && options.kind !== 'state' ? this.#named(child.name) : undefined; + + if ( + channelId !== undefined && + (options.channelId === undefined || channelId === options.channelId) + ) { + yield* this.#entriesIn(path.join(sitePath, child.name), site, channelId, entryFile); + } + } + } + } + + /** The entries one channel directory holds. */ + async *#entriesIn( + channelPath: string, + site: string, + channelId: string, + entryFile: RegExp, + ): AsyncGenerator { + this.signal?.throwIfAborted(); + + for (const file of await this.#children(channelPath, 'named')) { + const day = file.file ? entryFile.exec(file.name)?.[1] : undefined; + + if (day !== undefined) { + yield { kind: 'entry', key: { site, channelId, day } }; + } + } + } + + /** + * What a directory holds, and what each is — a symbolic link as what it + * points at, since a read follows one: a site moved to another disk and + * linked back is listed as it is read. + * + * Nothing where there is no such directory. Nothing, too, where it cannot be + * read and was `found` beside the sites in walking the cache — a mount's + * `.Trash-0` holds nothing this process wrote. Any other that cannot be read + * says so: one asked for by name, and anything inside a site, all of it this + * driver's. + */ + async #children( + dir: string, + how: 'named' | 'found', + ): Promise> { + let entries: Dirent[]; + + try { + entries = await fs.readdir(dir, { withFileTypes: true }); + } catch (error) { + const code = this.#code(error); + + if ( + code === 'ENOENT' || + code === 'ENOTDIR' || + (how === 'found' && (code === 'EACCES' || code === 'EPERM')) + ) { + return []; + } + + throw error; + } + + const children = entries.map((entry) => ({ + name: entry.name, + directory: entry.isDirectory(), + file: entry.isFile(), + link: false, + })); + + // A link is the one thing the entry cannot say, and is asked — only the + // links, a few at a time: a link to nothing is neither. + const links = entries.flatMap((entry, index) => (entry.isSymbolicLink() ? [index] : [])); + + await inParallel(links, WALK_CONCURRENCY, async (index) => { + const { name } = entries[index]!; + const target = await fs.stat(path.join(dir, name)).catch(() => undefined); + + children[index] = { + name, + directory: target?.isDirectory() === true, + file: target?.isFile() === true, + link: true, + }; + }); + + return children; + } + /** Remove all entries in one channel directory older than `before`. */ async #pruneChannelDir(channelPath: string, before: string, entryFile: RegExp): Promise { - const files = await fs.readdir(channelPath); const staleDays = new Set(); - for (const file of files) { - const day = entryFile.exec(file)?.[1]; + for (const file of await this.#children(channelPath, 'named')) { + const day = file.file ? entryFile.exec(file.name)?.[1] : undefined; // String comparison works for YYYY-MM-DD day strings. if (day !== undefined && day < before) { @@ -427,7 +591,7 @@ export abstract class FsCacheDriver implements CacheDriver { // A day's files together, and the days a few at a time: the entries of one // channel are the smallest pieces here, and waiting for each in turn is // most of what a prune used to spend its time on. - await this.#inParallel([...staleDays], PRUNE_CONCURRENCY, async (day) => + await inParallel([...staleDays], WALK_CONCURRENCY, async (day) => fs.rm(path.join(channelPath, `${day}.${this.extension}`), { force: true }), ); diff --git a/src/cache/main.ts b/src/cache/main.ts index e1408e8..5ce5cc0 100644 --- a/src/cache/main.ts +++ b/src/cache/main.ts @@ -1,6 +1,6 @@ export { isStale } from './staleness.js'; export { CacheManager, CACHE_SCHEMA, STATE_SCHEMA } from './manager.js'; -export { CODECS, fromRecord, toRecord } from './codec.js'; +export { CODECS, CorruptEntryError, fromRecord, toRecord } from './codec.js'; export type { CacheCodec, CodecName } from './codec.js'; export { FsCacheDriver } from './fs-driver.js'; export { MemoryCacheDriver } from './memory-driver.js'; diff --git a/src/cache/manager.ts b/src/cache/manager.ts index b274d21..88932df 100644 --- a/src/cache/manager.ts +++ b/src/cache/manager.ts @@ -21,15 +21,19 @@ * neither of them knows which driver is underneath. */ +import { errorMessage } from '../core/error.js'; import type { XmltvProgramme } from '../xmltv/types.js'; -import { codecOf, type CacheCodec, type CodecName } from './codec.js'; +import { codecOf, CorruptEntryError, type CacheCodec, type CodecName } from './codec.js'; import type { CacheDriver, + CachedKey, CacheEntryMeta, + CacheListOptions, CacheManagerOptions, CacheStore, ChannelDayKey, FoundMeta, + Inspected, StateEntry, StoredEntryMeta, StoredStateMeta, @@ -67,12 +71,14 @@ export const STATE_SCHEMA = 1; export class CacheManager implements CacheStore, AsyncDisposable { readonly #driver: CacheDriver; readonly #invalidate: CacheManagerOptions['invalidate']; + readonly #warn: CacheManagerOptions['warn']; /** What makes the driver's bytes, settled once: a named codec looked up, a driver's own taken as it is. */ readonly #codec: CacheCodec; constructor(options: CacheManagerOptions) { this.#driver = options.driver; this.#invalidate = options.invalidate; + this.#warn = options.warn; this.#codec = codecOf(options.driver.codec); } @@ -100,16 +106,7 @@ export class CacheManager implements CacheStore, AsyncDisposable { * these calls to run in parallel. */ async getMetas(keys: readonly ChannelDayKey[]): Promise> { - const driver = this.#driver; - const found: Array = []; - - if (driver.readMetas !== undefined) { - found.push(...(await driver.readMetas(keys))); - } else { - for (const key of keys) { - found.push(await driver.readMeta(key)); - } - } + const found = await this.#readMetas(keys); // Judged the same way one at a time is, which includes removing an entry // that cannot answer for itself — so a batch is a shortcut in how the store @@ -123,15 +120,58 @@ export class CacheManager implements CacheStore, AsyncDisposable { ); } + /** + * What the driver says of several keys, in the order asked: in one question + * where it can answer one, and a key at a time — here, so no driver has to + * write the loop — where it cannot. + */ + async #readMetas(keys: readonly ChannelDayKey[]): Promise> { + if (this.#driver.readMetas !== undefined) { + return this.#driver.readMetas(keys); + } + + const found: Array = []; + + for (const key of keys) { + found.push(await this.#driver.readMeta(key)); + } + + return found; + } + + /** + * An entry's programmes, if it is one to build a guide from — what + * {@link inspect} finds, and the one it would not use removed, so the day + * reads as never grabbed and the next run grabs it again. + * + * The meta is judged before the programmes are served: a store hands back + * whatever was last put there, and an entry that cannot answer for itself is + * not one to build a guide from. Nor is one whose programmes do not read back + * as what it says it holds — a line cut short, an edit — which a run would + * otherwise fail on, or serve as an empty day until it aged out. + */ async read(key: ChannelDayKey): Promise { - const found = await this.#driver.read(key); + const found = await this.inspect(key); + + if (found === undefined) { + return; + } + + if (found.status !== 'readable') { + await this.#driver.delete(key); - // The meta is judged before the programmes are served: a store hands back - // whatever was last put there, and an entry that cannot answer for itself is - // not one to build a guide from. - return found !== undefined && (await this.#verified(key, found.meta)) !== undefined - ? this.#codec.decode(found.body) - : undefined; + if (found.status === 'corrupt') { + this.#warn?.( + `${key.channelId} ${key.day}: removed from the cache, its programmes unreadable ` + + `(${errorMessage(found.error)}) — the day is grabbed again by the next run`, + key, + ); + } + + return; + } + + return found.programmes; } async write( @@ -208,21 +248,19 @@ export class CacheManager implements CacheStore, AsyncDisposable { * is the whole of what a cache has to do about it. */ async getState(site: string, key: string): Promise { - const found = await this.#driver.readState(site, key); + const found = await this.inspectState(site, key); if (found === undefined) { return; } - const meta = this.#stateMeta(found.meta); - - if (meta === undefined) { + if (found.status !== 'readable') { await this.#driver.deleteState(site, key); return; } - return { data: found.data, meta }; + return { data: found.data, meta: found.meta }; } async setState( @@ -243,6 +281,116 @@ export class CacheManager implements CacheStore, AsyncDisposable { return this.#driver.prune(options); } + /** + * Everything the store holds, by key, narrowed by whatever of `options` is + * given — see {@link CacheDriver.list}. Exactly that: a driver may answer + * with more than it was asked, and what it adds is left out here, so no + * caller has to check again. + */ + async *list(options: CacheListOptions = {}): AsyncIterable { + const { site, channelId, kind } = options; + + // Said plainly for a driver written before `list` was asked of every one, + // rather than as whatever calling nothing throws. + if (typeof this.#driver.list !== 'function') { + throw new TypeError( + "This cache's driver has no list(), which a driver needs for anything to look inside it — " + + 'see "A driver of your own" in docs/api.md', + ); + } + + for await (const held of this.#driver.list(options)) { + if ( + (kind === undefined || held.kind === kind) && + (site === undefined || (held.kind === 'entry' ? held.key.site : held.site) === site) && + (channelId === undefined || (held.kind === 'entry' && held.key.channelId === channelId)) + ) { + yield held; + } + } + } + + /** + * What several entries say about themselves, as {@link getMetas} judges them + * — and nothing removed: one a run would remove comes back saying why. What + * `epg cache stats` counts with. + * + * `undefined` for a key the store has no entry under. + */ + async inspectMetas( + keys: readonly ChannelDayKey[], + ): Promise | undefined>> { + const found = await this.#readMetas(keys); + + return keys.map((key, index) => { + const entry = found[index]; + + if (entry === undefined) { + return; + } + + return this.#judgedEntry(key, entry.meta); + }); + } + + /** + * One entry and its programmes, as {@link read} judges it — and left where it + * is, if a run would remove it. What `epg cache show` writes. + */ + async inspect( + key: ChannelDayKey, + ): Promise | undefined> { + const found = await this.#driver.read(key); + + if (found === undefined) { + return; + } + + const judged = this.#judgedEntry(key, found.meta); + + if (judged.status !== 'readable') { + return judged; + } + + // A meta that reads and programmes its codec says are damaged — a line + // cut short, an edit — is one entry to say so of, not the end of a look or + // a run. Whatever else a codec throws may be the codec's doing rather + // than the entry's, and is thrown: taken for a damaged entry, a codec that + // fails on every one would empty the cache. + try { + return { ...judged, programmes: this.#codec.decode(found.body, judged.meta) }; + } catch (error) { + if (!(error instanceof CorruptEntryError)) { + throw error; + } + + return { status: 'corrupt', meta: judged.meta, error }; + } + } + + /** + * One group of a site's state, as {@link getState} judges it — and left + * where it is, if a run would remove it. What `epg cache state` shows. + */ + async inspectState( + site: string, + key: string, + ): Promise | undefined> { + const found = await this.#driver.readState(site, key); + + if (found === undefined) { + return; + } + + const judged = this.#judged( + found.meta, + STATE_SCHEMA, + (meta) => typeof meta.writtenAt === 'string' && typeof meta.writtenBy === 'string', + ); + + return judged.status === 'readable' ? { ...judged, data: found.data } : judged; + } + async close(): Promise { await this.#driver.close?.(); } @@ -276,46 +424,65 @@ export class CacheManager implements CacheStore, AsyncDisposable { key: ChannelDayKey, meta: Partial | undefined, ): Promise { - if ( - meta === undefined || - typeof meta.grabbedAt !== 'string' || - typeof meta.programmeCount !== 'number' || - typeof meta.writtenBy !== 'string' || - // A shape this code does not write is one it cannot read as it was meant, - // whether the entry is older than this version or newer than it: a cache - // shared with something that has moved on is not ours to interpret. - meta.schema !== CACHE_SCHEMA - ) { + const judged = this.#judgedEntry(key, meta); + + if (judged.status !== 'readable') { await this.#driver.delete(key); return; } - const stored = meta as StoredEntryMeta; - - if (this.#invalidate?.(stored, key) === true) { - await this.#driver.delete(key); + return judged.meta; + } - return; - } + /** + * An entry's meta, where it is one a run would use — or, where it is not, + * why: the one judgement, for a run that then removes the entry and for a + * look that leaves it. What {@link #judged} says of any meta, and the + * config's `invalidate` beside it. + */ + #judgedEntry( + key: ChannelDayKey, + meta: Partial | undefined, + ): Inspected<{ meta: StoredEntryMeta }> { + const judged = this.#judged( + meta, + CACHE_SCHEMA, + // An entry that cannot say when it was grabbed or how much it holds is + // one no decision can be made about: not fresh, not stale, not empty. + (found) => + typeof found.grabbedAt === 'string' && + typeof found.programmeCount === 'number' && + typeof found.writtenBy === 'string', + ); - return stored; + return judged.status === 'readable' && this.#invalidate?.(judged.meta, key) === true + ? { status: 'invalidated', meta: judged.meta } + : judged; } /** - * The envelope of a state group worth reading, or nothing. + * What something in the store says about itself — an entry's meta, a + * group's envelope — judged against the shape this code writes, `schema`, and + * the fields `complete` asks for: one a run would read, or why not. * - * Nothing to invalidate here and nothing to count: a group says when it was - * written and what wrote it, and the schema decides as it does for an entry — - * a shape this code does not write is one it cannot read as it was meant, - * whether older or newer. + * A shape this code does not write is one it cannot read as it was meant, + * whether older than this version or newer: a cache shared with something + * that has moved on is not ours to interpret. It is asked first, so an older + * one is of another schema, whatever fields it has, rather than merely one + * that cannot be read. */ - #stateMeta(meta: Partial | undefined): StoredStateMeta | undefined { - return meta !== undefined && - typeof meta.writtenAt === 'string' && - typeof meta.writtenBy === 'string' && - meta.schema === STATE_SCHEMA - ? (meta as StoredStateMeta) - : undefined; + #judged( + meta: Partial | undefined, + schema: number, + complete: (meta: Partial) => boolean, + ): Inspected<{ meta: M }, M> { + if (typeof meta?.schema === 'number' && meta.schema !== schema) { + return { status: 'incompatible', meta: meta as Partial & { schema: number } }; + } + + return meta?.schema === schema && complete(meta) + ? { status: 'readable', meta: meta as M } + : { status: 'unreadable', meta }; } } diff --git a/src/cache/memory-driver.ts b/src/cache/memory-driver.ts index 41fb799..f5bbfac 100644 --- a/src/cache/memory-driver.ts +++ b/src/cache/memory-driver.ts @@ -30,6 +30,8 @@ import type { CacheCodec, CodecName } from './codec.js'; import type { CacheDriver, + CachedKey, + CacheListOptions, ChannelDayKey, FoundEntry, FoundMeta, @@ -164,6 +166,41 @@ export class MemoryCacheDriver implements CacheDriver { return data === undefined ? undefined : (JSON.parse(JSON.stringify(data)) as unknown); } + async *list(options: CacheListOptions = {}): AsyncIterable { + const { site: only, channelId: channel, kind } = options; + + for (const [site, channels] of kind === 'state' ? [] : this.#narrowed(this.#sites, only)) { + for (const [channelId, days] of this.#narrowed(channels, channel)) { + for (const day of days.keys()) { + yield { kind: 'entry', key: { site, channelId, day } }; + } + } + } + + // A channel's entries are all a channel has: state is a site's. + for (const [site, groups] of kind === 'entry' || channel !== undefined + ? [] + : this.#narrowed(this.#state, only)) { + for (const key of groups.keys()) { + yield { kind: 'state', site, key }; + } + } + } + + /** + * A map's one pair under `key`, asked for rather than walked to — or all of + * them, where no key is given. + */ + #narrowed(map: Map, key: string | undefined): Iterable<[string, V]> { + if (key === undefined) { + return map; + } + + const value = map.get(key); + + return value === undefined ? [] : [[key, value]]; + } + async prune(options: { before: string }): Promise { let removed = 0; diff --git a/src/cache/no-cache-driver.ts b/src/cache/no-cache-driver.ts index 0765e4f..84acd41 100644 --- a/src/cache/no-cache-driver.ts +++ b/src/cache/no-cache-driver.ts @@ -14,7 +14,7 @@ */ import type { CacheCodec, CodecName } from './codec.js'; -import type { CacheDriver, FoundEntry, FoundMeta, FoundState } from './types.js'; +import type { CacheDriver, CachedKey, FoundEntry, FoundMeta, FoundState } from './types.js'; export class NoCacheDriver implements CacheDriver { /** Never asked to make or read anything, and named so it is a codec a thread has. */ @@ -48,4 +48,7 @@ export class NoCacheDriver implements CacheDriver { async writeState(): Promise {} async deleteState(): Promise {} + + /** Holding nothing, it has nothing to list. */ + async *list(): AsyncIterable {} } diff --git a/src/cache/sqlite-driver.ts b/src/cache/sqlite-driver.ts index 8853cdf..a796e54 100644 --- a/src/cache/sqlite-driver.ts +++ b/src/cache/sqlite-driver.ts @@ -19,13 +19,15 @@ * filesystem driver is careful about cannot happen here at all. */ -import { mkdirSync } from 'node:fs'; +import { accessSync, constants, existsSync, mkdirSync, statSync } from 'node:fs'; import * as path from 'node:path'; import { DatabaseSync } from 'node:sqlite'; import type { StatementSync } from 'node:sqlite'; import type { CacheCodec, CodecName } from './codec.js'; import type { CacheDriver, + CachedKey, + CacheListOptions, ChannelDayKey, FoundEntry, FoundMeta, @@ -40,8 +42,9 @@ import type { * A different question from an entry's `schema`, which says what one row * *holds*: this says what the columns *are*, and a mismatch is not something * re-grabbing a day could fix, since the statements would not run at all. So the - * table is dropped and made again — a cache is disposable, and that is the whole - * of the migration story here as everywhere else in this module. + * table is dropped and made again, by the first write — until then a read + * finds nothing — since a cache is disposable, and that is the whole of the + * migration story here as everywhere else in this module. * * 1 — site, channel, day, the four meta fields, and the programmes as JSON. * 2 — a `state` table beside it: what each site remembers between runs. @@ -50,6 +53,12 @@ import type { */ const TABLE_VERSION = 3; +/** How many keys {@link SqliteCacheDriver.list} reads at a time. */ +const LIST_PAGE = 1000; + +/** What asks a database which version of the tables it holds. */ +const VERSION = 'PRAGMA user_version'; + /** What the database is called inside the cache directory. */ const DEFAULT_FILE = 'cache.sqlite'; @@ -163,8 +172,16 @@ export class SqliteCacheDriver implements CacheDriver { /** The records' array, as JSON — kept in a row as the bytes the codec made. */ readonly codec: CodecName | CacheCodec = 'json'; - readonly #db: DatabaseSync; + readonly #file: string; readonly #signal: AbortSignal | undefined; + /** Opened when first asked for anything, and only then — see {@link #prepared}. */ + #db: DatabaseSync | undefined; + /** Whether the tables are the ones this code makes, as far as it has found. */ + #current = false; + /** Whether the file is ready to write: logging ahead, and the tables this code's. */ + #writable = false; + /** Whether {@link close} was asked for — after which nothing is opened again. */ + #closed = false; /** * Statements prepared once and reused, which is most of why this is quick: a * run asks the same handful of questions about every channel-day in the @@ -172,60 +189,161 @@ export class SqliteCacheDriver implements CacheDriver { */ readonly #statements = new Map(); + /** + * Nothing is opened or made here: a driver made only to be looked into — + * `epg cache` — must not make a database where there was none, or replace + * the tables of one another version wrote. See {@link #prepared}. + */ constructor(options: SqliteCacheDriverOptions) { this.#signal = options.signal; + this.#file = options.file ?? path.join(options.dir, DEFAULT_FILE); + + // Refused now rather than at the first write: a run asks the cache before + // it fetches anything, so a database that could never be made would be + // found out only after every request had been made for nothing. + if (this.#file !== ':memory:' && !existsSync(this.#file)) { + this.#canMake(this.#file); + } + } - const file = options.file ?? path.join(options.dir, DEFAULT_FILE); + /** + * Whether the file could be made where it is to go, asked without making + * anything: the nearest directory there is, on its way, has to be writable. + */ + #canMake(file: string): void { + let dir = path.dirname(path.resolve(file)); - // SQLite will not make the directory its file goes in, and on a first run - // that directory is as likely as not to be missing. - if (file !== ':memory:') { - mkdirSync(path.dirname(path.resolve(file)), { recursive: true }); + while (!existsSync(dir) && path.dirname(dir) !== dir) { + dir = path.dirname(dir); } - this.#db = new DatabaseSync(file); - // Write-ahead logging, so a merge reading the cache is not held up by a grab - // writing it; and the relaxed sync because this is a cache — a day lost to a - // power cut is a day grabbed again. - this.#db.exec('PRAGMA journal_mode = WAL'); - this.#db.exec('PRAGMA synchronous = NORMAL'); + if (!statSync(dir).isDirectory()) { + throw new Error(`The sqlite cache cannot be made at ${file}: ${dir} is not a directory`); + } + + try { + accessSync(dir, constants.W_OK); + } catch (cause) { + throw new Error(`The sqlite cache cannot be made at ${file}: ${dir} is not writable`, { + cause, + }); + } + } + + #open(): void { + const db = new DatabaseSync(this.#file); + + // The relaxed sync because this is a cache — a day lost to a power cut is a + // day grabbed again. + db.exec('PRAGMA synchronous = NORMAL'); // Wait rather than fail when another process holds the write lock: grabbing // two sites into one cache from two commands is a reasonable thing to do. - this.#db.exec('PRAGMA busy_timeout = 5000'); - this.#ensureTable(); + db.exec('PRAGMA busy_timeout = 5000'); + this.#db = db; + } + + #version(): number { + return (this.#statement(VERSION).get() as { user_version: number }).user_version; + } + + /** + * A statement on the open database, prepared the first time it is asked for + * and reused after. Only ever asked once {@link #prepared} has opened it. + */ + #statement(sql: string): StatementSync { + let statement = this.#statements.get(sql); + + if (statement === undefined) { + statement = this.#db!.prepare(sql); + this.#statements.set(sql, statement); + } + + return statement; } /** The table as this code expects it — made, or made again. */ #ensureTable(): void { - const { user_version: version } = this.#db.prepare('PRAGMA user_version').get() as { - user_version: number; - }; - - if (version === TABLE_VERSION) { + if (this.#version() === TABLE_VERSION) { return; } - this.#db.exec('DROP TABLE IF EXISTS entries'); - this.#db.exec('DROP TABLE IF EXISTS state'); + const db = this.#db!; + + db.exec('DROP TABLE IF EXISTS entries'); + db.exec('DROP TABLE IF EXISTS state'); for (const statement of CREATE_TABLE) { - this.#db.exec(statement); + db.exec(statement); } // Interpolated because a pragma takes no parameter — and it is an integer // constant of ours, not anything a caller can reach. - this.#db.exec(`PRAGMA user_version = ${TABLE_VERSION}`); + db.exec(`PRAGMA user_version = ${TABLE_VERSION}`); } - #prepared(sql: string): StatementSync { - let statement = this.#statements.get(sql); + /** + * A statement, prepared once and reused, on a database opened the first time + * anything is asked of it. + * + * For a write, the file and its directory are made first, and the tables — + * made again where another version made them. For a read there is no + * statement where there is nothing this code can read: no file yet, or + * tables of another version. That read answers as an empty store would, with + * nothing made and nothing replaced — so a look changes nothing, and a run + * finds nothing it can read and grabs again, as when the tables were replaced + * the moment it opened. Asked again by each read while the answer is no, + * which is one pragma: a run in another process may have made them since. + */ + #prepared(mode: 'read' | 'change', sql: string): StatementSync | undefined; + #prepared(mode: 'write', sql: string): StatementSync; + #prepared(mode: 'read' | 'change' | 'write', sql: string): StatementSync | undefined { + // Closed is closed: a driver used after it would otherwise open a database + // again, which nothing would ever close. + if (this.#closed) { + throw new Error('This sqlite cache is closed'); + } - if (statement === undefined) { - statement = this.#db.prepare(sql); - this.#statements.set(sql, statement); + // Nothing to read or to change where there is no file yet, or its tables + // are another version's: only a write makes or replaces them. + if (mode !== 'write' && !this.#current) { + if (this.#db === undefined) { + if (this.#file !== ':memory:' && !existsSync(this.#file)) { + return undefined; + } + + this.#open(); + } + + this.#current = this.#version() === TABLE_VERSION; + + if (!this.#current) { + return undefined; + } } - return statement; + // Ready to change the file: write-ahead logging, so a merge reading the + // cache is not held up by a grab writing it — and, for a write, the file + // and its tables made first. Said by the first change, whatever was read + // before it, since it is a setting the file keeps: a look leaves the file + // as it found it, and a run puts it back. + if (mode !== 'read' && !this.#writable) { + if (this.#db === undefined) { + // SQLite will not make the directory its file goes in, and on a first + // run that directory is as likely as not to be missing. + if (this.#file !== ':memory:') { + mkdirSync(path.dirname(path.resolve(this.#file)), { recursive: true }); + } + + this.#open(); + } + + this.#db!.exec('PRAGMA journal_mode = WAL'); + this.#ensureTable(); + this.#current = true; + this.#writable = true; + } + + return this.#statement(sql); } #meta(row: MetaRow): StoredEntryMeta { @@ -242,7 +360,7 @@ export class SqliteCacheDriver implements CacheDriver { // The payload column is left out of the query rather than read and dropped, // which is what lets `entries_meta` answer this on its own. - const row = this.#prepared(`SELECT ${META_COLUMNS} FROM entries ${BY_KEY}`).get( + const row = this.#prepared('read', `SELECT ${META_COLUMNS} FROM entries ${BY_KEY}`)?.get( key.site, key.channelId, key.day, @@ -267,14 +385,20 @@ export class SqliteCacheDriver implements CacheDriver { async readMetas(keys: readonly ChannelDayKey[]): Promise> { this.#signal?.throwIfAborted(); - if (keys.length === 0) { - return []; + const statement = + keys.length === 0 + ? undefined + : this.#prepared( + 'read', + `SELECT site, channel, day, ${META_COLUMNS} FROM entries + WHERE (site, channel, day) IN (VALUES ${keys.map(() => '(?, ?, ?)').join(', ')})`, + ); + + if (statement === undefined) { + return keys.map(() => undefined); } - const rows = this.#prepared( - `SELECT site, channel, day, ${META_COLUMNS} FROM entries - WHERE (site, channel, day) IN (VALUES ${keys.map(() => '(?, ?, ?)').join(', ')})`, - ).all( + const rows = statement.all( ...keys.flatMap((key) => [key.site, key.channelId, key.day]), ) as unknown as KeyedMetaRow[]; @@ -298,11 +422,10 @@ export class SqliteCacheDriver implements CacheDriver { async read(key: ChannelDayKey): Promise { this.#signal?.throwIfAborted(); - const row = this.#prepared(`SELECT ${META_COLUMNS}, programmes FROM entries ${BY_KEY}`).get( - key.site, - key.channelId, - key.day, - ) as EntryRow | undefined; + const row = this.#prepared( + 'read', + `SELECT ${META_COLUMNS}, programmes FROM entries ${BY_KEY}`, + )?.get(key.site, key.channelId, key.day) as EntryRow | undefined; return row && { meta: this.#meta(row), body: row.programmes }; } @@ -314,6 +437,7 @@ export class SqliteCacheDriver implements CacheDriver { // there before it — the promise the filesystem driver keeps with an atomic // rename, kept here by the database. this.#prepared( + 'write', `INSERT INTO entries (site, channel, day, ${META_COLUMNS}, programmes) VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (site, channel, day) DO UPDATE SET @@ -339,16 +463,21 @@ export class SqliteCacheDriver implements CacheDriver { async delete(key: ChannelDayKey): Promise { this.#signal?.throwIfAborted(); - this.#prepared(`DELETE FROM entries ${BY_KEY}`).run(key.site, key.channelId, key.day); + // On what is there: nothing is made to delete from. + this.#prepared('change', `DELETE FROM entries ${BY_KEY}`)?.run( + key.site, + key.channelId, + key.day, + ); } async readState(site: string, key: string): Promise { this.#signal?.throwIfAborted(); - const row = this.#prepared(`SELECT ${STATE_COLUMNS} FROM state WHERE site = ? AND key = ?`).get( - site, - key, - ) as StateRow | undefined; + const row = this.#prepared( + 'read', + `SELECT ${STATE_COLUMNS} FROM state WHERE site = ? AND key = ?`, + )?.get(site, key) as StateRow | undefined; if (row === undefined) { return; @@ -374,6 +503,7 @@ export class SqliteCacheDriver implements CacheDriver { this.#signal?.throwIfAborted(); this.#prepared( + 'write', `INSERT INTO state (site, key, ${STATE_COLUMNS}) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT (site, key) DO UPDATE SET @@ -397,7 +527,79 @@ export class SqliteCacheDriver implements CacheDriver { async deleteState(site: string, key: string): Promise { this.#signal?.throwIfAborted(); - this.#prepared('DELETE FROM state WHERE site = ? AND key = ?').run(site, key); + // On what is there: nothing is made to delete from. + this.#prepared('change', 'DELETE FROM state WHERE site = ? AND key = ?')?.run(site, key); + } + + /** + * The entries, a page at a time in the order of `entries_key` — which + * answers each page without touching a payload — and then the groups. + * + * Pages rather than one statement stepped through as its keys are handed + * out: a statement still being stepped holds the connection's read open, and + * a caller that wrote as it went — deleting what it was handed, say — would + * have its writes left uncommitted, and the write lock held against every + * other process, until the listing ended. Each page is read whole before a + * key of it is handed out, so nothing is held across a yield, and a listing + * of 70,000 is never held at once either. + */ + async *list(options: CacheListOptions = {}): AsyncIterable { + this.#signal?.throwIfAborted(); + + const { site, channelId, kind } = options; + const narrowed = [ + ...(site === undefined ? [] : [site]), + ...(channelId === undefined ? [] : [channelId]), + ]; + + if (kind !== 'state') { + let after: { site: string; channel: string; day: string } | undefined; + + do { + const page = (this.#prepared( + 'read', + `SELECT site, channel, day FROM entries${this.#where(site, channelId, after !== undefined)} + ORDER BY site, channel, day LIMIT ${LIST_PAGE}`, + )?.all( + ...narrowed, + ...(after === undefined ? [] : [after.site, after.channel, after.day]), + ) ?? []) as unknown as Array<{ site: string; channel: string; day: string }>; + + for (const row of page) { + yield { kind: 'entry', key: { site: row.site, channelId: row.channel, day: row.day } }; + } + + this.#signal?.throwIfAborted(); + after = page.length === LIST_PAGE ? page.at(-1) : undefined; + } while (after !== undefined); + } + + // A channel's entries are all a channel has: state is a site's. A site's + // groups are a handful, and read at once. + if (kind !== 'entry' && channelId === undefined) { + const groups = (this.#prepared( + 'read', + `SELECT site, key FROM state${this.#where(site)}`, + )?.all(...narrowed) ?? []) as unknown as Array<{ site: string; key: string }>; + + for (const row of groups) { + yield { kind: 'state', site: row.site, key: row.key }; + } + } + } + + /** + * What narrows a listing, as SQL — its parameters given in the same order — + * and, for a page after the first, where the last one ended. + */ + #where(site: string | undefined, channelId?: string, after = false): string { + const said = [ + ...(site === undefined ? [] : ['site = ?']), + ...(channelId === undefined ? [] : ['channel = ?']), + ...(after ? ['(site, channel, day) > (?, ?, ?)'] : []), + ]; + + return said.length === 0 ? '' : ` WHERE ${said.join(' AND ')}`; } async prune(options: { before: string }): Promise { @@ -405,15 +607,19 @@ export class SqliteCacheDriver implements CacheDriver { // The whole prune in one statement the database plans itself: no directory // walk, nothing to do a few at a time, and nothing to stop half way — which - // is why this is the one method whose signal is only asked at the start. - const { changes } = this.#prepared('DELETE FROM entries WHERE day < ?').run(options.before); - - return Number(changes); + // is why this is the one method whose signal is only asked at the start. On + // what is there, as a delete is. + return Number( + this.#prepared('change', 'DELETE FROM entries WHERE day < ?')?.run(options.before).changes ?? + 0, + ); } async close(): Promise { + this.#closed = true; this.#statements.clear(); - this.#db.close(); + this.#db?.close(); + this.#db = undefined; } /** The same as {@link close}, for `await using`. */ diff --git a/src/cache/types.ts b/src/cache/types.ts index 73f6eed..788b3db 100644 --- a/src/cache/types.ts +++ b/src/cache/types.ts @@ -201,6 +201,56 @@ export interface FoundState { data: unknown; } +/** + * One thing a store holds, by key, as {@link CacheDriver.list} names it: a + * channel-day's entry, or a group of a site's state. + */ +export type CachedKey = + | { kind: 'entry'; key: ChannelDayKey } + | { kind: 'state'; site: string; key: string }; + +/** + * Why a run would not use something a look at a cache found — and so would + * remove it, for the next run to grab or fetch again: + * + * - `unreadable`: it says nothing about itself this version can read; + * - `incompatible`: it was written in a shape this version does not read, + * older or newer, its `schema` the shape it was written in; + * - `invalidated`: the manager's `invalidate` voids it — an entry only; + * - `corrupt`: what it says about itself is readable and what it holds is not + * — an entry whose codec throws `CorruptEntryError` on its programmes, as + * those of `CODECS` do for a count other than the one it was written with; + * `error`, what the codec threw. + * + * Each with `meta`, what it says about itself as the store had it — `M` being + * what it should say: an entry's meta, or a group's envelope. For one that is + * unreadable that is whatever was there, or nothing. + */ +export type Unusable = + | { status: 'unreadable'; meta: Partial | undefined } + | { status: 'incompatible'; meta: Partial & { schema: number } } + | { status: 'invalidated'; meta: M } + | { status: 'corrupt'; meta: M; error: Error }; + +/** + * What a look at a cache found — `CacheManager`'s `inspect` methods: what a run + * would use, with `status: 'readable'`, or why it would not. A run removes the + * second kind; a look leaves it where it is. + */ +export type Inspected = + | ({ status: 'readable' } & T) + | Unusable; + +/** What {@link CacheDriver.list} is asked for: everything, or less of it. */ +export interface CacheListOptions { + /** One site's keys only. */ + site?: string; + /** One channel's entries only — of {@link site}, where it is given. */ + channelId?: string; + /** Only entries, or only groups of state. */ + kind?: CachedKey['kind']; +} + /** * Where cached programmes actually live. * @@ -284,6 +334,27 @@ export interface CacheDriver { readState(site: string, key: string): Promise; writeState(site: string, key: string, data: unknown, meta: StoredStateMeta): Promise; deleteState(site: string, key: string): Promise; + /** + * Everything the store holds — each channel-day's entry and each group of a + * site's state — by key, and not what is in it: one of them is what + * {@link read} and {@link readState} are for. + * + * What `epg cache` looks inside a store with, since nothing else here could + * find out what is in one without being told the keys first. Keys and not + * metas, so a store that keeps an entry's meta inside the entry — a file + * each — need not open every one of them to answer. + * + * Narrowed by whatever of {@link CacheListOptions} is given, which a store + * kept by site and channel can answer without looking at the rest. A driver + * that cannot narrow by one may answer with more than was asked — a caller + * checks what it is handed — but never with less. In whatever order suits + * the store, and one key at a time, so a store too big to hold its own index + * in memory need not: a caller that wants them sorted sorts them. + * + * Required, as the state trio is: holding nothing is a legitimate answer and + * it is one line — `NoCacheDriver`'s yields nothing. + */ + list(options?: CacheListOptions): AsyncIterable; /** Let go of whatever is held open — a database handle, a connection. */ close?(): Promise; } @@ -370,4 +441,14 @@ export interface CacheManagerOptions { * and nothing further. */ invalidate?: (meta: StoredEntryMeta, key: ChannelDayKey) => boolean; + /** + * Where the manager says it removed an entry whose programmes could not be + * read — a line cut short, an edit — so that day is grabbed again by the + * next run, and is missing from anything built from the cache until then. + * + * Only that: an entry removed for a schema this version does not read, or by + * {@link invalidate}, is what an upgrade or a config asked for, and comes by + * the thousand. One whose programmes are broken is something gone wrong. + */ + warn?: (message: string, key: ChannelDayKey) => void; } diff --git a/src/cli/cache.ts b/src/cli/cache.ts new file mode 100644 index 0000000..b5bcd39 --- /dev/null +++ b/src/cli/cache.ts @@ -0,0 +1,742 @@ +/** + * `epg cache` — what is in the cache, and pruning it. + * + * Since a driver keeps an entry as its codec's bytes, a cache is no longer + * something to read with `cat` or `sqlite3`: a row holds a blob, and a file + * holds whatever the driver framed its meta with. So the command reads it + * through the `CacheManager`, as a run does — whatever the driver underneath — + * and says what it found. + * + * Looking changes nothing. A run removes an entry it cannot vouch for, so the + * day is grabbed again; this reports one and leaves it, and the next run + * removes it as it always would. + */ + +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import type { Writable } from 'node:stream'; +import { toRecord } from '../cache/codec.js'; +import { CACHE_SCHEMA, STATE_SCHEMA, type CacheManager } from '../cache/manager.js'; +import type { ChannelDayKey, Inspected, StoredEntryMeta, Unusable } from '../cache/types.js'; +import { errorMessage } from '../core/error.js'; +import { count } from '../core/count.js'; +import { inParallel } from '../core/parallel.js'; +import { writeLines } from '../core/streams.js'; +import { isRemembered } from '../grabber/remember.js'; +import { bagEntriesIn, channelListIn, StateKey } from '../grabber/state.js'; +import type { XmltvProgramme } from '../xmltv/types.js'; +import { writeXmltvStream } from '../xmltv/serialize.js'; +import type { ReportFormat } from './format.js'; + +/** The subcommands, in the order the usage lists them. */ +export const CACHE_COMMANDS = ['show', 'stats', 'state', 'prune'] as const; + +export type CacheCommand = (typeof CACHE_COMMANDS)[number]; + +/** What `show`, `stats` and `state` read a cache with. */ +export interface CacheReading { + /** The config's, so an entry its `invalidate` voids is reported as void. */ + cache: CacheManager; + format?: ReportFormat; + stdout: Writable; + stderr: Writable; + signal?: AbortSignal; +} + +/** + * How many channels' metas are read at once. A store that answers a channel's + * window in one question is asked it a few channels at a time, and one that + * reads a key at a time — a file each — a few files at a time. + */ +const META_CONCURRENCY = 8; + +/** + * Why a run would not use what a look found, as a person reads it — an + * entry's meta, or a group's envelope, whose schema this version reads. + */ +function why(found: Unusable, what: 'meta' | 'envelope'): string { + switch (found.status) { + case 'unreadable': + return `its ${what} is not one this version can read`; + case 'incompatible': + return ( + `written as schema ${found.meta.schema}, and this version reads ` + + String(what === 'meta' ? CACHE_SCHEMA : STATE_SCHEMA) + ); + case 'invalidated': + return "the config's invalidate voids it"; + case 'corrupt': + return `its programmes cannot be read: ${errorMessage(found.error)}`; + } +} + +/** + * A site's entries, those of one channel and one day where asked, in order. A + * channel and a day name one key, which is asked about directly rather than + * found by listing the site. + */ +async function entriesOf( + cache: CacheManager, + site: string, + channelId?: string, + day?: string, +): Promise { + if (channelId !== undefined && day !== undefined) { + return [{ site, channelId, day }]; + } + + const keys: ChannelDayKey[] = []; + + for await (const held of cache.list({ + site, + kind: 'entry', + ...(channelId === undefined ? {} : { channelId }), + })) { + if (held.kind === 'entry') { + keys.push(held.key); + } + } + + return keys.sort((a, b) => a.channelId.localeCompare(b.channelId) || a.day.localeCompare(b.day)); +} + +/** + * `epg cache show [channel] [day]`: the programmes cached for a site, a + * channel of it or one day of that — as an XMLTV document, or a record per line + * with `--format json`. 1 where nothing is cached for what was asked, or + * nothing of it that can be read. + */ +export async function showCache( + reading: CacheReading, + site: string, + channelId?: string, + day?: string, + indent: string | number = 2, +): Promise { + const { cache, stdout, stderr } = reading; + const asked = [site, channelId, day].filter(Boolean).join(' '); + const keys = await entriesOf(cache, site, channelId, day); + let leftOut = 0; + + // An entry at a time, so a site's fortnight is never all in memory at once — + // and one this version cannot read is said and left out, as a run would + // leave it out of a guide. + async function* readable(): AsyncGenerator { + for (const key of keys) { + reading.signal?.throwIfAborted(); + + let found; + + try { + found = await cache.inspect(key); + } catch (error) { + // One entry the store could not open — a file this user may not read — + // is one to say so of, as one this version cannot read is. A look that + // was stopped is not. + reading.signal?.throwIfAborted(); + + leftOut++; + await writeLines( + stderr, + `${key.channelId} ${key.day}: left out — could not be read: ${errorMessage(error)}`, + ); + continue; + } + + // Not there, or gone since it was listed: a run in another process + // pruned it. + if (found === undefined) { + continue; + } + + if (found.status !== 'readable') { + leftOut++; + await writeLines(stderr, `${key.channelId} ${key.day}: left out — ${why(found, 'meta')}`); + continue; + } + + yield found.programmes; + } + } + + const entries = readable(); + const first = await entries.next(); + + // Said, and 1, before a document is begun: nothing cached for what was + // asked, or nothing of it that can be read, is not an empty guide. + if (first.done === true) { + await writeLines( + stderr, + leftOut === 0 + ? `Nothing cached for ${asked}.` + : `Nothing cached for ${asked} that can be read.`, + ); + + return 1; + } + + // The first entry read, and then the rest as they are. + async function* all(): AsyncGenerator { + yield first.value; + yield* entries; + } + + if (reading.format === 'json') { + for await (const list of all()) { + // An entry's records in one write: it is the write a pipe is waited on. + await writeLines(stdout, ...list.map((programme) => JSON.stringify(toRecord(programme)))); + } + + return 0; + } + + async function* programmes(): AsyncGenerator { + for await (const list of all()) { + yield* list; + } + } + + await pipeline( + Readable.from(writeXmltvStream({ channels: [], programmes: programmes() }, { indent })), + stdout, + { end: false, ...(reading.signal ? { signal: reading.signal } : {}) }, + ); + + return 0; +} + +/** What one site has cached, as `epg cache stats` reports it. */ +export interface SiteCacheStats { + site: string; + channels: number; + /** Channel-days held, whether or not this version can read them. */ + entries: number; + /** + * Of those, the ones this version cannot read — written in a shape it does + * not read, or with a meta nobody can — left in place here. A run removes + * one when it next asks about its day. + */ + unreadable: number; + /** Of those, the ones the config's `invalidate` voids, as a run would. */ + invalidated: number; + /** + * Of those, the ones the store could not be asked about — a file this user + * may not read — each said on stderr. + */ + failed: number; + /** Of the readable ones, those that came back with no programmes. */ + empty: number; + programmes: number; + /** The first and last day held. */ + days?: { first: string; last: string }; + /** When the readable entries were grabbed, earliest and latest. */ + grabbed?: { first: string; last: string }; + /** The groups of state the site keeps. */ + state: string[]; +} + +export interface CacheStats { + sites: SiteCacheStats[]; + totals: Omit & { sites: number }; +} + +/** An entry the store could not be asked about, and what it threw. */ +interface Failed { + status: 'failed'; + error: unknown; +} + +/** + * What a channel's entries say about themselves, as {@link + * CacheManager.inspectMetas} answers — and, where the store could not be asked + * about one, what it threw, so one entry this user may not read is not taken + * for its channel. + * + * A store that answers a channel's window in one question is asked it, and a + * key at a time once that has failed. One that is asked a key at a time anyway + * is asked so from the start: the same reads, and none of them twice. + */ +async function metasOf( + reading: CacheReading, + keys: ChannelDayKey[], +): Promise | Failed | undefined>> { + if (reading.cache.driver.readMetas !== undefined) { + try { + return await reading.cache.inspectMetas(keys); + } catch (error) { + reading.signal?.throwIfAborted(); + + if (keys.length === 1) { + return [{ status: 'failed', error }]; + } + } + } + + const found: Array | Failed | undefined> = []; + + for (const key of keys) { + try { + found.push((await reading.cache.inspectMetas([key]))[0]); + } catch (error) { + reading.signal?.throwIfAborted(); + + found.push({ status: 'failed', error }); + } + } + + return found; +} + +/** + * How much a cache holds, per site — or for one site — from what each entry + * says about itself: no programmes are read. + */ +export async function cacheStats(reading: CacheReading, only?: string): Promise { + const { cache, stderr } = reading; + /** Each site's counts so far — and no keys: a cache is read as it is listed. */ + const sites = new Map< + string, + { + counted: SiteCacheStats; + channels: Set; + /** When its readable entries were grabbed, as instants: an ISO time may carry any offset. */ + grabbed?: { first: number; last: number }; + } + >(); + + const of = (site: string) => { + let counting = sites.get(site); + + if (counting === undefined) { + counting = { + counted: { + site, + channels: 0, + entries: 0, + unreadable: 0, + invalidated: 0, + failed: 0, + empty: 0, + programmes: 0, + state: [], + }, + channels: new Set(), + }; + sites.set(site, counting); + } + + return counting; + }; + + // A channel's days together, as a run asks about its window. A store lists + // a channel's keys one after another — sqlite in order, a file store a + // directory at a time — and one that does not has a channel asked about more + // than once, which counts the same. + async function* channelsListed(): AsyncGenerator { + let keys: ChannelDayKey[] = []; + + for await (const held of cache.list(only === undefined ? {} : { site: only })) { + if (held.kind === 'state') { + of(held.site).counted.state.push(held.key); + continue; + } + + if ( + keys.length > 0 && + (keys[0]!.site !== held.key.site || keys[0]!.channelId !== held.key.channelId) + ) { + yield keys; + keys = []; + } + + keys.push(held.key); + } + + if (keys.length > 0) { + yield keys; + } + } + + // Every site's channels in the one pool, so a cache of many small sites is + // read as many at a time as one large one. + await inParallel(channelsListed(), META_CONCURRENCY, async (keys) => { + reading.signal?.throwIfAborted(); + + const { site, channelId } = keys[0]!; + const counting = of(site); + const { counted } = counting; + + counting.channels.add(channelId); + + const found = await metasOf(reading, keys); + + for (const [index, key] of keys.entries()) { + const entry = found[index]; + + // Gone since it was listed: a run in another process pruned it. + if (entry === undefined) { + continue; + } + + counted.entries++; + counted.days ??= { first: key.day, last: key.day }; + + if (key.day < counted.days.first) { + counted.days.first = key.day; + } else if (key.day > counted.days.last) { + counted.days.last = key.day; + } + + if (entry.status === 'failed') { + counted.failed++; + await writeLines( + stderr, + `${site} ${key.channelId} ${key.day}: could not be read: ${errorMessage(entry.error)}`, + ); + continue; + } + + if (entry.status === 'invalidated') { + counted.invalidated++; + continue; + } + + if (entry.status !== 'readable') { + counted.unreadable++; + continue; + } + + const { meta } = entry; + + counted.programmes += meta.programmeCount; + counted.empty += meta.programmeCount === 0 ? 1 : 0; + const at = Date.parse(meta.grabbedAt); + + if (!Number.isNaN(at)) { + counting.grabbed = { + first: Math.min(at, counting.grabbed?.first ?? at), + last: Math.max(at, counting.grabbed?.last ?? at), + }; + } + } + }); + + const stats = [...sites.values()] + .map(({ counted, channels, grabbed }): SiteCacheStats => ({ + ...counted, + channels: channels.size, + state: counted.state.toSorted(), + ...(grabbed === undefined + ? {} + : { + grabbed: { + first: new Date(grabbed.first).toISOString(), + last: new Date(grabbed.last).toISOString(), + }, + }), + })) + .sort((a, b) => (a.site < b.site ? -1 : a.site > b.site ? 1 : 0)); + + const sum = (field: keyof CacheStats['totals']): number => + stats.reduce((total, site) => total + (field === 'sites' ? 1 : site[field]), 0); + + return { + sites: stats, + totals: { + sites: stats.length, + channels: sum('channels'), + entries: sum('entries'), + unreadable: sum('unreadable'), + invalidated: sum('invalidated'), + failed: sum('failed'), + empty: sum('empty'), + programmes: sum('programmes'), + }, + }; +} + +/** An ISO instant to the minute, which is all a person looking at a cache wants of one. */ +function minute(iso: string): string { + const at = Date.parse(iso); + + return Number.isNaN(at) ? iso : `${new Date(at).toISOString().slice(0, 16).replace('T', ' ')}Z`; +} + +/** `epg cache stats [site]`: the report, as text or JSON. */ +export async function writeCacheStats(reading: CacheReading, only?: string): Promise { + const stats = await cacheStats(reading, only); + + if (reading.format === 'json') { + await writeLines(reading.stdout, JSON.stringify(stats, undefined, 2)); + + return 0; + } + + if (stats.sites.length === 0) { + await writeLines( + reading.stdout, + only === undefined ? 'Nothing cached.' : `Nothing cached for ${only}.`, + ); + + return 0; + } + + const lines: string[] = []; + + for (const site of stats.sites) { + lines.push( + `${site.site} — ${count(site.channels, 'channel')}, ${count(site.entries, 'channel-day')}` + + (site.days === undefined + ? '' + : site.days.first === site.days.last + ? ` on ${site.days.first}` + : ` from ${site.days.first} to ${site.days.last}`) + + `, ${count(site.programmes, 'programme')}`, + ); + + const details = [ + ...(site.grabbed === undefined + ? [] + : minute(site.grabbed.first) === minute(site.grabbed.last) + ? [`grabbed ${minute(site.grabbed.first)}`] + : [`grabbed ${minute(site.grabbed.first)} to ${minute(site.grabbed.last)}`]), + ...(site.empty > 0 ? [`${count(site.empty, 'empty day')}`] : []), + ...(site.unreadable > 0 ? [`${site.unreadable} unreadable`] : []), + ...(site.invalidated > 0 ? [`${site.invalidated} voided by invalidate`] : []), + ...(site.failed > 0 ? [`${site.failed} could not be read`] : []), + ]; + + if (details.length > 0) { + lines.push(` ${details.join(' · ')}`); + } + + if (site.state.length > 0) { + lines.push(` remembers: ${site.state.join(', ')}`); + } + } + + const { totals } = stats; + + if (totals.sites > 1) { + lines.push( + '', + `${count(totals.sites, 'site')}, ${count(totals.channels, 'channel')}, ` + + `${count(totals.entries, 'channel-day')}, ${count(totals.programmes, 'programme')}`, + ); + } + + await writeLines(reading.stdout, ...lines); + + return 0; +} + +/** + * How much a group holds, as a run would read it: a channel list's channels, + * all or none, and a bag's keys — every other group is a bag — besides what of + * it a run cannot read, and what a site's `remember` kept that has expired. + * + * No more than that: a run drops other keys by rules of its own, as it drops + * the validators of days before its window, and a look has no window. + */ +function itemsIn(group: string, data: unknown, now: number): string { + if (group === StateKey.CHANNELS) { + const list = channelListIn(data); + + return list === undefined ? 'no channel list a run can read' : count(list.length, 'channel'); + } + + if (!Array.isArray(data)) { + return 'no keys a run can read'; + } + + const pairs = bagEntriesIn(data).length; + const kept = keptIn(group, data, now).length; + const besides = [ + ...(pairs > kept ? [`${pairs - kept} expired`] : []), + ...(data.length > pairs ? [`${data.length - pairs} unreadable`] : []), + ]; + + return count(kept, 'key') + (besides.length > 0 ? `, and ${besides.join(' and ')}` : ''); +} + +/** + * The pairs of a bag a run keeps: those with a key — and, of what a site's + * `remember` kept, those whose `until` is still ahead. + */ +function keptIn(group: string, data: unknown, now: number): Array<[string, unknown]> { + const entries = bagEntriesIn(data); + + return group === StateKey.REMEMBERED + ? entries.filter(([, value]) => isRemembered(value, now)) + : entries; +} + +/** + * The keys of a group, as a run would read it: a channel list's channel ids, + * or a bag's keys. Nothing for a group a run reads none from. + */ +function keysOf(group: string, data: unknown, now: number): string[] | undefined { + if (group === StateKey.CHANNELS) { + return channelListIn(data)?.map((channel) => channel.xmltvId); + } + + return Array.isArray(data) ? keptIn(group, data, now).map(([key]) => key) : undefined; +} + +/** + * `epg cache state [group] [--keys]`: the groups a site remembers, or + * what one of them holds — or only its keys. 1 where there is nothing of the + * kind, or nothing this version can read. + */ +export async function showState( + reading: CacheReading, + site: string, + group?: string, + keysOnly = false, +): Promise { + const { cache, stdout, stderr } = reading; + + if (group === undefined) { + const groups: string[] = []; + + // Only the groups: the site's entries are not looked at. + for await (const held of cache.list({ site, kind: 'state' })) { + if (held.kind === 'state') { + groups.push(held.key); + } + } + + if (groups.length === 0) { + await writeLines(stderr, `Nothing remembered for ${site}.`); + + return 1; + } + + const now = Date.now(); + const described = []; + + for (const key of groups.sort()) { + let found; + + try { + found = await cache.inspectState(site, key); + } catch (error) { + // One group the store could not open is one to describe, as one this + // version cannot read is. + reading.signal?.throwIfAborted(); + + described.push({ + group: key, + status: 'failed' as const, + why: `could not be read: ${errorMessage(error)}`, + }); + continue; + } + + // Gone since it was listed. + if (found === undefined) { + continue; + } + + described.push( + found.status === 'readable' + ? { + group: key, + ...found.meta, + status: found.status, + holds: itemsIn(key, found.data, now), + } + : { group: key, status: found.status, why: why(found, 'envelope') }, + ); + } + + // Every group gone since it was listed: a run in another process removed + // them, and there is nothing to describe. + if (described.length === 0) { + await writeLines(stderr, `Nothing remembered for ${site}.`); + + return 1; + } + + // 1 where nothing of it can be read, as `show` answers — though what could + // not be is still described. + const code = described.some((one) => one.status === 'readable') ? 0 : 1; + + if (reading.format === 'json') { + await writeLines(stdout, JSON.stringify(described, undefined, 2)); + + return code; + } + + const width = Math.max(...described.map((one) => one.group.length)); + + await writeLines( + stdout, + ...described.map( + (one) => + `${one.group.padEnd(width)} ` + + (one.status === 'readable' + ? `${one.holds}, written ${minute(one.writtenAt)} by ${one.writtenBy}` + : one.status === 'failed' + ? one.why + : `${one.why} — fetched again when next wanted`), + ), + ); + + return code; + } + + const inspected = await cache.inspectState(site, group); + + if (inspected === undefined) { + await writeLines(stderr, `Nothing remembered for ${site} under ${group}.`); + + return 1; + } + + if (inspected.status !== 'readable') { + await writeLines( + stderr, + `${site} ${group}: ${why(inspected, 'envelope')} — fetched again when next wanted`, + ); + + return 1; + } + + const { data, meta } = inspected; + + if (keysOnly) { + const keys = keysOf(group, data, Date.now()); + + if (keys === undefined) { + await writeLines(stderr, `${site} ${group} holds no keys to list.`); + + return 1; + } + + // One write, of however many: a group can hold more keys than a call can + // take arguments. + if (reading.format === 'json') { + await writeLines(stdout, JSON.stringify(keys)); + } else if (keys.length > 0) { + await writeLines(stdout, keys.join('\n')); + } + + return 0; + } + + // The data alone as text, for a pipe into `jq`; with what the store vouches + // for it as JSON. + // `null` for a group written empty, which a store keeps as no data at all: + // `undefined` is not JSON, and the text would be read as such. + await writeLines( + stdout, + JSON.stringify( + reading.format === 'json' ? { meta, data: data ?? null } : (data ?? null), + undefined, + 2, + ), + ); + + return 0; +} diff --git a/src/cli/plan.ts b/src/cli/plan.ts index b7c6c06..738fced 100644 --- a/src/cli/plan.ts +++ b/src/cli/plan.ts @@ -35,6 +35,7 @@ import { DEFAULT_DAYS, resolveSite } from '../grabber/site.js'; import { SiteStateHandle } from '../grabber/state.js'; import type { AnySiteConfig, GrabberChannel } from '../grabber/types.js'; import type { ReportFormat } from './format.js'; +import { count } from '../core/count.js'; /** Where a site's channel list came from, which is what it cost to know it. */ export type ChannelsFrom = 'config' | 'cache' | 'fetched' | 'failed'; @@ -289,11 +290,6 @@ export async function planRun( }; } -/** `1 day`, `3 days` — because a report that says "1 channels" reads as a bug. */ -function count(n: number, one: string, many = `${one}s`): string { - return `${n} ${n === 1 ? one : many}`; -} - /** What each provenance costs, said once rather than per row. */ const FROM: Record = { config: 'in the config', diff --git a/src/cli/run.ts b/src/cli/run.ts index 628a3ac..6abb160 100644 --- a/src/cli/run.ts +++ b/src/cli/run.ts @@ -26,7 +26,14 @@ import type { MissingAllowance } from '../grabber/main.js'; import { GrabberError } from '../core/error.js'; import { OptionError, parseOptions } from '../core/options.js'; import { dayIn, dayToDate, resolveTimeZone, toDayString } from '../core/days.js'; -import { reportingTo, silent, stamped, LEVELS, type EventLevel } from '../core/events.js'; +import { + reportingTo, + silent, + stamped, + LEVELS, + type Emit, + type EventLevel, +} from '../core/events.js'; import { FAILURE_MODES, reporterFor, REPORTER_NAMES } from '../core/reporters.js'; import { drain, writeFlushed, writeLines } from '../core/streams.js'; import { initGrabber } from './scaffold.js'; @@ -37,6 +44,8 @@ import { outputOptions } from '../xmltv/serialize.js'; import { renderReport, validateFile } from './validate.js'; import type { CompressionFormat } from '../core/output.js'; import type { GrabSummary } from '../grabber/types.js'; +import type { CacheCommand } from './cache.js'; +import { count } from '../core/count.js'; export const USAGE = `Usage: epg [options] @@ -49,7 +58,7 @@ Commands: channels Report which wanted channels will get no guide, and why serve Hold the merged guide behind HTTP for a consumer that polls try Put one site, channel and day through, showing every step - prune Remove cached days older than a given day + cache Look inside the cache — show, stats, state — or prune it init-grabber Write a tv_grab_* executable for the config, next to it Options: @@ -75,8 +84,8 @@ Options: extensions, comma-separated (e.g. lcn,uniqueID) --no-extensions build/merge/serve/filter: leave every provider extension out, for a guide that validates against the DTD - --indent build/merge/serve/filter: pretty-print with this - indentation + --indent build/merge/serve/filter/cache show: pretty-print with + this indentation --profile Shape the guide for one consumer: tvheadend or jellyfin. build/merge/serve/filter. Changes the guide, never the cache, so switching it refetches nothing @@ -92,7 +101,6 @@ Options: into --against, in place — a *.channels.xml, a playlist or a guide. With -o, writes there and leaves the original alone - --before prune only: remove days before YYYY-MM-DD (default: today) --log-level How much to report: error, warn, info (default) or debug -v, --verbose Same as --log-level debug -q, --quiet Same as --log-level error. Beats --verbose if both are given @@ -119,6 +127,17 @@ validate options: --format text (default) or json --strict Count warnings as failures too +cache commands: + epg cache show [channel] [day] What is cached, as an XMLTV document, + or a programme per line with --format json + epg cache stats [site] How much is cached, per site + epg cache state [group] What a site remembers between runs: + its groups, or what one holds + epg cache prune Remove cached days older than today + --format show, stats, state: text (default) or json + --keys state with a group: only the keys it holds + --before prune: remove days before YYYY-MM-DD (default: today) + init-grabber options: --description What --description prints (default: the country and name) --grabber-version The grabber's own version, as its --version @@ -207,11 +226,6 @@ const CONFIG_CANDIDATES = ['epg.config.ts', 'epg.config.js', 'epg.config.mjs']; /** The commands `--channels` narrows. The rest are told so rather than ignoring it. */ const SELECTABLE = ['build', 'grab', 'merge', 'serve']; -/** `2 channels`, `1 channel` — a count is read, so it should read. */ -function plural(count: number, noun: string): string { - return `${count} ${noun}${count === 1 ? '' : 's'}`; -} - /** `--threads`: bare for one per core, or how many at most. */ function poolSize(raw: string, flag: string): true | number { if (raw === 'true') { @@ -247,7 +261,7 @@ const COMMANDS = [ 'validate', 'channels', 'filter', - 'prune', + 'cache', 'init-grabber', ]; @@ -406,6 +420,132 @@ async function validateGuide( return report.ok ? 0 : EXIT_FAILED; } +/** + * The flags each `epg cache` subcommand acts on, beyond {@link EVERY_CACHE_COMMAND}. + * `prune`'s default day is today, which the `--timezone` says. + */ +const CACHE_FLAGS: Record> = { + show: new Set(['format', 'indent']), + stats: new Set(['format']), + state: new Set(['format', 'keys']), + prune: new Set(['before', 'timezone']), +}; + +/** What every `epg cache` subcommand takes: where the config and cache are, and how much to say. */ +const EVERY_CACHE_COMMAND: ReadonlySet = new Set([ + 'config', + 'cache-dir', + 'cache-driver', + 'log-level', + 'quiet', + 'verbose', + 'reporter', + 'failures', +]); + +/** + * `epg cache `: looking inside the cache, and pruning + * it. The three that look are loaded only when asked for, and change nothing. + */ +async function cacheCommand( + config: ResolvedConfig, + [subcommand, ...args]: string[], + values: Record & { + format?: ReportFormat; + keys?: boolean; + before?: string; + indent?: string | number; + }, + emit: Emit, + stdout: Writable, + stderr: Writable, + signal: AbortSignal | undefined, +): Promise { + const { CACHE_COMMANDS, showCache, showState, writeCacheStats } = await import('./cache.js'); + + if (!CACHE_COMMANDS.includes(subcommand as CacheCommand)) { + throw new UsageError( + subcommand === undefined + ? `cache needs one of ${CACHE_COMMANDS.join(', ')}` + : `Unknown cache command: ${subcommand}`, + ); + } + + const command = subcommand as CacheCommand; + const takes = { show: 3, stats: 1, state: 2, prune: 0 }[command]; + const [site, second, third] = args; + + // Everything a command line can get wrong, before the cache is opened: a + // mistake is said with the usage, and a store is not opened for nothing. + if (args.length > takes) { + throw new UsageError(`cache ${command} takes at most ${count(takes, 'argument')}`); + } + + if ((command === 'show' || command === 'state') && site === undefined) { + throw new UsageError( + command === 'show' + ? 'cache show needs a site, e.g. cache show example.tv one.example.tv' + : 'cache state needs a site, e.g. cache state example.tv', + ); + } + + if (values.keys && (command !== 'state' || second === undefined)) { + throw new UsageError( + '--keys is for cache state with a group, e.g. cache state example.tv state', + ); + } + + // A flag a subcommand would ignore is refused rather than ignored: `cache + // show --before …` would otherwise show every day with nothing said, and + // `cache stats --days 3` the whole cache. + for (const [flag, value] of Object.entries(values)) { + if (value !== undefined && !EVERY_CACHE_COMMAND.has(flag) && !CACHE_FLAGS[command].has(flag)) { + throw new UsageError(`--${flag} is not for cache ${command}`); + } + } + + // XMLTV is what is indented; a record per line has nothing to indent. + if (values.indent !== undefined && values.format === 'json') { + throw new UsageError('--indent is for cache show as XMLTV, not --format json'); + } + + const day = command === 'show' && third !== undefined ? dayString(third, 'day') : undefined; + + await using cache = await createCacheStore(config, signal); + const reading = { + // The manager the config makes, which carries its `invalidate`: an entry a + // run would void is reported as void. + cache, + ...(values.format ? { format: values.format } : {}), + stdout, + stderr, + ...(signal ? { signal } : {}), + }; + + // Awaited before returning, each: the cache is closed on the way out of this + // block, and a look handed back still running would read a closed one. + switch (command) { + case 'show': + return await showCache(reading, site!, second, day, values.indent); + case 'stats': + return await writeCacheStats(reading, site); + case 'state': + return await showState(reading, site!, second, values.keys === true); + case 'prune': { + // Today, or the day before it for a site keeping a calendar whose window + // starts there — which the guide still reads once it has been grabbed. + const before = + values.before ?? + earliestDay(config.sites, dayIn(new Date(), config.timeZone), config.timeZone); + const removed = await cache.prune({ before }); + + emit({ type: 'prune:done', removed, before }); + + return 0; + } + } +} + /** `epg filter ` — write the guide again with only some of its channels. */ async function filterCommand( file: string | undefined, @@ -472,7 +612,7 @@ async function filterCommand( `${report.missing.length} of ${channels?.size ?? 0} channels are not in ${file}: ${report.missing.join(', ')}`, ] : []), - `${plural(report.kept, 'channel')}, ${plural(report.programmes, 'programme')}`, + `${count(report.kept, 'channel')}, ${count(report.programmes, 'programme')}`, ); return 0; @@ -586,6 +726,8 @@ async function execute( // not something a command line can do — the same limit as `extensions`. profile: { type: 'string', choices: OUTPUT_PROFILE_NAMES }, before: { type: 'string', transform: dayString }, + // `epg cache state` only: a group's keys rather than what it holds. + keys: { type: 'boolean' }, quiet: { type: 'boolean', short: 'q' }, verbose: { type: 'boolean', short: 'v' }, 'log-level': { type: 'string', choices: LEVELS }, @@ -612,6 +754,12 @@ async function execute( const command = positionals[0] ?? 'build'; + // Moved, and said where to, so a job that ran it learns more than that it + // is gone. + if (command === 'prune') { + throw new UsageError('Unknown command: prune — it is epg cache prune now'); + } + if (!COMMANDS.includes(command)) { throw new UsageError(`Unknown command: ${command}`); } @@ -918,20 +1066,8 @@ async function execute( ...(signal ? { signal } : {}), }); } - case 'prune': { - // Today, or the day before it for a site keeping a calendar whose window - // starts there — which the guide still reads once it has been grabbed. - const before = - values.before ?? - earliestDay(config.sites, dayIn(new Date(), config.timeZone), config.timeZone); - - await using cache = await createCacheStore(config, signal); - const removed = await cache.prune({ before }); - - emit({ type: 'prune:done', removed, before }); - - return 0; - } + case 'cache': + return cacheCommand(config, positionals.slice(1), values, emit, stdout, stderr, signal); default: { // Unreachable: `COMMANDS` is checked above, so a command with no case // here fails to compile rather than silently pruning the cache — which diff --git a/src/cli/try.ts b/src/cli/try.ts index 6659e57..20d77f3 100644 --- a/src/cli/try.ts +++ b/src/cli/try.ts @@ -42,6 +42,7 @@ import type { AnySiteConfig, GrabberChannel, ParsedProgramme } from '../grabber/ import { ProgrammeBuilder } from '../xmltv/builder.js'; import { serializeProgramme } from '../xmltv/serialize.js'; import type { XmltvProgramme } from '../xmltv/types.js'; +import { count } from '../core/count.js'; /** How much of a payload is shown when nothing says otherwise. */ const PAYLOAD_PREVIEW = 2000; @@ -477,9 +478,7 @@ export async function tryChannelDay( out.push(''); } - out.push( - ` ${programmes.length} programme${programmes.length === 1 ? '' : 's'} in ${Date.now() - began}ms`, - ); + out.push(` ${count(programmes.length, 'programme')} in ${Date.now() - began}ms`); for (const programme of programmes) { // Deliberately unshaped: no `outputOptions`, no profile, whatever the diff --git a/src/core/count.ts b/src/core/count.ts new file mode 100644 index 0000000..5b5e949 --- /dev/null +++ b/src/core/count.ts @@ -0,0 +1,7 @@ +/** + * A number and what it counts, in the plural where it is not one: `1 channel`, + * `3 channel-days`, and `many` for a noun whose plural is not an `s` away. + */ +export function count(n: number, one: string, many = `${one}s`): string { + return `${n} ${n === 1 ? one : many}`; +} diff --git a/src/core/parallel.ts b/src/core/parallel.ts new file mode 100644 index 0000000..3bc320d --- /dev/null +++ b/src/core/parallel.ts @@ -0,0 +1,64 @@ +/** + * Run `work` over `items`, `limit` of them at a time. + * + * For thousands of small operations that only wait — on the disk, mostly: + * doing them one after another spends the whole time waiting, while doing them + * all at once is the file descriptor storm the rest of this package is careful + * to avoid. Node's own file operations run on a threadpool of four by default, + * so there is little to gain past a handful in flight. + * + * The items may come as they are made — a listing of a store, read a page at a + * time — and are taken as a worker is free for one, so no more of them are held + * than are being worked on. + * + * The first failure is the answer, and nothing more is started after it: the + * rest would only go on in the background, for a caller that has stopped + * waiting on them. Items still to come are not asked for, and their source is + * told so. + */ +export async function inParallel( + items: Iterable | AsyncIterable, + limit: number, + work: (item: T) => Promise, +): Promise { + // A limit of none would start no worker and resolve having run nothing, + // which a caller would take for having run everything. + if (!Number.isInteger(limit) || limit < 1) { + throw new RangeError(`inParallel needs a limit of 1 or more (got ${limit})`); + } + + const iterator = + Symbol.asyncIterator in items ? items[Symbol.asyncIterator]() : items[Symbol.iterator](); + let failed = false; + + const worker = async (): Promise => { + try { + while (!failed) { + const next = await iterator.next(); + + if (next.done === true || failed) { + return; + } + + await work(next.value); + } + } catch (error) { + failed = true; + throw error; + } + }; + + // No more workers than there are items, where that is known. + const workers = Array.isArray(items) ? Math.min(limit, items.length) : limit; + + try { + await Promise.all(Array.from({ length: workers }, worker)); + } catch (error) { + // The failure is the answer, whatever closing the source says. + try { + await iterator.return?.(); + } catch {} + + throw error; + } +} diff --git a/src/core/reporters.ts b/src/core/reporters.ts index f034099..3cafbc3 100644 --- a/src/core/reporters.ts +++ b/src/core/reporters.ts @@ -27,6 +27,7 @@ import { } from './events.js'; import { errorChain, errorMessage, MAX_CAUSE_DEPTH } from './error.js'; import { queueLine } from './streams.js'; +import { count } from './count.js'; /** A reporter this package ships, by name — what a config can ask for. */ export const REPORTER_NAMES = ['text', 'json', 'progress'] as const; @@ -59,11 +60,6 @@ export interface ReporterRuntime { /** A reporter that wants those before it can be built. */ export type ReporterFactory = (runtime: ReporterRuntime) => Reporter; -/** `N thing` / `N things`, since almost every line here counts something. */ -function count(n: number, one: string, many = `${one}s`): string { - return `${n} ${n === 1 ? one : many}`; -} - /** * A duration, in the unit somebody reading a nightly run would use. * @@ -268,10 +264,10 @@ export function render(event: EpgEvent, prefix = true): string | undefined { return 'Stopped serving'; case 'prune:done': // One wording where there were two: a prune after a grab used to say - // "Pruned N cached day(s) older than X" and `epg prune` "Pruned N cached - // entr(y|ies) before X", for the same act on the same store. + // "Pruned N cached day(s) older than X" and the prune command "Pruned N + // cached entr(y|ies) before X", for the same act on the same store. return ( - `Pruned ${event.removed} cached entr${event.removed === 1 ? 'y' : 'ies'} ` + + `Pruned ${count(event.removed, 'cached entry', 'cached entries')} ` + `before ${event.before}` ); default: diff --git a/src/grabber/remember.ts b/src/grabber/remember.ts index f6ea35c..4e0087a 100644 --- a/src/grabber/remember.ts +++ b/src/grabber/remember.ts @@ -41,6 +41,18 @@ type Outcome = { value: unknown } | { error: unknown; refused?: true }; */ export type Away = (wait: () => Promise) => Promise; +/** + * Whether a value kept in the `remembered` group is still remembered at `now` + * — what a run keeps of the group, and what `epg cache state` counts. A pair + * that is not one is forgotten with the rest: whatever wrote it, it is no + * answer to anything. + */ +export function isRemembered(entry: unknown, now: number): boolean { + const until = (entry as Partial | null | undefined)?.until; + + return typeof until === 'number' && until > now; +} + /** One site's remembered values for the length of a run. */ export class Memory { readonly #handle: SiteStateHandle; @@ -191,9 +203,7 @@ export class Memory { this.#kept ??= this.#handle.bag(StateKey.REMEMBERED).then( (kept) => { for (const [key, entry] of kept) { - // A pair that is not one is forgotten with the rest: whatever wrote - // it, it is no answer to anything. - if (typeof entry?.until !== 'number' || !(entry.until > this.#now)) { + if (!isRemembered(entry, this.#now)) { kept.delete(key); } } diff --git a/src/grabber/site-run.ts b/src/grabber/site-run.ts index 1e63ca2..f27be4a 100644 --- a/src/grabber/site-run.ts +++ b/src/grabber/site-run.ts @@ -22,6 +22,7 @@ import type { KyInstance } from 'ky'; import PQueue from 'p-queue'; import { isStale } from '../cache/main.js'; import type { CacheStore } from '../cache/types.js'; +import { count } from '../core/count.js'; import type { Emit, GrabCounts } from '../core/events.js'; import { ProgrammeBuilder } from '../xmltv/builder.js'; import type { XmltvProgramme } from '../xmltv/types.js'; @@ -573,9 +574,29 @@ export class SiteRun { const { cache, emit, grabbedAt } = this.#run; const site = this.#site; const key = { site, channelId: channel.xmltvId, day }; - const mine = parsed - .map((entry) => ({ ...built(entry), channel: channel.xmltvId })) - .sort((a, b) => a.start.getTime() - b.start.getTime()); + const mine: XmltvProgramme[] = []; + let undated = 0; + + for (const entry of parsed) { + const programme = { ...built(entry), channel: channel.xmltvId }; + + // A programme with no time it starts at is no listing, and no codec + // writes one it can read back: kept, it would make the entry one that + // never reads, removed and grabbed again by every run. + if (!(programme.start instanceof Date) || Number.isNaN(programme.start.getTime())) { + undated++; + } else { + mine.push(programme); + } + } + + if (undated > 0) { + this.#says.warn( + `${channel.xmltvId} ${day}: ${count(undated, 'programme')} without a valid start left out`, + ); + } + + mine.sort((a, b) => a.start.getTime() - b.start.getTime()); if (!this.#written.has(id)) { this.#written.add(id); diff --git a/src/grabber/state.ts b/src/grabber/state.ts index beb9690..458c558 100644 --- a/src/grabber/state.ts +++ b/src/grabber/state.ts @@ -154,6 +154,46 @@ interface StateGroup { settle(): void; } +/** + * The channel list a {@link StateKey.CHANNELS} group holds, or nothing where + * it holds none a run can use — what a run reads, and what `epg cache state` + * reports. + */ +export function channelListIn(data: unknown): GrabberChannel[] | undefined { + // One shape: the envelope, whose first key is the list itself. + const list = + data === null || typeof data !== 'object' + ? undefined + : (data as { channels?: unknown }).channels; + + // All or nothing, unlike a bag's entries: half a channel list would mean + // grabbing half a site, which is worse than fetching the list again. + return Array.isArray(list) && + list.every( + (channel: unknown) => + typeof channel === 'object' && + channel !== null && + typeof (channel as GrabberChannel).xmltvId === 'string' && + typeof (channel as GrabberChannel).siteId === 'string', + ) + ? (list as GrabberChannel[]) + : undefined; +} + +/** + * The `[key, value]` pairs a bag holds that a run keeps — each with a key that + * is a string — and none of anything else: what a run reads, and what `epg + * cache state` reports. + */ +export function bagEntriesIn(data: unknown): Array<[string, unknown]> { + return Array.isArray(data) + ? data.filter( + (entry: unknown): entry is [string, unknown] => + Array.isArray(entry) && entry.length === 2 && typeof entry[0] === 'string', + ) + : []; +} + /** * The channel list a site fetched, and how old it is allowed to get. * @@ -188,34 +228,12 @@ export class ChannelsGroup implements StateGroup { */ static from(found: StateEntry | undefined): ChannelsGroup { return new ChannelsGroup( - ChannelsGroup.#listIn(found?.data), + channelListIn(found?.data), found?.meta.writtenAt, ChannelsGroup.#metadataIn(found?.data), ); } - /** The list an entry holds, or nothing where what it holds is not one. */ - static #listIn(data: unknown): GrabberChannel[] | undefined { - // One shape: the envelope, whose first key is the list itself. - const list = - data === null || typeof data !== 'object' - ? undefined - : (data as { channels?: unknown }).channels; - - // All or nothing, unlike a bag's entries: half a channel list would mean - // grabbing half a site, which is worse than fetching the list again. - return Array.isArray(list) && - list.every( - (channel: unknown) => - typeof channel === 'object' && - channel !== null && - typeof (channel as GrabberChannel).xmltvId === 'string' && - typeof (channel as GrabberChannel).siteId === 'string', - ) - ? (list as GrabberChannel[]) - : undefined; - } - /** What the site said about the list it stored, where it said anything. */ static #metadataIn(data: unknown): unknown { return data === null || typeof data !== 'object' @@ -320,15 +338,7 @@ class BagGroup implements StateGroup { * one unreadable value is better than forgetting the token beside it. */ static from(key: string, found: StateEntry | undefined): BagGroup { - return new BagGroup( - key, - Array.isArray(found?.data) - ? found.data.filter( - (entry: unknown): entry is [string, V] => - Array.isArray(entry) && entry.length === 2 && typeof entry[0] === 'string', - ) - : [], - ); + return new BagGroup(key, bagEntriesIn(found?.data) as Array<[string, V]>); } readonly key: string; diff --git a/src/main.ts b/src/main.ts index e3a9861..fcc3f6f 100644 --- a/src/main.ts +++ b/src/main.ts @@ -204,6 +204,7 @@ export { FsNdjsonCacheDriver, FsXmltvCacheDriver, CODECS, + CorruptEntryError, fromRecord, toRecord, isStale, diff --git a/src/serve/handler.ts b/src/serve/handler.ts index c6b5967..49792d0 100644 --- a/src/serve/handler.ts +++ b/src/serve/handler.ts @@ -980,7 +980,11 @@ export async function createGuideHandler( ].join(', '); const opened = options.cache === undefined; - const cache = options.cache ?? (await createCacheStore(config, options.signal)); + const cache = + options.cache ?? + (await createCacheStore(config, options.signal, { + warn: (message, key) => emit({ type: 'site:warning', site: key.site, message }), + })); /** Where a `derived` function says things, which is where a merge's do. */ const mergeSays: Says = { diff --git a/src/xmltv/scan.ts b/src/xmltv/scan.ts index 869f79e..068a418 100644 --- a/src/xmltv/scan.ts +++ b/src/xmltv/scan.ts @@ -124,6 +124,14 @@ const CREDIT_ROLES: ReadonlySet = new Set([ 'guest', ] satisfies CreditRole[]); +/** + * Input that is no XMLTV document at all — no root found, or a root other than + * `` — as against a fault of this code. A `TypeError` as it always was, and + * named as one; a class of its own so a reader that must tell the two apart — + * the cache, deciding whether an entry is damaged — can. + */ +export class XmltvInputError extends TypeError {} + /** * XMLTV scanner. Holds the scan state (`pos`, current tag scratch) as * instance fields; helpers write their "out" values there instead of @@ -306,7 +314,7 @@ export class XmltvScanner { */ #advance(buf: string, consumed: number, checkRootLimit = false): string { if (checkRootLimit && !this.#rootFound && buf.length - consumed > this.#rootScanLimit) { - throw new TypeError( + throw new XmltvInputError( `No root element found within the first ${this.#rootScanLimit} characters of input`, ); } @@ -2786,7 +2794,7 @@ export class XmltvScanner { // forward here (consumed === buf.length), so `checkRootLimit` on // #advance doesn't apply — check the whole scanned buffer directly. if (!this.#rootFound && buf.length > this.#rootScanLimit) { - throw new TypeError( + throw new XmltvInputError( `No root element found within the first ${this.#rootScanLimit} characters of input`, ); } @@ -2893,7 +2901,7 @@ export class XmltvScanner { } if (this.#tagName !== 'tv') { - throw new TypeError( + throw new XmltvInputError( `Invalid XMLTV document: expected root element , found <${this.#tagName}>`, ); } diff --git a/test/build.test.ts b/test/build.test.ts index a19e6de..ea3fa35 100644 --- a/test/build.test.ts +++ b/test/build.test.ts @@ -1,6 +1,6 @@ import { EventEmitter } from 'node:events'; import { existsSync } from 'node:fs'; -import { mkdtemp, readFile, readdir, stat } from 'node:fs/promises'; +import { mkdtemp, readFile, readdir, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; @@ -1768,3 +1768,62 @@ describe('an output profile', () => { expect(other).toContain('Movie / Drama'); }); }); + +describe('a cache that cannot be read or made', () => { + it('says, as the site, that a merge removed a day whose programmes would not read', async () => { + const dir = await tempDir(); + const warnings: string[] = []; + + await runGrab(config(dir), { now: NOW }); + + const file = join(dir, 'cache', 'example.com', 'one.example', `${TODAY}.ndjson`); + const lines = (await readFile(file, 'utf8')).trimEnd().split('\n'); + + // The programme line cut short, after a meta line that still reads. + await writeFile(file, `${lines[0]}\n${lines[1]!.slice(0, 10)}\n`); + + await runMerge(config(dir), { + now: NOW, + reporter: (event) => { + if (event.type === 'site:warning') { + warnings.push(`${event.site}: ${event.message}`); + } + }, + }); + + expect(warnings).toEqual([ + expect.stringMatching( + new RegExp( + `^example\\.com: one\\.example ${TODAY}: removed from the cache, its programmes unreadable \\(.+\\) — the day is grabbed again by the next run$`, + ), + ), + ]); + expect(existsSync(file)).toBe(false); + }); + + it("says a sqlite cache's place is wrong as a place, not as a runtime", async () => { + const dir = await tempDir(); + const notADirectory = join(dir, 'notes.txt'); + + await writeFile(notADirectory, ''); + + const sqlite = await import('../src/cache/sqlite-driver.js').catch(() => undefined); + + if (sqlite === undefined) { + return; + } + + await expect( + createCacheStore( + config(dir, { cache: { dir: join(notADirectory, 'cache'), driver: 'sqlite' } }), + ), + ).rejects.toThrow( + `The sqlite cache cannot be made at ${join(notADirectory, 'cache', 'cache.sqlite')}: ${notADirectory} is not a directory`, + ); + await expect( + createCacheStore( + config(dir, { cache: { dir: join(notADirectory, 'cache'), driver: 'sqlite' } }), + ), + ).rejects.not.toThrow('needs Node 24'); + }); +}); diff --git a/test/cache-command.test.ts b/test/cache-command.test.ts new file mode 100644 index 0000000..e9628a9 --- /dev/null +++ b/test/cache-command.test.ts @@ -0,0 +1,923 @@ +import { Writable } from 'node:stream'; +import { describe, expect, it } from 'vitest'; +import { CacheManager, MemoryCacheDriver, STATE_SCHEMA } from '../src/cache/main.js'; +import type { + CacheManagerOptions, + ChannelDayKey, + StoredEntryMeta, + StoredStateMeta, +} from '../src/cache/main.js'; +import { cacheStats, showCache, showState, writeCacheStats } from '../src/cli/cache.js'; +import type { CacheReading } from '../src/cli/cache.js'; +import type { XmltvProgramme } from '../src/xmltv/types.js'; + +/** Collects everything written, so stdout and stderr can be asserted on. */ +class Sink extends Writable { + private chunks: string[] = []; + + override _write(chunk: Buffer | string, _encoding: string, done: () => void): void { + this.chunks.push(String(chunk)); + done(); + } + + get text(): string { + return this.chunks.join(''); + } +} + +function programme(channel: string, title: string): XmltvProgramme { + return { channel, start: new Date('2026-07-17T18:00:00.000Z'), title: [{ value: title }] }; +} + +/** A cache with a little of everything in it, and how to read it. */ +async function cached(invalidate?: CacheManagerOptions['invalidate']): Promise<{ + driver: MemoryCacheDriver; + reading: CacheReading & { stdout: Sink; stderr: Sink }; +}> { + const driver = new MemoryCacheDriver(); + const cache = new CacheManager({ driver, ...(invalidate ? { invalidate } : {}) }); + + await cache.write( + { site: 'a.tv', channelId: 'two', day: '2026-07-18' }, + [programme('two', 'Late')], + { + grabbedAt: '2026-07-18T03:00:00.000Z', + }, + ); + await cache.write( + { site: 'a.tv', channelId: 'one', day: '2026-07-17' }, + [programme('one', 'News'), programme('one', 'Film')], + { grabbedAt: '2026-07-17T03:00:00.000Z' }, + ); + await cache.write({ site: 'a.tv', channelId: 'one', day: '2026-07-19' }, [], { + grabbedAt: '2026-07-19T03:00:00.000Z', + }); + await cache.write( + { site: 'b.tv', channelId: 'one', day: '2026-07-17' }, + [programme('one', 'B')], + { + grabbedAt: '2026-07-17T04:00:00.000Z', + }, + ); + await cache.setState('a.tv', 'channels', { + channels: [ + { xmltvId: 'one', siteId: '1' }, + { xmltvId: 'two', siteId: '2' }, + ], + }); + await cache.setState('a.tv', 'state', [ + ['cursor', 42], + ['token', 'abc'], + ]); + + return { + driver, + reading: { + cache, + stdout: new Sink(), + stderr: new Sink(), + }, + }; +} + +/** Spoil an entry the way an older version, or an editor, would have left it. */ +async function spoil( + driver: MemoryCacheDriver, + meta: Record, + key = { site: 'a.tv', channelId: 'two', day: '2026-07-18' }, +): Promise { + const found = (await driver.read(key))!; + + await driver.write(key, found.body, { ...found.meta, ...meta } as StoredEntryMeta); +} + +describe('epg cache stats', () => { + it('counts each site from what its entries say about themselves', async () => { + const { reading } = await cached(); + + expect(await cacheStats(reading)).toEqual({ + sites: [ + { + site: 'a.tv', + channels: 2, + entries: 3, + unreadable: 0, + invalidated: 0, + failed: 0, + empty: 1, + programmes: 3, + days: { first: '2026-07-17', last: '2026-07-19' }, + grabbed: { first: '2026-07-17T03:00:00.000Z', last: '2026-07-19T03:00:00.000Z' }, + state: ['channels', 'state'], + }, + expect.objectContaining({ + site: 'b.tv', + channels: 1, + entries: 1, + programmes: 1, + state: [], + }), + ], + totals: { + sites: 2, + channels: 3, + entries: 4, + unreadable: 0, + invalidated: 0, + failed: 0, + empty: 1, + programmes: 4, + }, + }); + expect((await cacheStats(reading, 'b.tv')).sites.map((site) => site.site)).toEqual(['b.tv']); + }); + + it('counts an entry it cannot vouch for apart, and leaves it', async () => { + const { driver, reading } = await cached(); + + await spoil(driver, { schema: 0 }); + + const [site] = (await cacheStats(reading, 'a.tv')).sites; + + expect(site).toMatchObject({ entries: 3, unreadable: 1, programmes: 2 }); + // Still there: looking is not a run. + expect( + await driver.readMeta({ site: 'a.tv', channelId: 'two', day: '2026-07-18' }), + ).toBeDefined(); + }); + + it('counts the day of an entry it cannot read among the days held', async () => { + const { driver, reading } = await cached(); + + await spoil(driver, { schema: 0 }, { site: 'a.tv', channelId: 'one', day: '2026-07-19' }); + + expect((await cacheStats(reading, 'a.tv')).sites[0]).toMatchObject({ + unreadable: 1, + empty: 0, + days: { first: '2026-07-17', last: '2026-07-19' }, + grabbed: { first: '2026-07-17T03:00:00.000Z', last: '2026-07-18T03:00:00.000Z' }, + }); + }); + + it("counts what the config's invalidate voids apart from what it cannot read", async () => { + const { reading } = await cached( + (meta, key) => key.channelId === 'one' && meta.programmeCount > 0, + ); + + expect((await cacheStats(reading, 'a.tv')).sites[0]).toMatchObject({ + unreadable: 0, + invalidated: 1, + programmes: 1, + }); + }); + + /** Make a memory store answer a channel's window in one question, as sqlite does. */ + function batching(driver: MemoryCacheDriver): void { + Object.assign(driver, { + readMetas: async (keys: ChannelDayKey[]) => + Promise.all(keys.map((key) => driver.readMeta(key))), + }); + } + + for (const [store, batched, reads] of [ + // Each key once, as the manager would read them anyway. + ['a key at a time', false, 4], + // A failed question, and then a key at a time: a channel of one day is + // not asked about twice. + ['a channel at a time', true, 6], + ] as const) { + it(`counts an entry the store cannot open, says why, and goes on — asked ${store}`, async () => { + const { driver, reading } = await cached(); + const readMeta = driver.readMeta.bind(driver); + let read = 0; + + // A file this user may not read: one in a channel of two days, and the + // one day of another. + driver.readMeta = async (key) => { + read++; + + if ( + `${key.site} ${key.channelId} ${key.day}` === 'a.tv one 2026-07-17' || + key.channelId === 'two' + ) { + throw Object.assign(new Error(`EACCES: permission denied, open '${key.day}'`), { + code: 'EACCES', + }); + } + + return readMeta(key); + }; + + if (batched) { + batching(driver); + } + + const stats = await cacheStats(reading); + + expect(stats.sites[0]).toMatchObject({ + site: 'a.tv', + entries: 3, + failed: 2, + unreadable: 0, + empty: 1, + programmes: 0, + days: { first: '2026-07-17', last: '2026-07-19' }, + }); + expect(stats.sites[1]).toMatchObject({ site: 'b.tv', failed: 0, programmes: 1 }); + expect(stats.totals.failed).toBe(2); + expect(reading.stderr.text.split('\n').toSorted()).toEqual([ + '', + "a.tv one 2026-07-17: could not be read: EACCES: permission denied, open '2026-07-17'", + "a.tv two 2026-07-18: could not be read: EACCES: permission denied, open '2026-07-18'", + ]); + expect(read).toBe(reads); + }); + + it(`stops for a look that was stopped, rather than counting what it could not read — asked ${store}`, async () => { + const { driver, reading } = await cached(); + const controller = new AbortController(); + + if (batched) { + batching(driver); + } + + driver.readMeta = async () => { + controller.abort(); + + throw new Error('interrupted'); + }; + + await expect(cacheStats({ ...reading, signal: controller.signal })).rejects.toMatchObject({ + name: 'AbortError', + }); + + // A site of one channel of one day, which is asked about once. + const once = new AbortController(); + + driver.readMeta = async () => { + once.abort(); + + throw new Error('interrupted'); + }; + + await expect(cacheStats({ ...reading, signal: once.signal }, 'b.tv')).rejects.toMatchObject({ + name: 'AbortError', + }); + + // Stopped while a channel is asked about a day at a time. + const later = new AbortController(); + const fresh = await cached(); + const readMeta = fresh.driver.readMeta.bind(fresh.driver); + let asked = 0; + + if (batched) { + batching(fresh.driver); + } + + fresh.driver.readMeta = async (key) => { + if (key.channelId !== 'one') { + return readMeta(key); + } + + // Its second day, a key at a time — after a failed question, for a + // store that is asked one. + if (++asked > (batched ? 3 : 1)) { + later.abort(); + } + + throw new Error('EACCES'); + }; + + await expect( + cacheStats({ ...fresh.reading, signal: later.signal }, 'a.tv'), + ).rejects.toMatchObject({ name: 'AbortError' }); + }); + } + + it('says what of a site it could not use, and why each could not be', async () => { + const { driver, reading } = await cached((_meta, key) => key.day === '2026-07-19'); + const readMeta = driver.readMeta.bind(driver); + + await spoil(driver, { schema: 0 }); + driver.readMeta = async (key) => { + if (key.day === '2026-07-17' && key.site === 'a.tv') { + throw new Error('EACCES'); + } + + return readMeta(key); + }; + + await writeCacheStats(reading, 'a.tv'); + + expect(reading.stdout.text).toContain( + ' 1 unreadable · 1 voided by invalidate · 1 could not be read\n', + ); + // Nothing promised of the next run: a site the config has dropped is not + // grabbed again. + expect(reading.stdout.text).not.toContain('next run'); + }); + + it('writes a line or two a site, and the totals where there is more than one', async () => { + const { reading } = await cached(); + + await writeCacheStats(reading); + + expect(reading.stdout.text).toBe( + [ + 'a.tv — 2 channels, 3 channel-days from 2026-07-17 to 2026-07-19, 3 programmes', + ' grabbed 2026-07-17 03:00Z to 2026-07-19 03:00Z · 1 empty day', + ' remembers: channels, state', + // One day, and one moment, said as one. + 'b.tv — 1 channel, 1 channel-day on 2026-07-17, 1 programme', + ' grabbed 2026-07-17 04:00Z', + '', + '2 sites, 3 channels, 4 channel-days, 4 programmes', + '', + ].join('\n'), + ); + }); + + it('writes no totals for one site', async () => { + const { reading } = await cached(); + + await writeCacheStats(reading, 'b.tv'); + + expect(reading.stdout.text).toBe( + 'b.tv — 1 channel, 1 channel-day on 2026-07-17, 1 programme\n grabbed 2026-07-17 04:00Z\n', + ); + }); + + it('orders when entries were grabbed as instants, whatever offset each was written with', async () => { + const driver = new MemoryCacheDriver(); + const cache = new CacheManager({ driver }); + const key = { site: 'z.tv', channelId: 'one', day: '2026-07-17' }; + + // 10:00 at +02:00 is 08:00Z: earlier than 09:00Z, though it sorts after it + // as text. + await cache.write(key, [programme('one', 'A')], { grabbedAt: '2026-07-17T10:00:00+02:00' }); + await cache.write({ ...key, day: '2026-07-18' }, [programme('one', 'B')], { + grabbedAt: '2026-07-17T09:00:00.000Z', + }); + + const reading = { cache, stdout: new Sink(), stderr: new Sink() }; + + expect((await cacheStats(reading)).sites[0]!.grabbed).toEqual({ + first: '2026-07-17T08:00:00.000Z', + last: '2026-07-17T09:00:00.000Z', + }); + + await writeCacheStats(reading); + + expect(reading.stdout.text).toContain('grabbed 2026-07-17 08:00Z to 2026-07-17 09:00Z'); + }); + + it('reads many small sites at once, rather than a site at a time', async () => { + const driver = new MemoryCacheDriver(); + const cache = new CacheManager({ driver }); + const readMeta = driver.readMeta.bind(driver); + let reading = 0; + let most = 0; + + for (let site = 0; site < 20; site++) { + await cache.write({ site: `s${site}.tv`, channelId: 'one', day: '2026-07-17' }, []); + } + + driver.readMeta = async (key) => { + reading++; + most = Math.max(most, reading); + await new Promise((resolve) => setTimeout(resolve, 1)); + reading--; + + return readMeta(key); + }; + + const stats = await cacheStats({ cache, stdout: new Sink(), stderr: new Sink() }); + + expect(stats.totals).toMatchObject({ sites: 20, channels: 20, entries: 20 }); + expect(stats.sites.map((site) => site.site)).toEqual( + Array.from({ length: 20 }, (_, site) => `s${site}.tv`).toSorted(), + ); + expect(most).toBeGreaterThan(1); + }); + + it('reads what it has listed before the listing is done', async () => { + const { driver, reading } = await cached(); + const list = driver.list.bind(driver); + const readMeta = driver.readMeta.bind(driver); + const happened: string[] = []; + + driver.list = async function* (options) { + for await (const held of list(options)) { + happened.push('listed'); + yield held; + } + + happened.push('listing done'); + }; + driver.readMeta = async (key) => { + happened.push('read'); + + return readMeta(key); + }; + + await cacheStats(reading); + + expect(happened.indexOf('read')).toBeLessThan(happened.indexOf('listing done')); + }); + + it('counts a channel the store lists in pieces once, and every day of it', async () => { + const { driver, reading } = await cached(); + const list = driver.list.bind(driver); + + // A store that answers in whatever order suits it: a channel's days apart. + driver.list = async function* (options) { + const held = []; + + for await (const one of list(options)) { + held.push(one); + } + + // By day: one 2026-07-17, two 2026-07-18, one 2026-07-19. + yield* held.toSorted((a, b) => + (a.kind === 'entry' ? a.key.day : '').localeCompare(b.kind === 'entry' ? b.key.day : ''), + ); + }; + + expect((await cacheStats(reading, 'a.tv')).sites[0]).toMatchObject({ + channels: 2, + entries: 3, + empty: 1, + programmes: 3, + days: { first: '2026-07-17', last: '2026-07-19' }, + }); + }); + + it("names a site's groups in order, whatever order the store keeps them in", async () => { + const { reading } = await cached(); + + await reading.cache.setState('a.tv', 'remembered', []); + await reading.cache.setState('a.tv', 'cursor', []); + + expect((await cacheStats(reading, 'a.tv')).sites[0]!.state).toEqual([ + 'channels', + 'cursor', + 'remembered', + 'state', + ]); + }); + + it('says so where nothing is cached', async () => { + const { reading } = await cached(); + + expect(await writeCacheStats(reading, 'c.tv')).toBe(0); + expect(reading.stdout.text).toBe('Nothing cached for c.tv.\n'); + }); +}); + +describe('epg cache show', () => { + it('shows a site a channel at a time, a day at a time, in order', async () => { + const { reading } = await cached(); + + await showCache({ ...reading, format: 'json' }, 'a.tv'); + + expect( + reading.stdout.text + .trim() + .split('\n') + .map((line) => (JSON.parse(line) as { title: Array<{ value: string }> }).title[0]!.value), + ).toEqual(['News', 'Film', 'Late']); + // Records, as the cache keeps them: a date in XMLTV form. + expect(JSON.parse(reading.stdout.text.split('\n')[0]!)).toMatchObject({ + start: '20260717180000', + }); + }); + + it('says why it leaves an entry out', async () => { + const { driver, reading } = await cached(); + + await spoil(driver, { schema: 0 }); + + // Nothing left to show: said, and 1, with no document begun. + expect(await showCache(reading, 'a.tv', 'two')).toBe(1); + expect(reading.stderr.text).toBe( + 'two 2026-07-18: left out — written as schema 0, and this version reads 1\n' + + 'Nothing cached for a.tv two that can be read.\n', + ); + expect(reading.stdout.text).toBe(''); + }); + + it('writes an entry a line a programme, in one write', async () => { + const { reading } = await cached(); + const writes: string[] = []; + const write = reading.stdout.write.bind(reading.stdout); + + reading.stdout.write = ((chunk: string, ...rest: never[]) => { + writes.push(chunk); + + return write(chunk, ...rest); + }) as typeof reading.stdout.write; + + await showCache({ ...reading, format: 'json' }, 'a.tv'); + + // Two entries with programmes, and the empty one writes nothing. + expect(writes.filter((chunk) => chunk !== '')).toHaveLength(2); + }); + + it('goes on past an entry whose programmes cannot be read, saying why', async () => { + const { driver, reading } = await cached(); + const key = { site: 'a.tv', channelId: 'two', day: '2026-07-18' }; + const found = (await driver.read(key))!; + + await driver.write(key, new TextEncoder().encode('not json'), found.meta as StoredEntryMeta); + + expect(await showCache({ ...reading, format: 'json' }, 'a.tv')).toBe(0); + // What the codec said, which is JSON.parse's own words. + expect(reading.stderr.text).toMatch( + /^two 2026-07-18: left out — its programmes cannot be read: .+\n$/, + ); + expect(reading.stdout.text.trim().split('\n')).toHaveLength(2); + }); + + it('goes on past an entry the store cannot open, saying why', async () => { + const { driver, reading } = await cached(); + const read = driver.read.bind(driver); + + driver.read = async (key) => { + if (key.channelId === 'two') { + throw new Error("EACCES: permission denied, open '2026-07-18.ndjson'"); + } + + return read(key); + }; + + expect(await showCache({ ...reading, format: 'json' }, 'a.tv')).toBe(0); + expect(reading.stderr.text).toBe( + "two 2026-07-18: left out — could not be read: EACCES: permission denied, open '2026-07-18.ndjson'\n", + ); + expect(reading.stdout.text.trim().split('\n')).toHaveLength(2); + + // And nothing else to show is said as such, with 1. + const alone = await cached(); + + alone.driver.read = driver.read; + + expect(await showCache(alone.reading, 'a.tv', 'two')).toBe(1); + expect(alone.reading.stderr.text).toContain('Nothing cached for a.tv two that can be read.'); + }); + + it('stops a show that was stopped, rather than leaving out what it could not read', async () => { + const { driver, reading } = await cached(); + const controller = new AbortController(); + + driver.read = async () => { + controller.abort(); + + throw new Error('interrupted'); + }; + + await expect( + showCache({ ...reading, format: 'json', signal: controller.signal }, 'a.tv'), + ).rejects.toMatchObject({ name: 'AbortError' }); + expect(reading.stderr.text).toBe(''); + }); + + it("says so for a meta it cannot read, and for one the config's invalidate voids", async () => { + const unreadable = await cached(); + + await spoil(unreadable.driver, { grabbedAt: undefined }); + await showCache(unreadable.reading, 'a.tv', 'two'); + + expect(unreadable.reading.stderr.text).toContain('its meta is not one this version can read'); + + const voided = await cached(() => true); + + await showCache(voided.reading, 'b.tv'); + + expect(voided.reading.stderr.text).toContain( + "one 2026-07-17: left out — the config's invalidate voids it", + ); + }); +}); + +describe('what epg cache asks a store for', () => { + /** A memory store that says what it was asked to list. */ + async function recorded(): Promise<{ + asked: unknown[]; + reading: CacheReading & { stdout: Sink; stderr: Sink }; + }> { + const { driver, reading } = await cached(); + const asked: unknown[] = []; + const list = driver.list.bind(driver); + + driver.list = (options) => { + asked.push(options); + + return list(options); + }; + + return { asked, reading }; + } + + it('reads a channel-day it is named by, and lists nothing', async () => { + const { asked, reading } = await recorded(); + + expect(await showCache(reading, 'a.tv', 'one', '2026-07-17')).toBe(0); + expect(asked).toEqual([]); + expect(reading.stdout.text.match(/ { + const { asked, reading } = await recorded(); + + await showCache(reading, 'a.tv', 'one'); + await showState(reading, 'a.tv'); + + expect(asked).toEqual([ + { site: 'a.tv', kind: 'entry', channelId: 'one' }, + { site: 'a.tv', kind: 'state' }, + ]); + }); +}); + +describe('epg cache state', () => { + it('lists the groups, saying what each holds', async () => { + const { reading } = await cached(); + + expect(await showState(reading, 'a.tv')).toBe(0); + expect(reading.stdout.text).toMatch( + /^channels {2}2 channels, written .+ by .+\nstate {5}2 keys, written /, + ); + }); + + it('lists a channel list by its channels, and a bag by its keys', async () => { + const channels = await cached(); + const bag = await cached(); + + await showState(channels.reading, 'a.tv', 'channels', true); + await showState(bag.reading, 'a.tv', 'state', true); + + expect(channels.reading.stdout.text).toBe('one\ntwo\n'); + expect(bag.reading.stdout.text).toBe('cursor\ntoken\n'); + }); + + it('counts a group as a run would read it, and lists the keys a run reads', async () => { + const { reading } = await cached(); + + await reading.cache.setState('a.tv', 'custom', { a: 1, b: 2 }); + await reading.cache.setState('a.tv', 'list', [1, 2, 3]); + await reading.cache.setState('a.tv', 'mixed', [ + ['cursor', 1], + [2, 3], + ]); + await showState(reading, 'a.tv'); + + // A bag is a list of pairs with string keys, and a run keeps those alone. + expect(reading.stdout.text).toMatch(/^custom {4}no keys a run can read, written /m); + expect(reading.stdout.text).toMatch(/^list {6}0 keys, and 3 unreadable, written /m); + expect(reading.stdout.text).toMatch(/^mixed {5}1 key, and 1 unreadable, written /m); + + const mixed = await cached(); + + await mixed.reading.cache.setState('a.tv', 'mixed', [ + ['cursor', 1], + [2, 3], + ]); + + expect(await showState(mixed.reading, 'a.tv', 'mixed', true)).toBe(0); + expect(mixed.reading.stdout.text).toBe('cursor\n'); + expect(await showState(reading, 'a.tv', 'custom', true)).toBe(1); + expect(reading.stderr.text).toContain('a.tv custom holds no keys to list.'); + }); + + it('reads a channel list all or not at all, as a run does', async () => { + const { reading } = await cached(); + + await reading.cache.setState('a.tv', 'channels', { channels: [{ xmltvId: 'one' }, null] }); + await showState(reading, 'a.tv'); + + expect(reading.stdout.text).toMatch(/^channels {2}no channel list a run can read, written /m); + expect(await showState(reading, 'a.tv', 'channels', true)).toBe(1); + expect(reading.stderr.text).toContain('a.tv channels holds no keys to list.'); + }); + + it('lists more keys than a call could take as arguments', async () => { + const { reading } = await cached(); + const many = Array.from({ length: 200_000 }, (_, i) => [`k${i}`, i]); + + await reading.cache.setState('a.tv', 'many', many); + + expect(await showState(reading, 'a.tv', 'many', true)).toBe(0); + expect(reading.stdout.text.split('\n')).toHaveLength(200_001); + }); + + it('counts what a site remembered and has since forgotten as expired', async () => { + const { reading } = await cached(); + const now = Date.now(); + + await reading.cache.setState('a.tv', 'remembered', [ + ['kept', { until: now + 3_600_000, value: 1 }], + ['gone', { until: now - 3_600_000, value: 2 }], + ['torn', { value: 3 }], + [4, { until: now + 3_600_000, value: 4 }], + ]); + await showState(reading, 'a.tv'); + + expect(reading.stdout.text).toMatch( + /^remembered {2}1 key, and 2 expired and 1 unreadable, written /m, + ); + + const keys = await cached(); + + await keys.reading.cache.setState('a.tv', 'remembered', [ + ['kept', { until: now + 3_600_000, value: 1 }], + ['gone', { until: now - 3_600_000, value: 2 }], + ]); + await showState(keys.reading, 'a.tv', 'remembered', true); + + expect(keys.reading.stdout.text).toBe('kept\n'); + }); + + it("claims nothing of the keys a run drops by rules of its own, as a validator's day", async () => { + const { reading } = await cached(); + + // A run keeps the validators of its window alone; a look has no window. + await reading.cache.setState('a.tv', 'validators', [ + ['2020-01-01 one', { etag: '"a"' }], + ['2999-01-01 one', { etag: '"b"' }], + ]); + await showState(reading, 'a.tv'); + + expect(reading.stdout.text).toMatch(/^validators {2}2 keys, written /m); + }); + + it('describes a group the store cannot open, and goes on to the rest', async () => { + const { driver, reading } = await cached(); + const readState = driver.readState.bind(driver); + + driver.readState = async (site, key) => { + if (key === 'state') { + throw new Error('EACCES: permission denied'); + } + + return readState(site, key); + }; + + expect(await showState(reading, 'a.tv')).toBe(0); + expect(reading.stdout.text).toMatch(/^channels {2}2 channels, written /m); + expect(reading.stdout.text).toMatch( + /^state {5}could not be read: EACCES: permission denied\n/m, + ); + + const json = await cached(); + + json.driver.readState = driver.readState; + + await showState({ ...json.reading, format: 'json' }, 'a.tv'); + + expect(JSON.parse(json.reading.stdout.text)).toContainEqual({ + group: 'state', + status: 'failed', + why: 'could not be read: EACCES: permission denied', + }); + }); + + it('says when a group was written as an instant, whatever offset it was written with', async () => { + const { driver, reading } = await cached(); + + await driver.writeState('c.tv', 'state', [], { + writtenAt: '2026-07-17T10:00:00+02:00', + schema: STATE_SCHEMA, + writtenBy: '0.9.0', + }); + await showState(reading, 'c.tv'); + + expect(reading.stdout.text).toContain('written 2026-07-17 08:00Z by 0.9.0'); + }); + + it('says nothing is remembered where every group went since it was listed', async () => { + const { driver, reading } = await cached(); + + // A run in another process removing them between the listing and the look. + driver.list = async function* () { + yield { kind: 'state', site: 'a.tv', key: 'gone' }; + }; + + expect(await showState(reading, 'a.tv')).toBe(1); + expect(reading.stderr.text).toBe('Nothing remembered for a.tv.\n'); + }); + + it('exits 1 for a site none of whose groups it can read, and still describes them', async () => { + const { driver, reading } = await cached(); + + await driver.writeState('c.tv', 'state', [], { + writtenAt: '2026-07-17T00:00:00.000Z', + schema: STATE_SCHEMA + 1, + writtenBy: '9.9.9', + }); + + expect(await showState(reading, 'c.tv')).toBe(1); + expect(reading.stdout.text).toContain('state written as schema 2, and this version reads 1'); + }); + + it('writes a group written empty as JSON null', async () => { + const { reading } = await cached(); + + await reading.cache.setState('a.tv', 'empty', undefined); + await showState(reading, 'a.tv', 'empty'); + + expect(reading.stdout.text).toBe('null\n'); + }); + + it('lists the groups as JSON, one it cannot read among them', async () => { + const { driver, reading } = await cached(); + + await driver.writeState('a.tv', 'state', [], { + writtenAt: '2026-07-17T00:00:00.000Z', + schema: STATE_SCHEMA + 1, + writtenBy: '9.9.9', + }); + await showState({ ...reading, format: 'json' }, 'a.tv'); + + expect(JSON.parse(reading.stdout.text)).toEqual([ + expect.objectContaining({ group: 'channels', status: 'readable', holds: '2 channels' }), + { + group: 'state', + status: 'incompatible', + why: 'written as schema 2, and this version reads 1', + }, + ]); + + const text = await cached(); + + await text.driver.writeState('a.tv', 'state', [], { + writtenAt: '2026-07-17T00:00:00.000Z', + schema: STATE_SCHEMA + 1, + writtenBy: '9.9.9', + }); + await showState(text.reading, 'a.tv'); + + expect(text.reading.stdout.text).toMatch( + /^state {5}written as schema 2, and this version reads 1 — fetched again when next wanted$/m, + ); + }); + + it('says so for a group whose envelope says nothing it can read', async () => { + const { driver, reading } = await cached(); + + await driver.writeState('a.tv', 'state', [], { + schema: STATE_SCHEMA, + writtenBy: '9.9.9', + } as StoredStateMeta); + + expect(await showState(reading, 'a.tv', 'state')).toBe(1); + expect(reading.stderr.text).toBe( + 'a.tv state: its envelope is not one this version can read — fetched again when next wanted\n', + ); + }); + + it('says so for a group it cannot read, and exits 1', async () => { + const { driver, reading } = await cached(); + + await driver.writeState('a.tv', 'state', [], { + writtenAt: '2026-07-17T00:00:00.000Z', + schema: STATE_SCHEMA + 1, + writtenBy: '9.9.9', + }); + + expect(await showState(reading, 'a.tv', 'state')).toBe(1); + expect(reading.stderr.text).toContain( + 'a.tv state: written as schema 2, and this version reads 1 — fetched again when next wanted', + ); + }); +}); + +const sqlite = await import('../src/cache/sqlite-driver.js').catch(() => undefined); + +describe.skipIf(sqlite === undefined)( + 'epg cache stats over a store that reads a window at once', + () => { + it('counts the same as one read a key at a time', async () => { + const driver = new sqlite!.SqliteCacheDriver({ dir: '', file: ':memory:' }); + const cache = new CacheManager({ driver }); + + try { + await cache.write( + { site: 'a.tv', channelId: 'one', day: '2026-07-17' }, + [programme('one', 'A')], + { + grabbedAt: '2026-07-17T03:00:00.000Z', + }, + ); + await cache.write({ site: 'a.tv', channelId: 'one', day: '2026-07-18' }, [], { + grabbedAt: '2026-07-18T03:00:00.000Z', + }); + + expect(await cacheStats({ cache, stdout: new Sink(), stderr: new Sink() })).toMatchObject({ + sites: [{ site: 'a.tv', channels: 1, entries: 2, empty: 1, programmes: 1 }], + }); + } finally { + await cache.close(); + } + }); + }, +); diff --git a/test/cache.test.ts b/test/cache.test.ts index 4fa0ac6..c5b79cc 100644 --- a/test/cache.test.ts +++ b/test/cache.test.ts @@ -1,7 +1,7 @@ import * as fs from 'node:fs/promises'; import * as os from 'node:os'; import * as path from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { CACHE_SCHEMA, CacheManager, @@ -17,6 +17,8 @@ import { import type { CacheDriver, CacheEntryMeta, + CachedKey, + CacheListOptions, ChannelDayKey, FoundEntry, FoundMeta, @@ -27,7 +29,7 @@ import type { StoredProgramme, StoredStateMeta, } from '../src/cache/main.js'; -import { CODECS, type CacheCodec, type CodecName } from '../src/cache/codec.js'; +import { CODECS, CorruptEntryError, type CacheCodec, type CodecName } from '../src/cache/codec.js'; import { getXmltvOffset, getXmltvPrecision, @@ -892,6 +894,20 @@ describe('CacheManager', () => { this.groups.delete(`${site}|${key}`); } + async *list(): AsyncIterable { + for (const id of this.entries.keys()) { + const [site = '', channelId = '', day = ''] = id.split('|'); + + yield { kind: 'entry', key: { site, channelId, day } }; + } + + for (const id of this.groups.keys()) { + const [site = '', key = ''] = id.split('|'); + + yield { kind: 'state', site, key }; + } + } + async close(): Promise { this.closed++; } @@ -1452,11 +1468,32 @@ describe.skipIf(sqlite === undefined)('SqliteCacheDriver', () => { await using store = cache(new SqliteCacheDriver({ dir })); - // The meta is still good, and it is what the manager judges an entry by — so - // the entry stays, and a count disagreeing with what came back is what makes - // the day worth grabbing again. + // The meta is still good, so a staleness check alone keeps the entry; a + // read finds what it holds is not the two programmes it says, and removes + // it, so the day reads as never grabbed and the next run grabs it again. expect(await store.getMeta(key)).toMatchObject({ programmeCount: 2 }); - expect(await store.read(key)).toEqual([]); + expect(await store.read(key)).toBeUndefined(); + expect(await store.getMeta(key)).toBeUndefined(); + }); + + it('removes an entry whose programmes a codec cannot read at all, as a run reads it', async () => { + const lines = path.join(dir, 'lines'); + const store = ndjson({ dir: lines }); + + await store.write(key, [programme(), programme()]); + + const file = path.join(lines, key.site, key.channelId, `${key.day}.ndjson`); + const content = (await fs.readFile(file, 'utf8')).trimEnd().split('\n'); + + // The last programme cut short, which `JSON.parse` throws on — and which + // used to throw out of the merge with it. + await fs.writeFile( + file, + `${content.slice(0, -1).join('\n')}\n${content.at(-1)!.slice(0, 20)}\n`, + ); + + expect(await store.read(key)).toBeUndefined(); + expect(await store.getMeta(key)).toBeUndefined(); }); it('answers a batch of metas in one statement, in the order asked', async () => { @@ -1530,6 +1567,201 @@ describe.skipIf(sqlite === undefined)('SqliteCacheDriver', () => { expect(await store.read(key)).toHaveLength(1); }); + it('changes nothing it only reads, of tables another version made', async () => { + const file = path.join(dir, 'cache.sqlite'); + const first = new SqliteCacheDriver({ dir }); + + await cache(first).write(key, [programme()]); + await cache(first).setState(key.site, 'state', [['cursor', 1]]); + await first.close(); + + const { DatabaseSync } = await import('node:sqlite'); + const db = new DatabaseSync(file); + db.exec('PRAGMA user_version = 99'); + db.close(); + + const looked = new SqliteCacheDriver({ dir }); + const listed: unknown[] = []; + + for await (const held of looked.list()) { + listed.push(held); + } + + expect(listed).toEqual([]); + expect(await looked.readMeta(key)).toBeUndefined(); + expect(await looked.readMetas([key])).toEqual([undefined]); + expect(await looked.read(key)).toBeUndefined(); + expect(await looked.readState(key.site, 'state')).toBeUndefined(); + expect(await looked.prune({ before: '2099-01-01' })).toBe(0); + await looked.delete(key); + await looked.deleteState(key.site, 'state'); + await looked.close(); + + // The tables, and what is in them, as another version left them. + const after = new DatabaseSync(file); + + expect(after.prepare('PRAGMA user_version').get()).toEqual({ user_version: 99 }); + expect(after.prepare('SELECT count(*) AS n FROM entries').get()).toEqual({ n: 1 }); + expect(after.prepare('SELECT count(*) AS n FROM state').get()).toEqual({ n: 1 }); + after.close(); + }); + + it.skipIf(process.getuid?.() === 0)( + 'refuses a place it could never make the database, before anything is asked', + async () => { + const locked = path.join(dir, 'locked'); + + await fs.mkdir(locked, { mode: 0o500 }); + + try { + expect(() => new SqliteCacheDriver({ dir: path.join(locked, 'cache') })).toThrow( + `The sqlite cache cannot be made at ${path.join(locked, 'cache', 'cache.sqlite')}: ${locked} is not writable`, + ); + } finally { + await fs.chmod(locked, 0o700); + } + }, + ); + + it('lists a page at a time, and a change made while listing is made at once', async () => { + const file = path.join(dir, 'cache.sqlite'); + const driver = new SqliteCacheDriver({ dir }); + const store = cache(driver); + const days = Array.from({ length: 2500 }, (_, i) => + new Date(Date.UTC(2020, 0, 1) + i * 86_400_000).toISOString().slice(0, 10), + ); + + for (const day of days) { + await store.write({ ...key, day }, []); + } + + const { DatabaseSync } = await import('node:sqlite'); + const other = new DatabaseSync(file); + const seen: string[] = []; + + for await (const held of driver.list()) { + seen.push((held as { key: ChannelDayKey }).key.day); + + // Removed as it is handed out — and seen gone from another connection + // straight after, which a write left waiting on an open read would not be. + await store.delete((held as { key: ChannelDayKey }).key); + + if (seen.length === 1500) { + expect(other.prepare('SELECT count(*) AS n FROM entries').get()).toEqual({ n: 1000 }); + } + } + + other.close(); + await driver.close(); + + expect(seen).toEqual(days); + }); + + it('logs ahead for a prune too, which changes what is there', async () => { + const file = path.join(dir, 'cache.sqlite'); + const first = new SqliteCacheDriver({ dir }); + + await cache(first).write(key, [programme()]); + await first.close(); + + const { DatabaseSync } = await import('node:sqlite'); + const db = new DatabaseSync(file); + db.exec('PRAGMA journal_mode = DELETE'); + db.close(); + + const driver = new SqliteCacheDriver({ dir }); + + expect(await driver.prune({ before: '2099-01-01' })).toBe(1); + await driver.close(); + + const after = new DatabaseSync(file); + + expect(after.prepare('PRAGMA journal_mode').get()).toEqual({ journal_mode: 'wal' }); + after.close(); + }); + + it('opens nothing again once closed', async () => { + const driver = new SqliteCacheDriver({ dir }); + + await driver.close(); + + await expect(driver.readMeta(key)).rejects.toThrow('This sqlite cache is closed'); + await expect(cache(driver).write(key, [])).rejects.toThrow('This sqlite cache is closed'); + }); + + it('logs ahead from its first write, whatever it read before', async () => { + const file = path.join(dir, 'cache.sqlite'); + const first = new SqliteCacheDriver({ dir }); + + await cache(first).write(key, [programme()]); + await first.close(); + + // A copy someone took out of WAL to move it. + const { DatabaseSync } = await import('node:sqlite'); + const db = new DatabaseSync(file); + db.exec('PRAGMA journal_mode = DELETE'); + db.close(); + + const driver = new SqliteCacheDriver({ dir }); + + await driver.readMeta(key); + await cache(driver).write(key, [programme()]); + await driver.close(); + + const after = new DatabaseSync(file); + + expect(after.prepare('PRAGMA journal_mode').get()).toEqual({ journal_mode: 'wal' }); + after.close(); + }); + + it('steps two listings through at once, each to its end', async () => { + const driver = new SqliteCacheDriver({ dir }); + const store = cache(driver); + + for (const day of ['2026-07-17', '2026-07-18', '2026-07-19']) { + await store.write({ ...key, day }, []); + } + + const first = driver.list()[Symbol.asyncIterator](); + const second = driver.list()[Symbol.asyncIterator](); + const days: string[][] = [[], []]; + + // Turn about, so neither is read whole before the other starts. + for (let turn = 0; turn < 3; turn++) { + for (const [index, listing] of [first, second].entries()) { + const { value } = await listing.next(); + + days[index]!.push((value as { key: ChannelDayKey }).key.day); + } + } + + expect(days).toEqual([ + ['2026-07-17', '2026-07-18', '2026-07-19'], + ['2026-07-17', '2026-07-18', '2026-07-19'], + ]); + expect((await first.next()).done).toBe(true); + await driver.close(); + }); + + it('makes nothing where there is no database, until it writes', async () => { + const nowhere = path.join(dir, 'not', 'here'); + const driver = new SqliteCacheDriver({ dir: nowhere }); + + expect(await driver.readMeta(key)).toBeUndefined(); + expect(await driver.prune({ before: '2099-01-01' })).toBe(0); + + for await (const held of driver.list()) { + expect.unreachable(String(held)); + } + + expect(await fs.stat(nowhere).catch(() => undefined)).toBeUndefined(); + + await cache(driver).write(key, [programme()]); + + expect(await cache(driver).read(key)).toHaveLength(1); + await driver.close(); + }); + it("keeps a site's state in a row of its own, one per group", async () => { await using store = cache(new SqliteCacheDriver({ dir })); @@ -1672,7 +1904,7 @@ describe('a channel-day a thread encoded for its store', () => { const brief: CacheCodec = { encode: (programmes) => CODECS.json.encode(programmes.map(({ desc: _desc, ...kept }) => kept as XmltvProgramme)), - decode: (body) => CODECS.json.decode(body), + decode: (body, meta) => CODECS.json.decode(body, meta), }; class Brief extends MemoryCacheDriver { @@ -1741,3 +1973,802 @@ describe('a meta longer than one read', () => { }); } }); + +describe('what a store lists', () => { + let dir: string; + + beforeEach(async () => { + dir = await fs.mkdtemp(path.join(os.tmpdir(), 'epg-cache-list-')); + }); + + afterEach(async () => { + await fs.rm(dir, { recursive: true, force: true }); + }); + + /** What a listing named, as sortable lines. */ + async function listed(driver: CacheDriver, options?: CacheListOptions): Promise { + const names: string[] = []; + + for await (const held of driver.list(options)) { + names.push( + held.kind === 'entry' + ? `entry ${held.key.site} | ${held.key.channelId} | ${held.key.day}` + : `state ${held.site} | ${held.key}`, + ); + } + + return names.sort(); + } + + const meta = (): StoredStateMeta => ({ + writtenAt: '2026-07-17T12:00:00.000Z', + schema: STATE_SCHEMA, + writtenBy: __PKG_VERSION__, + }); + + const drivers: Array<[string, () => CacheDriver]> = [ + ['ndjson', () => new FsNdjsonCacheDriver({ dir })], + ['xmltv', () => new FsXmltvCacheDriver({ dir })], + ['memory', () => new MemoryCacheDriver()], + ...(sqlite === undefined + ? [] + : ([['sqlite', () => new sqlite.SqliteCacheDriver({ dir })]] as Array< + [string, () => CacheDriver] + >)), + ]; + + for (const [name, make] of drivers) { + describe(`the ${name} driver`, () => { + it('lists every entry and every group, and a site of them when asked', async () => { + const driver = make(); + const store = cache(driver); + + try { + await store.write({ site: 'a.tv', channelId: 'one', day: '2026-07-17' }, [programme()]); + await store.write({ site: 'a.tv', channelId: 'one', day: '2026-07-18' }, []); + await store.write({ site: 'a.tv', channelId: 'two', day: '2026-07-17' }, [programme()]); + await store.write({ site: 'b.tv', channelId: 'one', day: '2026-07-17' }, [programme()]); + await driver.writeState('a.tv', 'channels', { channels: [] }, meta()); + await driver.writeState('b.tv', 'state', [['cursor', 42]], meta()); + + expect(await listed(driver)).toEqual([ + 'entry a.tv | one | 2026-07-17', + 'entry a.tv | one | 2026-07-18', + 'entry a.tv | two | 2026-07-17', + 'entry b.tv | one | 2026-07-17', + 'state a.tv | channels', + 'state b.tv | state', + ]); + expect(await listed(driver, { site: 'b.tv' })).toEqual([ + 'entry b.tv | one | 2026-07-17', + 'state b.tv | state', + ]); + expect(await listed(driver, { site: 'c.tv' })).toEqual([]); + } finally { + await driver.close?.(); + } + }); + + it('narrows to one kind, and to one channel', async () => { + const driver = make(); + const store = cache(driver); + + try { + await store.write({ site: 'a.tv', channelId: 'one', day: '2026-07-17' }, []); + await store.write({ site: 'a.tv', channelId: 'two', day: '2026-07-17' }, []); + await store.write({ site: 'b.tv', channelId: 'one', day: '2026-07-17' }, []); + await driver.writeState('a.tv', 'state', [], meta()); + await driver.writeState('b.tv', 'state', [], meta()); + + expect(await listed(driver, { site: 'a.tv', kind: 'state' })).toEqual([ + 'state a.tv | state', + ]); + expect(await listed(driver, { kind: 'entry', site: 'b.tv' })).toEqual([ + 'entry b.tv | one | 2026-07-17', + ]); + // A channel's entries are all a channel has: state is a site's. + expect(await listed(driver, { site: 'a.tv', channelId: 'two' })).toEqual([ + 'entry a.tv | two | 2026-07-17', + ]); + expect(await listed(driver, { site: 'a.tv', channelId: 'two', kind: 'state' })).toEqual( + [], + ); + // And in every site, where it is looked for. + expect(await listed(driver, { channelId: 'one' })).toEqual([ + 'entry a.tv | one | 2026-07-17', + 'entry b.tv | one | 2026-07-17', + ]); + } finally { + await driver.close?.(); + } + }); + + it('names a key as it was written, whatever it had to be made into to be kept', async () => { + const driver = make(); + const store = cache(driver); + + try { + await store.write({ site: 'example.com:8080', channelId: '..', day: '2026-07-17' }, []); + await store.write({ site: 'example.com:8080', channelId: 'a/b', day: '2026-07-17' }, []); + await driver.writeState('example.com:8080', 'x/y', null, meta()); + + expect(await listed(driver, { site: 'example.com:8080' })).toEqual([ + 'entry example.com:8080 | .. | 2026-07-17', + 'entry example.com:8080 | a/b | 2026-07-17', + 'state example.com:8080 | x/y', + ]); + } finally { + await driver.close?.(); + } + }); + + it('lists nothing for a store that holds nothing', async () => { + const driver = make(); + + try { + expect(await listed(driver)).toEqual([]); + } finally { + await driver.close?.(); + } + }); + + it('lists what is left after a prune, and nothing removed', async () => { + const driver = make(); + const store = cache(driver); + + try { + await store.write({ site: 'a.tv', channelId: 'one', day: '2026-07-16' }, []); + await store.write({ site: 'a.tv', channelId: 'one', day: '2026-07-17' }, []); + await store.prune({ before: '2026-07-17' }); + + expect(await listed(driver)).toEqual(['entry a.tv | one | 2026-07-17']); + } finally { + await driver.close?.(); + } + }); + }); + } + + it('passes over a file a cache directory holds that the driver did not write', async () => { + const driver = new FsNdjsonCacheDriver({ dir }); + + await cache(driver).write({ site: 'a.tv', channelId: 'one', day: '2026-07-17' }, []); + // An entry being written, a note, an entry of another driver, a file at the + // top, and a directory whose name no encoding made. + await fs.writeFile(path.join(dir, 'a.tv', 'one', '.2026-07-18.ndjson.tmp'), ''); + await fs.writeFile(path.join(dir, 'a.tv', 'one', 'notes.txt'), ''); + await fs.writeFile(path.join(dir, 'a.tv', 'one', '2026-07-19.xml'), ''); + await fs.writeFile(path.join(dir, 'a.tv', 'README'), ''); + await fs.writeFile(path.join(dir, 'README'), ''); + // An entry in a site, and a channel, no encoding could have named. + await fs.mkdir(path.join(dir, '100%', 'one'), { recursive: true }); + await fs.writeFile(path.join(dir, '100%', 'one', '2026-07-17.ndjson'), ''); + await fs.mkdir(path.join(dir, 'a.tv', '50%')); + await fs.writeFile(path.join(dir, 'a.tv', '50%', '2026-07-17.ndjson'), ''); + // Names it would have made otherwise: a mount's, and a lower-case escape. + await fs.mkdir(path.join(dir, 'lost+found'), { mode: 0o000 }); + await fs.mkdir(path.join(dir, 'a.tv', 'a%2fb')); + await fs.writeFile(path.join(dir, 'a.tv', 'a%2fb', '2026-07-17.ndjson'), ''); + + expect(await listed(driver)).toEqual(['entry a.tv | one | 2026-07-17']); + }); + + it('lists through a link as a read follows one', async () => { + const driver = new FsNdjsonCacheDriver({ dir }); + const elsewhere = await fs.mkdtemp(path.join(os.tmpdir(), 'epg-cache-elsewhere-')); + + try { + const moved = new FsNdjsonCacheDriver({ dir: elsewhere }); + + await cache(moved).write({ site: 'big.tv', channelId: 'one', day: '2026-07-17' }, []); + await cache(moved).write({ site: 'big.tv', channelId: 'two', day: '2026-07-17' }, []); + await cache(driver).write({ site: 'a.tv', channelId: 'one', day: '2026-07-17' }, []); + // A site moved to another disk and linked back, a channel linked in, and + // one day's file linked in. + await fs.symlink(path.join(elsewhere, 'big.tv'), path.join(dir, 'big.tv')); + await fs.symlink(path.join(elsewhere, 'big.tv', 'two'), path.join(dir, 'a.tv', 'two')); + await fs.symlink( + path.join(elsewhere, 'big.tv', 'one', '2026-07-17.ndjson'), + path.join(dir, 'a.tv', 'one', '2026-07-18.ndjson'), + ); + + expect(await listed(driver)).toEqual([ + 'entry a.tv | one | 2026-07-17', + 'entry a.tv | one | 2026-07-18', + 'entry a.tv | two | 2026-07-17', + 'entry big.tv | one | 2026-07-17', + 'entry big.tv | two | 2026-07-17', + ]); + } finally { + await fs.rm(elsewhere, { recursive: true, force: true }); + } + }); + + it.skipIf(process.getuid?.() === 0)( + 'passes over a directory it cannot read, unless that is the one it was asked for', + async () => { + const driver = new FsNdjsonCacheDriver({ dir }); + + await cache(driver).write({ site: 'a.tv', channelId: 'one', day: '2026-07-17' }, []); + // A name a site could have, in a directory nobody here may read. + await fs.mkdir(path.join(dir, '.Trash-0'), { mode: 0o000 }); + + try { + expect(await listed(driver)).toEqual(['entry a.tv | one | 2026-07-17']); + // Looked for in every site, it is in one found by the walk. + expect(await listed(driver, { channelId: 'one' })).toEqual([ + 'entry a.tv | one | 2026-07-17', + ]); + await expect(listed(driver, { site: '.Trash-0' })).rejects.toThrow(/EACCES/); + } finally { + await fs.chmod(path.join(dir, '.Trash-0'), 0o700); + } + }, + ); + + it.skipIf(process.getuid?.() === 0)( + 'says so of a channel it cannot read, rather than list it as holding nothing', + async () => { + const driver = new FsNdjsonCacheDriver({ dir }); + + await cache(driver).write({ site: 'a.tv', channelId: 'one', day: '2026-07-17' }, []); + await cache(driver).write({ site: 'a.tv', channelId: 'locked', day: '2026-07-17' }, []); + // Written by a run as somebody else, who kept it to themselves. + await fs.chmod(path.join(dir, 'a.tv', 'locked'), 0o000); + + try { + for (const options of [{}, { site: 'a.tv' }, { channelId: 'locked' }]) { + await expect(listed(driver, options)).rejects.toThrow(/EACCES/); + } + + // Another channel, asked for alone, is not that one. + expect(await listed(driver, { site: 'a.tv', channelId: 'one' })).toEqual([ + 'entry a.tv | one | 2026-07-17', + ]); + expect(await listed(driver, { channelId: 'one' })).toEqual([ + 'entry a.tv | one | 2026-07-17', + ]); + } finally { + await fs.chmod(path.join(dir, 'a.tv', 'locked'), 0o700); + } + }, + ); + + it('prunes through a link as it lists through one, and leaves the link', async () => { + const driver = new FsNdjsonCacheDriver({ dir }); + const elsewhere = await fs.mkdtemp(path.join(os.tmpdir(), 'epg-cache-elsewhere-')); + + try { + const moved = new FsNdjsonCacheDriver({ dir: elsewhere }); + + await cache(moved).write({ site: 'big.tv', channelId: 'one', day: '2026-07-10' }, []); + await cache(moved).write({ site: 'big.tv', channelId: 'one', day: '2026-07-17' }, []); + await cache(moved).write({ site: 'other.tv', channelId: 'two', day: '2026-07-10' }, []); + await cache(driver).write({ site: 'a.tv', channelId: 'one', day: '2026-07-17' }, []); + // A site moved to another disk and linked back, and a channel linked in. + await fs.symlink(path.join(elsewhere, 'big.tv'), path.join(dir, 'big.tv')); + await fs.symlink(path.join(elsewhere, 'other.tv', 'two'), path.join(dir, 'a.tv', 'two')); + + expect(await driver.prune({ before: '2026-07-15' })).toBe(2); + expect(await listed(driver)).toEqual([ + 'entry a.tv | one | 2026-07-17', + 'entry big.tv | one | 2026-07-17', + ]); + // The links stay, the channel's emptied: where one points was somebody's + // choice. + expect((await fs.lstat(path.join(dir, 'big.tv'))).isSymbolicLink()).toBe(true); + expect((await fs.lstat(path.join(dir, 'a.tv', 'two'))).isSymbolicLink()).toBe(true); + expect(await fs.readdir(path.join(elsewhere, 'other.tv', 'two'))).toEqual([]); + } finally { + await fs.rm(elsewhere, { recursive: true, force: true }); + } + }); + + it.skipIf(process.getuid?.() === 0)( + 'prunes past a directory it cannot read, or could not have named', + async () => { + const driver = new FsNdjsonCacheDriver({ dir }); + + await cache(driver).write({ site: 'a.tv', channelId: 'one', day: '2026-07-10' }, []); + await cache(driver).write({ site: 'a.tv', channelId: 'one', day: '2026-07-17' }, []); + // A mount's: one nobody here may read, and an empty one no site is called. + await fs.mkdir(path.join(dir, '.Trash-0', 'files'), { recursive: true }); + await fs.chmod(path.join(dir, '.Trash-0'), 0o000); + await fs.mkdir(path.join(dir, 'lost+found')); + + try { + expect(await driver.prune({ before: '2026-07-15' })).toBe(1); + expect((await fs.stat(path.join(dir, 'lost+found'))).isDirectory()).toBe(true); + + // A channel of a site nobody here may read is this driver's, and says so. + await cache(driver).write({ site: 'a.tv', channelId: 'locked', day: '2026-07-10' }, []); + await fs.chmod(path.join(dir, 'a.tv', 'locked'), 0o000); + + await expect(driver.prune({ before: '2026-07-15' })).rejects.toThrow(/EACCES/); + } finally { + await fs.chmod(path.join(dir, '.Trash-0'), 0o700); + await fs.chmod(path.join(dir, 'a.tv', 'locked'), 0o700).catch(() => undefined); + } + }, + ); + + it('lists nothing from a cache directory not made yet', async () => { + const driver = new FsNdjsonCacheDriver({ dir: path.join(dir, 'not', 'yet') }); + + expect(await listed(driver)).toEqual([]); + expect(await listed(driver, { site: 'a.tv' })).toEqual([]); + }); + + it('lists nothing from a store that keeps nothing', async () => { + expect(await listed(new NoCacheDriver())).toEqual([]); + }); +}); + +describe('a look inside the cache', () => { + const key = { site: 'a.tv', channelId: 'one', day: '2026-07-17' }; + const other = { ...key, channelId: 'two' }; + const missing = { ...key, channelId: 'three' }; + + /** A cache with one entry a run would use, and one it would remove. */ + async function looked( + invalidate?: (meta: StoredEntryMeta, at: ChannelDayKey) => boolean, + ): Promise<{ driver: MemoryCacheDriver; store: CacheManager }> { + const driver = new MemoryCacheDriver(); + const store = new CacheManager({ driver, ...(invalidate ? { invalidate } : {}) }); + + await store.write(key, [programme()], { grabbedAt: '2026-07-17T03:00:00.000Z' }); + await store.write(other, [programme()], { grabbedAt: '2026-07-17T03:00:00.000Z' }); + + const found = (await driver.read(other))!; + + await driver.write(other, found.body, { ...(found.meta as StoredEntryMeta), schema: 0 }); + + return { driver, store }; + } + + it('judges metas as a run does, and leaves the ones a run would remove', async () => { + const { driver, store } = await looked(); + + expect(await store.inspectMetas([key, other, missing])).toEqual([ + { + status: 'readable', + meta: stamped({ grabbedAt: '2026-07-17T03:00:00.000Z', programmeCount: 1 }), + }, + { + status: 'incompatible', + meta: { + ...stamped({ grabbedAt: '2026-07-17T03:00:00.000Z', programmeCount: 1 }), + schema: 0, + }, + }, + undefined, + ]); + expect(await driver.readMeta(other)).toBeDefined(); + + // Where a run, asking the same, removes it. + expect(await store.getMetas([other])).toEqual([undefined]); + expect(await driver.readMeta(other)).toBeUndefined(); + }); + + it("says what the config's invalidate voids, and leaves that too", async () => { + const { driver, store } = await looked((_meta, at) => at.channelId === 'one'); + + const meta = stamped({ grabbedAt: '2026-07-17T03:00:00.000Z', programmeCount: 1 }); + + expect(await store.inspectMetas([key])).toEqual([{ status: 'invalidated', meta }]); + expect(await store.inspect(key)).toEqual({ status: 'invalidated', meta }); + expect(await driver.readMeta(key)).toBeDefined(); + }); + + it('reads an entry with its programmes, or says why it would not', async () => { + const { driver, store } = await looked(); + + expect(await store.inspect(key)).toEqual({ + status: 'readable', + meta: stamped({ grabbedAt: '2026-07-17T03:00:00.000Z', programmeCount: 1 }), + programmes: [programme()], + }); + + const found = (await driver.read(key))!; + + await driver.write(key, found.body, { + ...(found.meta as StoredEntryMeta), + grabbedAt: 7 as never, + }); + + // What it said, as the store had it. + expect(await store.inspect(key)).toEqual({ + status: 'unreadable', + meta: { ...(found.meta as StoredEntryMeta), grabbedAt: 7 }, + }); + + // Of another schema, whatever fields it has: an older entry need not have + // today's. + await driver.write(key, found.body, { schema: 0 } as StoredEntryMeta); + + expect(await store.inspect(key)).toEqual({ status: 'incompatible', meta: { schema: 0 } }); + expect(await store.inspect(missing)).toBeUndefined(); + expect(await driver.read(key)).toBeDefined(); + }); + + it("reads a group of a site's state, and leaves one a run would remove", async () => { + const { driver, store } = await looked(); + + await store.setState('a.tv', 'state', [['cursor', 42]], { + writtenAt: '2026-07-17T03:00:00.000Z', + }); + await driver.writeState('a.tv', 'old', [], { + writtenAt: '2026-07-17T03:00:00.000Z', + schema: STATE_SCHEMA + 1, + writtenBy: '9.9.9', + }); + + await driver.writeState('a.tv', 'torn', [], { + schema: STATE_SCHEMA, + writtenBy: '9.9.9', + } as StoredStateMeta); + + expect(await store.inspectState('a.tv', 'state')).toEqual({ + status: 'readable', + data: [['cursor', 42]], + meta: { + writtenAt: '2026-07-17T03:00:00.000Z', + schema: STATE_SCHEMA, + writtenBy: __PKG_VERSION__, + }, + }); + expect(await store.inspectState('a.tv', 'old')).toEqual({ + status: 'incompatible', + meta: { writtenAt: '2026-07-17T03:00:00.000Z', schema: STATE_SCHEMA + 1, writtenBy: '9.9.9' }, + }); + expect(await store.inspectState('a.tv', 'torn')).toEqual({ + status: 'unreadable', + meta: { schema: STATE_SCHEMA, writtenBy: '9.9.9' }, + }); + expect(await store.inspectState('a.tv', 'none')).toBeUndefined(); + expect(await driver.readState('a.tv', 'old')).toBeDefined(); + + // Where a run, asking the same, removes it. + expect(await store.getState('a.tv', 'old')).toBeUndefined(); + expect(await driver.readState('a.tv', 'old')).toBeUndefined(); + }); + + it('lists exactly what it was asked for, whatever its driver answers with', async () => { + const driver = new MemoryCacheDriver(); + const store = new CacheManager({ driver }); + + await store.write(key, []); + await store.write({ ...key, site: 'b.tv' }, []); + await store.setState('a.tv', 'state', []); + + // A driver that narrows by nothing at all, which it may. + const unnarrowed = Object.assign(Object.create(driver) as MemoryCacheDriver, { + list: () => driver.list(), + }); + const keys: string[] = []; + + for await (const held of new CacheManager({ driver: unnarrowed }).list({ + site: 'a.tv', + kind: 'entry', + })) { + keys.push(held.kind === 'entry' ? `${held.key.site} ${held.key.channelId}` : held.key); + } + + expect(keys).toEqual(['a.tv one']); + }); + + it('says an entry whose programmes cannot be read is corrupt, and why', async () => { + const { driver, store } = await looked(); + const found = (await driver.read(key))!; + + // Bytes that are not the json codec's records at all, and not an array. + await driver.write(key, new TextEncoder().encode('[{"title": '), found.meta as StoredEntryMeta); + + expect(await store.inspect(key)).toEqual({ + status: 'corrupt', + meta: found.meta, + error: expect.any(CorruptEntryError), + }); + + await driver.write( + key, + new TextEncoder().encode('{"title": 1}'), + found.meta as StoredEntryMeta, + ); + + expect(await store.inspect(key)).toEqual({ + status: 'corrupt', + meta: found.meta, + error: new CorruptEntryError('An entry of the json codec holds no array of records'), + }); + }); + + it('knows a corrupt entry by its name, as another copy of this package throws one', () => { + const copy = Object.assign(new Error('cut short'), { name: 'CorruptEntryError' }); + + class Truncated extends CorruptEntryError {} + + expect(copy instanceof CorruptEntryError).toBe(true); + expect(new Truncated('cut short') instanceof CorruptEntryError).toBe(true); + expect(new Error('cut short') instanceof CorruptEntryError).toBe(false); + // Only an error is one, whatever else carries the name. + expect({ name: 'CorruptEntryError' } instanceof CorruptEntryError).toBe(false); + // A subclass is asked as any class is: another copy has none of it. + expect(new Truncated('cut short') instanceof Truncated).toBe(true); + expect(copy instanceof Truncated).toBe(false); + expect(new CorruptEntryError('cut short') instanceof Truncated).toBe(false); + }); + + it('has every codec it ships say what it cannot read is a corrupt entry, and why', () => { + const bytes = (text: string): Uint8Array => new TextEncoder().encode(text); + const thrown = (decode: () => unknown): unknown => { + try { + decode(); + } catch (error) { + return error; + } + }; + + for (const [codec, body] of [ + ['ndjson', '{"title": '], + ['json', '[{"title": '], + ['xmltv', ''], + // Records that read, with a date in them that does not. + ['ndjson', '{"start": "the evening"}'], + ['json', '[{"start": "the evening"}]'], + ] as const) { + const error = thrown(() => + CODECS[codec].decode( + bytes(body), + stamped({ grabbedAt: '2026-07-17T03:00:00.000Z', programmeCount: 1 }), + ), + ); + + expect(error).toBeInstanceOf(CorruptEntryError); + expect((error as Error).name).toBe('CorruptEntryError'); + expect((error as Error).message).toMatch( + new RegExp(`^An entry of the ${codec} codec does not read: .+`), + ); + // What the parser said, for whoever has to look into it. + expect((error as Error).cause).toBeInstanceOf(Error); + } + }); + + it("says so where a codec cannot read an entry's programmes at all", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'epg-cache-corrupt-')); + + try { + const store = ndjson({ dir }); + + await store.write(key, [programme(), programme()]); + + const file = path.join(dir, 'a.tv', 'one', '2026-07-17.ndjson'); + const lines = (await fs.readFile(file, 'utf8')).trimEnd().split('\n'); + + // The last programme cut short. + await fs.writeFile(file, `${lines.slice(0, -1).join('\n')}\n${lines.at(-1)!.slice(0, 20)}\n`); + + const inspected = await store.inspect(key); + + expect(inspected).toEqual({ + status: 'corrupt', + meta: expect.objectContaining({ programmeCount: 2 }), + error: expect.any(CorruptEntryError), + }); + expect((inspected as { error: Error }).error.cause).toBeInstanceOf(SyntaxError); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + }); + + it('says so of a driver with no list, as one written before every driver had one', async () => { + const driver = new MemoryCacheDriver(); + + (driver as { list?: unknown }).list = undefined; + + const listing = new CacheManager({ driver }).list()[Symbol.asyncIterator](); + + await expect(listing.next()).rejects.toThrow("This cache's driver has no list()"); + }); + + it('says an entry of a codec this package ships that holds fewer than it says is corrupt', async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'epg-cache-short-')); + + try { + const lines = ndjson({ dir: path.join(dir, 'lines') }); + const documents = xmltv({ dir: path.join(dir, 'documents') }); + + await lines.write(key, [programme(), programme(), programme()]); + await documents.write(key, [programme(), programme(), programme()]); + + // An ndjson entry cut between lines, and an xmltv document that ends + // early: each still reads, as fewer programmes than it says it holds. + const lineFile = path.join(dir, 'lines', 'a.tv', 'one', '2026-07-17.ndjson'); + const kept = (await fs.readFile(lineFile, 'utf8')).split('\n').slice(0, 2).join('\n'); + + await fs.writeFile(lineFile, `${kept}\n`); + + const documentFile = path.join(dir, 'documents', 'a.tv', 'one', '2026-07-17.xml'); + const document = await fs.readFile(documentFile, 'utf8'); + + await fs.writeFile(documentFile, document.slice(0, document.lastIndexOf(' Error | undefined): { + driver: MemoryCacheDriver; + store: CacheManager; + warned: string[]; + /** The meta each read handed the codec. */ + handed: StoredEntryMeta[]; + } { + const driver = new MemoryCacheDriver(); + const warned: string[] = []; + const handed: StoredEntryMeta[] = []; + + Object.defineProperty(driver, 'codec', { + value: { + encode: CODECS.json.encode, + decode: (body: Uint8Array, meta: StoredEntryMeta) => { + const error = fails(body); + + if (error !== undefined) { + throw error; + } + + handed.push(meta); + + return CODECS.json.decode(body, meta).slice(1); + }, + } satisfies CacheCodec, + }); + + return { + driver, + store: new CacheManager({ driver, warn: (message) => warned.push(message) }), + warned, + handed, + }; + } + + it('throws what is no fault of the bytes, rather than take the entry for a damaged one', async () => { + const store = cache(new MemoryCacheDriver()); + + await store.write(key, [programme()]); + + // A fault of this code's own, as a release could ship one, where the + // parser it calls throws on input that is fine. + const parse = vi.spyOn(JSON, 'parse').mockImplementation(() => { + throw new TypeError('a fault of its own'); + }); + + try { + const thrown = await store.read(key).catch((error: unknown) => error); + + expect(thrown).toEqual(new TypeError('a fault of its own')); + expect(thrown).not.toBeInstanceOf(CorruptEntryError); + } finally { + parse.mockRestore(); + } + + // Kept, for a release without the fault to read. + expect(await store.read(key)).toHaveLength(1); + }); + + it('reads back a programme of no channel that the xmltv codec wrote', () => { + const written = [programme({ channel: '' })]; + + expect( + CODECS.xmltv.decode( + CODECS.xmltv.encode(written), + stamped({ grabbedAt: '2026-07-17T03:00:00.000Z', programmeCount: 1 }), + ), + ).toMatchObject([{ channel: '' }]); + }); + + it('has every codec it ships hold an entry to the count it was written with', () => { + for (const codec of ['ndjson', 'json', 'xmltv'] as const) { + const body = CODECS[codec].encode([programme(), programme()]); + const meta = (programmeCount: number): StoredEntryMeta => ({ + schema: CACHE_SCHEMA, + writtenBy: '0.9.0', + grabbedAt: '2026-07-17T03:00:00.000Z', + programmeCount, + }); + + expect(CODECS[codec].decode(body, meta(2))).toHaveLength(2); + expect(() => CODECS[codec].decode(body, meta(3))).toThrow( + new CorruptEntryError( + `An entry of the ${codec} codec reads as 2 programmes, of the 3 it says it holds`, + ), + ); + } + }); + + it("leaves what a driver's own codec gives back to it, however many", async () => { + const { store, handed } = ownCodec(() => undefined); + + await store.write(key, [programme(), programme()]); + + // Fewer than it says is the codec's own business — handed the meta, to + // hold the entry to it or not. + expect(await store.inspect(key)).toMatchObject({ status: 'readable' }); + expect(await store.read(key)).toHaveLength(1); + expect(handed).toEqual([ + expect.objectContaining({ programmeCount: 2 }), + expect.objectContaining({ programmeCount: 2 }), + ]); + }); + + it("throws what a driver's own codec throws, other than a corrupt entry, and keeps the entry", async () => { + const { driver, store, warned } = ownCodec((body) => + body.length === 0 ? new Error('a bug in the codec') : undefined, + ); + + await store.write(key, [programme(), programme()]); + + const found = (await driver.read(key))!; + + await driver.write(key, new Uint8Array(), found.meta as StoredEntryMeta); + + await expect(store.inspect(key)).rejects.toThrow('a bug in the codec'); + await expect(store.read(key)).rejects.toThrow('a bug in the codec'); + // Kept: a codec that throws on every entry must not empty the cache. + expect(await driver.readMeta(key)).toBeDefined(); + expect(warned).toEqual([]); + }); + + it("removes what a driver's own codec says is a corrupt entry, as it does its own", async () => { + // Thrown by this package, and by a copy of it the driver's own package brought. + for (const error of [ + new CorruptEntryError('cut short'), + Object.assign(new Error('cut short'), { name: 'CorruptEntryError' }), + ]) { + const { driver, store, warned } = ownCodec((body) => (body.length === 0 ? error : undefined)); + + await store.write(key, [programme(), programme()]); + + const found = (await driver.read(key))!; + + await driver.write(key, new Uint8Array(), found.meta as StoredEntryMeta); + + expect(await store.inspect(key)).toEqual({ status: 'corrupt', meta: found.meta, error }); + // Looking left it; a run removes it, and says so. + expect(await driver.readMeta(key)).toBeDefined(); + expect(await store.read(key)).toBeUndefined(); + expect(await driver.readMeta(key)).toBeUndefined(); + expect(warned).toEqual([ + 'one 2026-07-17: removed from the cache, its programmes unreadable (cut short) — ' + + 'the day is grabbed again by the next run', + ]); + } + }); + + it('lists what its driver lists', async () => { + const { store } = await looked(); + const keys: string[] = []; + + for await (const held of store.list({ site: 'a.tv' })) { + keys.push(held.kind === 'entry' ? held.key.channelId : held.key); + } + + expect(keys.sort()).toEqual(['one', 'two']); + }); +}); diff --git a/test/cli.test.ts b/test/cli.test.ts index 573c982..12bf683 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -1,5 +1,5 @@ import { existsSync } from 'node:fs'; -import { mkdtemp, readdir, readFile, stat, writeFile } from 'node:fs/promises'; +import { chmod, mkdtemp, readdir, readFile, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { Writable } from 'node:stream'; @@ -8,6 +8,12 @@ import { describe, expect, it, vi } from 'vitest'; import { runCli } from '../src/cli/run.js'; import { defaultDescription } from '../src/cli/scaffold.js'; +/** Whether this runtime has `node:sqlite`, which the sqlite driver needs. */ +const hasSqlite = await import('node:sqlite').then( + () => true, + () => false, +); + /** Collects everything written, so stdout and stderr can be asserted on. */ class Sink extends Writable { private chunks: string[] = []; @@ -1985,14 +1991,19 @@ describe('epg', () => { await run(['build', '--config', config, '--quiet', '--offset', '-2']); - const pruned = await run(['prune', '--config', config, '--before', '2099-01-01']); + const pruned = await run(['cache', 'prune', '--config', config, '--before', '2099-01-01']); expect(pruned.code).toBe(0); expect(pruned.stdout).toMatch(/Pruned \d+ cached entr/); // A string cutoff that looks like a date would prune far too much. - const invalid = await run(['prune', '--config', config, '--before', '2026-99-99']); + const invalid = await run(['cache', 'prune', '--config', config, '--before', '2026-99-99']); expect(invalid.code).toBe(2); expect(invalid.stderr).toContain('Invalid --before value: 2026-99-99'); + + // Moved under `cache`, and not kept where it was. + const moved = await run(['prune', '--config', config]); + expect(moved.code).toBe(2); + expect(moved.stderr).toContain('Unknown command: prune — it is epg cache prune now'); }); it('merges from the cache without grabbing', async () => { @@ -2201,3 +2212,326 @@ describe('flushing', () => { expect(stderr.text).toContain('FAILED [example.tv] one.example.tv'); }); }); + +describe('epg cache', () => { + /** A site that remembers something, so there is state to look at too. */ + async function rememberingConfig(dir: string): Promise { + return configFile( + dir, + `export default { + sites: [{ + site: 'example.tv', + channels: [{ xmltvId: 'one.example.tv', siteId: '1', name: 'One' }], + async request({ day }) { return { day }; }, + parseDay: ({ channel, day, state }) => { + state.set('cursor', 42); + + return [{ + channel: channel.xmltvId, + start: new Date(day + 'T06:00:00.000Z'), + title: [{ value: 'Show' }], + }]; + }, + }], + days: 2, + output: ${JSON.stringify(join(dir, 'guide.xml'))}, + cache: { dir: ${JSON.stringify(join(dir, 'cache'))} }, + };`, + ); + } + + async function grabbed(): Promise<{ dir: string; config: string; days: string[] }> { + const dir = await tempDir(); + const config = await rememberingConfig(dir); + + expect((await run(['grab', '--config', config, '--quiet'])).code).toBe(0); + + const stats = await run(['cache', 'stats', '--config', config, '--format', 'json']); + const { days } = ( + JSON.parse(stats.stdout) as { sites: [{ days: { first: string; last: string } }] } + ).sites[0]; + + return { dir, config, days: [days.first, days.last] }; + } + + // A driver with a connection to close, which the look must not close under + // itself before it has finished reading. + it.skipIf(!hasSqlite)('reads a sqlite cache to the end before closing it', async () => { + const dir = await tempDir(); + const config = await rememberingConfig(dir); + const driver = ['--cache-driver', 'sqlite']; + + expect((await run(['grab', '--config', config, '--quiet', ...driver])).code).toBe(0); + + const shown = await run(['cache', 'show', 'example.tv', '--config', config, ...driver]); + expect(shown.stderr).toBe(''); + expect(shown.code).toBe(0); + expect(shown.stdout.match(/ { + const { config, days } = await grabbed(); + + const xml = await run(['cache', 'show', 'example.tv', '--config', config]); + expect(xml.code).toBe(0); + expect(xml.stdout).toContain('Show'); + // Indented, for a person to read. + expect(xml.stdout).toContain('\n JSON.parse(line) as { title: unknown }); + expect(records).toHaveLength(2); + expect(records[0]!.title).toEqual([{ value: 'Show' }]); + }); + + it('says so, and exits 1, where nothing is cached for what was asked', async () => { + const { config } = await grabbed(); + + const site = await run(['cache', 'show', 'nowhere.tv', '--config', config]); + expect(site.code).toBe(1); + expect(site.stderr).toContain('Nothing cached for nowhere.tv.'); + + const day = await run([ + 'cache', + 'show', + 'example.tv', + 'one.example.tv', + '2001-01-01', + '--config', + config, + ]); + expect(day.code).toBe(1); + expect(day.stderr).toContain('Nothing cached for example.tv one.example.tv 2001-01-01.'); + }); + + it('counts what is cached per site', async () => { + const { config, days } = await grabbed(); + + const text = await run(['cache', 'stats', '--config', config]); + expect(text.code).toBe(0); + expect(text.stdout).toContain( + `example.tv — 1 channel, 2 channel-days from ${days[0]} to ${days[1]}, 2 programmes`, + ); + expect(text.stdout).toContain('remembers: state'); + + const json = await run([ + 'cache', + 'stats', + 'example.tv', + '--config', + config, + '--format', + 'json', + ]); + expect(JSON.parse(json.stdout)).toMatchObject({ + sites: [ + { + site: 'example.tv', + channels: 1, + entries: 2, + unreadable: 0, + empty: 0, + programmes: 2, + state: ['state'], + }, + ], + totals: { sites: 1, entries: 2, programmes: 2 }, + }); + + const none = await run(['cache', 'stats', 'nowhere.tv', '--config', config]); + expect(none.code).toBe(0); + expect(none.stdout).toContain('Nothing cached for nowhere.tv.'); + }); + + it('reports an entry it cannot vouch for, and leaves it where it is', async () => { + const { dir, config, days } = await grabbed(); + const file = join(dir, 'cache', 'example.tv', 'one.example.tv', `${days[0]}.ndjson`); + + await writeFile(file, 'not an entry\n'); + + const stats = await run(['cache', 'stats', '--config', config]); + expect(stats.stdout).toContain(' · 1 unreadable\n'); + + const shown = await run(['cache', 'show', 'example.tv', '--config', config]); + expect(shown.code).toBe(0); + expect(shown.stderr).toContain(`one.example.tv ${days[0]}: left out`); + expect(shown.stdout.match(/ { + const { dir, config, days } = await grabbed(); + const file = join(dir, 'cache', 'example.tv', 'one.example.tv', `${days[0]}.ndjson`); + + await chmod(file, 0o000); + + try { + const stats = await run(['cache', 'stats', '--config', config]); + expect(stats.code).toBe(0); + expect(stats.stdout).toContain(' · 1 could not be read\n'); + expect(stats.stderr).toContain( + `example.tv one.example.tv ${days[0]}: could not be read: EACCES`, + ); + + const shown = await run(['cache', 'show', 'example.tv', '--config', config]); + expect(shown.code).toBe(0); + expect(shown.stderr).toContain(`one.example.tv ${days[0]}: left out — could not be read`); + expect(shown.stdout.match(/ { + const { config } = await grabbed(); + + const groups = await run(['cache', 'state', 'example.tv', '--config', config]); + expect(groups.code).toBe(0); + expect(groups.stdout).toMatch(/^state +1 key, written \d{4}-\d{2}-\d{2} \d{2}:\d{2}Z by /m); + + const data = await run(['cache', 'state', 'example.tv', 'state', '--config', config]); + expect(JSON.parse(data.stdout)).toEqual([['cursor', 42]]); + + const withMeta = await run([ + 'cache', + 'state', + 'example.tv', + 'state', + '--config', + config, + '--format', + 'json', + ]); + expect(JSON.parse(withMeta.stdout)).toMatchObject({ + meta: { schema: 1 }, + data: [['cursor', 42]], + }); + + const keys = await run(['cache', 'state', 'example.tv', 'state', '--keys', '--config', config]); + expect(keys.stdout).toBe('cursor\n'); + + const missing = await run(['cache', 'state', 'example.tv', 'validators', '--config', config]); + expect(missing.code).toBe(1); + expect(missing.stderr).toContain('Nothing remembered for example.tv under validators.'); + + const nowhere = await run(['cache', 'state', 'nowhere.tv', '--config', config]); + expect(nowhere.code).toBe(1); + expect(nowhere.stderr).toContain('Nothing remembered for nowhere.tv.'); + }); + + it('opens no store for a command line it refuses', async () => { + const dir = await tempDir(); + const opened = join(dir, 'opened'); + // A driver of the config's own, which says when it is made. + const config = await configFile( + dir, + `import { writeFileSync } from 'node:fs'; + + export default { + sites: [${siteSource()}], + output: ${JSON.stringify(join(dir, 'guide.xml'))}, + cache: { + driver: () => { + writeFileSync(${JSON.stringify(opened)}, ''); + + throw new Error('not for this test'); + }, + }, + };`, + ); + + for (const argv of [ + ['cache', 'show'], + ['cache', 'show', 'example.tv', 'one.example.tv', '2026-99-99'], + ['cache', 'state', 'example.tv', '--keys'], + ['cache', 'show', 'example.tv', '-o', 'day.xml'], + ]) { + expect((await run([...argv, '--config', config])).code).toBe(2); + } + + expect(existsSync(opened)).toBe(false); + }); + + it('refuses what it cannot do, with the usage', async () => { + const { config } = await grabbed(); + const refused = async (argv: string[], message: string): Promise => { + const result = await run([...argv, '--config', config]); + + expect(result.code).toBe(2); + expect(result.stderr).toContain(message); + expect(result.stderr).toContain('Usage: epg'); + }; + + await refused(['cache'], 'cache needs one of show, stats, state, prune'); + await refused(['cache', 'look'], 'Unknown cache command: look'); + await refused(['cache', 'show'], 'cache show needs a site'); + await refused(['cache', 'state'], 'cache state needs a site'); + await refused(['cache', 'stats', 'a', 'b'], 'cache stats takes at most 1 argument'); + await refused( + ['cache', 'state', 'example.tv', '--keys'], + '--keys is for cache state with a group', + ); + await refused( + ['cache', 'show', 'example.tv', 'one.example.tv', 'tuesday'], + 'Invalid day value: tuesday', + ); + + // A flag a subcommand would only ignore. + for (const [argv, message] of [ + [['cache', 'show', 'example.tv', '--before', '2026-07-01'], '--before is not for cache show'], + [['cache', 'stats', '--indent', '2'], '--indent is not for cache stats'], + [['cache', 'prune', '--format', 'json'], '--format is not for cache prune'], + [['cache', 'stats', '--days', '3'], '--days is not for cache stats'], + [['cache', 'state', 'example.tv', '--refresh'], '--refresh is not for cache state'], + // What shapes a guide written somewhere, which no cache command writes. + [['cache', 'show', 'example.tv', '-o', 'day.xml'], '--output is not for cache show'], + [['cache', 'show', 'example.tv', '--profile', 'jellyfin'], '--profile is not for cache show'], + [['cache', 'show', 'example.tv', '--no-extensions'], '--extensions is not for cache show'], + [ + ['cache', 'show', 'example.tv', '--format', 'json', '--indent', '4'], + '--indent is for cache show as XMLTV, not --format json', + ], + ] as const) { + await refused([...argv], message); + } + }); +}); diff --git a/test/grabber.test.ts b/test/grabber.test.ts index 00d0252..ec506c3 100644 --- a/test/grabber.test.ts +++ b/test/grabber.test.ts @@ -215,6 +215,31 @@ function makeDaysConfig(overrides: Partial = {}): DaysConfig { } describe('grab', () => { + it('leaves out a programme without a valid start, saying so, so the entry reads back', async () => { + const cache = new CacheManager({ driver: new MemoryCacheDriver() }); + const report = collect(); + const config = makeConfig({ + parseDay: ({ day }) => [ + { start: new Date('not a time'), title: [{ value: 'Lost' }] } as never, + programme(`${day}T06:00:00.000Z`), + ], + }); + + await grab([config], { cache, now: NOW, reporter: report.reporter }); + + // Kept, the entry would read as broken and be grabbed again every run. + expect( + await cache.inspect({ site: 'example.com', channelId: 'one.example', day: TODAY }), + ).toMatchObject({ status: 'readable', meta: { programmeCount: 1 } }); + expect(report.events).toContainEqual( + expect.objectContaining({ + type: 'site:warning', + site: 'example.com', + message: `one.example ${TODAY}: 1 programme without a valid start left out`, + }), + ); + }); + it('skips fresh cache entries and fetches stale ones', async () => { const cache = new MemoryCache(); const fetchedDays: string[] = []; diff --git a/test/parallel.test.ts b/test/parallel.test.ts new file mode 100644 index 0000000..b99343a --- /dev/null +++ b/test/parallel.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from 'vitest'; +import { inParallel } from '../src/core/parallel.js'; + +describe('inParallel', () => { + it('runs every item, no more than limit at once', async () => { + let running = 0; + let most = 0; + const done: number[] = []; + + await inParallel([1, 2, 3, 4, 5, 6, 7], 3, async (item) => { + running++; + most = Math.max(most, running); + await new Promise((resolve) => setTimeout(resolve, 1)); + done.push(item); + running--; + }); + + expect(done.sort()).toEqual([1, 2, 3, 4, 5, 6, 7]); + expect(most).toBe(3); + }); + + it('refuses a limit that would run nothing', async () => { + for (const limit of [0, -1, Number.NaN, 1.5]) { + await expect(inParallel([1], limit, async () => {})).rejects.toThrow(RangeError); + } + }); + + it('starts nothing more once an item has failed', async () => { + const started: number[] = []; + + await expect( + inParallel( + Array.from({ length: 100 }, (_, i) => i), + 2, + async (item) => { + started.push(item); + await new Promise((resolve) => setTimeout(resolve, 1)); + + if (item === 0) { + throw new Error('unreadable'); + } + }, + ), + ).rejects.toThrow('unreadable'); + + // Wait out anything still going, then see what was begun: the two at + // first, and at most the one the other worker took before it heard. + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(started.length).toBeLessThanOrEqual(3); + }); + + it('takes items as they are made, no further ahead than there are workers', async () => { + const done: number[] = []; + let made = 0; + let most = 0; + + async function* listing(): AsyncGenerator { + for (let item = 0; item < 20; item++) { + made++; + // Made and not yet done: one a worker each, this one included. + most = Math.max(most, made - done.length); + yield item; + } + } + + await inParallel(listing(), 3, async (item) => { + await new Promise((resolve) => setTimeout(resolve, 1)); + done.push(item); + }); + + expect(done.toSorted((a, b) => a - b)).toEqual(Array.from({ length: 20 }, (_, i) => i)); + expect(most).toBe(3); + }); + + it('closes a source it stops taking from, and answers with the failure', async () => { + let closed = false; + + async function* listing(): AsyncGenerator { + try { + for (let item = 0; ; item++) { + yield item; + } + } finally { + closed = true; + } + } + + await expect( + inParallel(listing(), 2, async (item) => { + if (item === 3) { + throw new Error('unreadable'); + } + }), + ).rejects.toThrow('unreadable'); + expect(closed).toBe(true); + }); + + it('takes nothing more from a source once an item has failed', async () => { + const started: number[] = []; + + async function* listing(): AsyncGenerator { + yield 0; + // The next one is slow to come, and comes after the failure. + await new Promise((resolve) => setTimeout(resolve, 10)); + yield 1; + } + + await expect( + inParallel(listing(), 2, async (item) => { + started.push(item); + + if (item === 0) { + throw new Error('unreadable'); + } + }), + ).rejects.toThrow('unreadable'); + + await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(started).toEqual([0]); + }); + + it('answers with what the source threw', async () => { + async function* listing(): AsyncGenerator { + yield 1; + throw new Error('EACCES'); + } + + await expect(inParallel(listing(), 2, async () => {})).rejects.toThrow('EACCES'); + }); +}); diff --git a/test/readme-examples.ts b/test/readme-examples.ts index cd71a91..4e40a16 100644 --- a/test/readme-examples.ts +++ b/test/readme-examples.ts @@ -14,8 +14,12 @@ import ky, { HTTPError } from 'ky'; import { createSecureServer } from 'node:http2'; import { createServer as createHttpsServer } from 'node:https'; import Fastify from 'fastify'; +import { gunzipSync, gzipSync } from 'node:zlib'; import { build, + CacheManager, + CODECS, + CorruptEntryError, createGuideHandler, grabEvery, defineConfig, @@ -32,8 +36,10 @@ import { } from '../src/main.js'; import type { GrabberChannel, + CacheCodec, CacheDriver, CacheDriverFactory, + CachedKey, CacheStore, ChannelDayKey, OutputProfile, @@ -712,15 +718,46 @@ export const namedAfterTheLineup = defineConfig({ }), }); +// --- docs/api.md: epg-tools/cache ------------------------------------------ +export const looked = async (cache: CacheManager): Promise => { + const found = await cache.inspect({ site: 'example.tv', channelId: 'one', day: '2026-07-17' }); + + return found === undefined + ? 'nothing cached' + : found.status === 'readable' + ? `${found.programmes.length} of ${found.meta.programmeCount}` + : found.status; +}; + // --- docs/api.md: A driver of your own ------------------------------------- +export const gzipped: CacheCodec = { + encode: (programmes) => gzipSync(CODECS.json.encode(programmes)), + decode(body, meta) { + let json: Uint8Array; + + try { + json = gunzipSync(body); + } catch (error) { + // Bytes that are not gzip are an entry damaged, not a codec. + throw new CorruptEntryError('An entry of the gzipped codec is not gzip', { cause: error }); + } + + // Which says so itself of records it cannot read, or fewer than `meta` says. + return CODECS.json.decode(json, meta); + }, +}; + /** * The store behind the example driver. A real one is a client — Redis, * Postgres, a bucket — and stands in here as a `Map`, since what the * documentation is promising is the shape of the driver rather than the client. */ -const rows = new Map(); -const groups = new Map(); +const rows = new Map(); +const groups = new Map< + string, + { site: string; key: string; meta: StoredStateMeta; data: unknown } +>(); class KeyValueCacheDriver implements CacheDriver { // What an entry's bytes are: the records' array, as JSON. A site in a thread @@ -737,6 +774,11 @@ class KeyValueCacheDriver implements CacheDriver { return `${this.#prefix}:${key.site}:${key.channelId}:${key.day}`; } + /** Whether a row or a group is this cache's, and of the site asked about. */ + #holds(id: string, site: string, only: string | undefined): boolean { + return id.startsWith(`${this.#prefix}:`) && (only === undefined || site === only); + } + async readMeta(key: ChannelDayKey): Promise { const row = rows.get(this.#id(key)); @@ -749,7 +791,7 @@ class KeyValueCacheDriver implements CacheDriver { // A copy of the view: the bytes may sit in a buffer with room to spare. async write(key: ChannelDayKey, body: Uint8Array, meta: StoredEntryMeta): Promise { - rows.set(this.#id(key), { meta, body: new Uint8Array(body) }); + rows.set(this.#id(key), { key, meta, body: new Uint8Array(body) }); } async delete(key: ChannelDayKey): Promise { @@ -759,8 +801,8 @@ class KeyValueCacheDriver implements CacheDriver { async prune(options: { before: string }): Promise { let removed = 0; - for (const id of rows.keys()) { - if (id.slice(id.lastIndexOf(':') + 1) < options.before) { + for (const [id, row] of rows) { + if (this.#holds(id, row.key.site, undefined) && row.key.day < options.before) { rows.delete(id); removed++; } @@ -776,13 +818,30 @@ class KeyValueCacheDriver implements CacheDriver { } async writeState(site: string, key: string, data: unknown, meta: StoredStateMeta): Promise { - groups.set(`${this.#prefix}:${site}:${key}`, { meta, data }); + groups.set(`${this.#prefix}:${site}:${key}`, { site, key, meta, data }); } async deleteState(site: string, key: string): Promise { groups.delete(`${this.#prefix}:${site}:${key}`); } + // What `epg cache` looks inside a store with: every key it holds, from the + // key each row keeps beside it rather than picked back out of its id — a + // site called `example.com:8080` has a colon of its own. + async *list(options?: { site?: string }): AsyncIterable { + for (const [id, row] of rows) { + if (this.#holds(id, row.key.site, options?.site)) { + yield { kind: 'entry', key: row.key }; + } + } + + for (const [id, group] of groups) { + if (this.#holds(id, group.site, options?.site)) { + yield { kind: 'state', site: group.site, key: group.key }; + } + } + } + async close(): Promise {} }