From a1ebee5b16f675bee958b6cf8734db86dadd342a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=BDubom=C3=ADr=20Jesze?= Date: Sat, 17 Oct 2026 20:06:19 +0200 Subject: [PATCH] feat(merge)!: texts two sources spell differently are written once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When two programmes merged, their titles, descriptions and categories were unioned by the exact text, within each language. Two sources rarely write one the same to the letter, so a guide kept every spelling: "Dj Mixes Premiere" beside "DJ Mixes premiere", "film" beside "Film", a description cut off with "…" beside the whole of it. homepi carried 120 lines of transform to undo it. `merge.union` says per field which texts are one, within a language, the higher-priority source's kept. Titles and sub-titles are the same by the match's own `normalizeTitle`, so a config's title rule loosens both what is one broadcast and what is one title. A description another contains as whole words, but for case, spacing and a trailing "…", is dropped, and the one containing it takes its place. Categories, keywords and countries are the same but for case, spacing, hyphens and underscores, as a profile finds a genre. A field takes 'exact', 'caseless', 'match', 'contained', `{ contained: fn }` or a key function of its own, and `union: 'exact'` sets every field exact. It applies as each programme is written, not as two are merged, so matching still sees every title and a later source's spelling still finds its broadcast. `mergeProgrammes`, `mergeInto` and `backfillInto` keep every text, as before; `mergeProgrammeLists` takes a union and applies it to what it returns, and `resolveUnion` settles one into a `ResolvedUnion`, whose `apply` writes one programme. A loaded config carries its union settled, with the union it was settled from as its own fields, and a misspelt field or mode is refused as the config loads, before anything is grabbed. `ResolvedMatch` carries the `normalizeTitle` it compares by, and `genreKey` folds Unicode forms together. BREAKING CHANGE: a guide built from several sources writes one of two texts that say the same thing rather than both, so it changes on upgrade: fewer titles, categories and descriptions, none of them rewritten. Set `merge.union: 'exact'` to write every text as the sources gave them, as before. --- docs/api.md | 18 +- docs/configuration.md | 43 +++- src/config.ts | 19 +- src/main.ts | 3 + src/merge/channel.ts | 2 +- src/merge/guide.ts | 38 +++- src/merge/main.ts | 3 +- src/merge/programme.ts | 112 +++------ src/merge/text.ts | 75 ++++++ src/merge/types.ts | 71 +++++- src/merge/union.ts | 356 +++++++++++++++++++++++++++++ src/xmltv/genres.ts | 7 +- src/xmltv/profile.ts | 7 +- test/config.test.ts | 33 +++ test/genres.test.ts | 2 + test/merge.test.ts | 451 ++++++++++++++++++++++++++++++++++++- test/profile.test.ts | 9 +- test/readme-examples.ts | 14 ++ test/serve-handler.test.ts | 21 ++ 19 files changed, 1183 insertions(+), 101 deletions(-) create mode 100644 src/merge/text.ts create mode 100644 src/merge/union.ts diff --git a/docs/api.md b/docs/api.md index 6f1692c..c279b50 100644 --- a/docs/api.md +++ b/docs/api.md @@ -549,7 +549,7 @@ are not re-exported at all. | Grabber | `grab`, `defineSiteConfig`, `defineStreamSiteConfig`, `defineXmltvSite`, `defineM3uSite`, `defineXtreamSite`, `defineSchedulesDirectSite`, `schedulesDirectAccount`, `defineCommandSite`, `defineTvGrabCommandSite`, `resolveChannels`, `resolveSites`, `channelsFromM3u`, `channelsFromChannelsXml`, `guideUrlsFromM3u`, `xtreamChannelExtras`, `xtreamProgrammeExtras`, `schedulesDirectChannelExtras`, `schedulesDirectProgrammeExtras`, `SCHEDULES_DIRECT_CHANNEL_ID`, `channelElement`, `siteHttp`, `kyWith`, `sitePacing`, `retryAfterMs`, `isUnchanged`, `UnchangedError` | | Site state | `SiteStateHandle`, `StateKey`, `TrackedMap`, `channelsMaxAgeMs`, `DEFAULT_CHANNELS_MAX_AGE_DAYS` — see [what a site remembers](#what-a-site-remembers-between-runs) | | Selection | `channelSelection`, `unmatched` — what `channels` means once `derived` is taken into account, and what it asked for that nothing produced | -| Merge | `mergeProgrammes`, `mergeProgrammeLists`, `mergeInto`, `backfillInto`, `resolveMatch`, `normalizeTitle`, `titlesMatch`, `DEFAULT_MATCH`, `generateGuide`, `writeGuide`, `defaultChannelInfo` | +| Merge | `mergeProgrammes`, `mergeProgrammeLists`, `mergeInto`, `backfillInto`, `resolveMatch`, `normalizeTitle`, `titlesMatch`, `DEFAULT_MATCH`, `resolveUnion`, `ResolvedUnion`, `DEFAULT_UNION`, `generateGuide`, `writeGuide`, `defaultChannelInfo` | Plus the types for all of the above (`EpgConfig`, `SiteConfig`, `GrabberChannel`, `XmltvProgramme`, `XmltvChannel`, `GuideServer`, …), and @@ -991,6 +991,7 @@ a `Map` that records which keys were touched so nothing untouched is rewritten. `generateGuide`, `writeGuide`, `mergeProgrammes`, `mergeProgrammeLists`, `mergeInto`, `backfillInto`, `resolveMatch`, `normalizeTitle`, `titlesMatch`, +`resolveUnion`, `ResolvedUnion`, `DEFAULT_UNION`, `DEFAULT_MATCH`, `DEFAULT_FILL_STOP_MS`*, `mergeChannels`*, `defaultChannelInfo`, `channelSelection`, `unmatched`. @@ -1003,15 +1004,24 @@ or when the name says so, `.gz` / `.br` / `.zst`. See costs on a real one. `generateGuide` yields the document itself, so a caller piping it somewhere compresses it however they like. -`mergeProgrammeLists(lists, strategy, match?, fillStopMs?)` combines per-site -lists in priority order; `match` is a `ProgrammeMatch` object, a predicate of +`mergeProgrammeLists(lists, strategy, match?, fillStopMs?, union?)` combines +per-site lists in priority order; `match` is a `ProgrammeMatch` object, a predicate of your own, or `{ windowMs, matches }` — a predicate with the reach it needs. A bare predicate is taken to reach any distance (`windowMs: Infinity`), so it is asked about every pair; giving it a window is what keeps a merge linear. See [what counts as the same broadcast](./configuration.md#what-counts-as-the-same-broadcast). `fillStopMs` is for the `backfill` strategy: how long a programme with no `stop` is taken to -cover, `DEFAULT_FILL_STOP_MS` (six hours) unless given. +cover, `DEFAULT_FILL_STOP_MS` (six hours) unless given. `union` is +`merge.union`, applied to each programme of the result — see [texts that say +the same thing](./configuration.md#texts-that-say-the-same-thing); the pieces +below it leave every text in, and +`resolveUnion(union, resolveMatch(match).normalizeTitle).apply(programme)` is +what applies it to one programme, its titles compared as the match compares +them. A loaded config carries its `merge.union` already settled, as a +`ResolvedUnion` — whose own fields are the union it was settled from, so a +copy of it is that union — which `resolveUnion` takes as it is for the same +match and settles again for another. `mergeInto(target, incoming, resolveMatch(match))` is the piece underneath `mergeProgrammeLists`: one source's list folded into a sorted accumulator, which is also how the guide diff --git a/docs/configuration.md b/docs/configuration.md index 13251a0..2d005e8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1632,7 +1632,7 @@ questions: **who grabs** the channel, and **how what was grabbed is combined**. - `first-wins` — one ``, programmes only from the first covering site. The others are still grabbed, so switching to a strategy that uses them costs no refetch - `keep-all` — no deduplication - `programmeStrategy` (for `merge-programmes`) - - `merge` (default) — programmes describing the same broadcast become one element; language-tagged fields (`title`, `desc`, `category`, …) are unioned by `(lang, value)` — grab the same channel from a Slovak and an English source and get both languages in one programme + - `merge` (default) — programmes describing the same broadcast become one element; language-tagged fields (`title`, `desc`, `category`, …) are unioned within each language — grab the same channel from a Slovak and an English source and get both languages in one programme, and two spellings of one Slovak title as one; see [texts that say the same thing](#texts-that-say-the-same-thing) - `concat` — keep all programmes sorted by start - `backfill` — the first covering site contributes everything it has, and a lower-priority one only what falls in a hole it left; nothing is combined — see [filling the gaps](#filling-the-gaps) @@ -1729,6 +1729,47 @@ custom matcher stops being quick — the window is what keeps the walk short. merge: { match: (a, b) => … } // correct, and linear in nothing ``` +### Texts that say the same thing + +Two programmes that are one broadcast become one element, and their titles, +descriptions and categories come together within each language. Two sources +rarely write them the same to the letter — `DJ Mixes premiere` and +`Dj Mixes Premiere`, `Film` and `film`, a teaser cut off with `…` beside the +whole of it — so the guide takes such texts for one and writes the +higher-priority source's: + +| field | default | the same when | +|---|---|---| +| `title`, `subTitle` | `'match'` | the match's `normalizeTitle` makes them the same — by default case, accents and spacing aside, so a title rule that loosens what is one broadcast loosens what is one title too | +| `desc` | `'contained'` | one contains the other as whole words, but for case, spacing and a trailing `…` — a teaser cut off with `…` may end part way through one; the one containing it stands where the other stood, so a fuller description from a lower-priority source replaces the teaser it completes, while `Krimi` beside `Krimiseriál o …` stays | +| `category`, `keyword`, `country` | `'caseless'` | they are the same but for case, spacing, hyphens and underscores — as a [profile](#output-profiles) finds a category's genre, so `Sci-Fi` and `Sci Fi` are one | + +`merge.union` sets any of them to another mode: `'exact'` for the texts as +the sources gave them, as every field had them before, any of the others, +`{ contained: fn }` to take a source's own suffix off a description before it +is compared as `'contained'` compares, or a function of your own, under which +two texts are one when it answers the same for both. `union: 'exact'` sets +every field exact at once. + +```ts +merge: { + union: { + // A source that signs off every description with its own note. + desc: { contained: (value) => value.replace(/\s*\(skryté titulky\)$/, '') }, + // Two sources that number seasons differently: `Priatelia I.` and `Priatelia`. + title: (value) => value.replace(/\s+[IVX]+\.?$/, '').toLowerCase(), + }, +} +``` + +It applies as each programme is written, whatever sources it came from, and +not as two are merged: matching decides what is one broadcast by its titles, +so it sees every one of them, and a later source's spelling still matches the +spelling a texts rule will leave out. Different languages are never compared +— a missing language and an empty one are the same — and nothing a programme +carries is rewritten: a text is kept or left out whole. A field or a mode it +does not know is refused as the config is loaded, before anything is grabbed. + ### Cleaning up the output Two defects arrive from almost every source, and the guide is in a position to diff --git a/src/config.ts b/src/config.ts index d46c101..140a174 100644 --- a/src/config.ts +++ b/src/config.ts @@ -13,6 +13,8 @@ import { type StageDefaults, } from './core/answers.js'; import { resolveTimeZone } from './core/days.js'; +import { resolveMatch } from './merge/programme.js'; +import { resolveUnion } from './merge/union.js'; import type { CompressionFormat, CompressionOptions } from './core/output.js'; import type { MissingAllowance } from './grabber/missing.js'; import type { EpgServeConfig } from './serve/config.js'; @@ -342,7 +344,22 @@ export async function resolveConfigSource( ): Promise { const config = typeof source === 'function' ? await source(...supplied) : source; - return { ...config, timeZone: resolveTimeZone(config.timeZone) }; + // `merge.union` here as the zone is, rather than where the merge reads it: a + // misspelt field found by `epg build` would be found after the whole grab. + // Settled with the config's own match, and carried settled for the merge. + const merge = + config.merge === undefined || config.merge === null + ? undefined + : { + ...config.merge, + union: resolveUnion(config.merge.union, resolveMatch(config.merge.match).normalizeTitle), + }; + + return { + ...config, + ...(merge === undefined ? {} : { merge }), + timeZone: resolveTimeZone(config.timeZone), + }; } /** Identity helper for type inference in `epg.config.ts` files. */ diff --git a/src/main.ts b/src/main.ts index c6e1a60..7684bd2 100644 --- a/src/main.ts +++ b/src/main.ts @@ -308,6 +308,9 @@ export { normalizeTitle, resolveMatch, titlesMatch, + DEFAULT_UNION, + ResolvedUnion, + resolveUnion, generateGuide, writeGuide, defaultChannelInfo, diff --git a/src/merge/channel.ts b/src/merge/channel.ts index 0bcd729..689e2bd 100644 --- a/src/merge/channel.ts +++ b/src/merge/channel.ts @@ -1,5 +1,5 @@ import type { XmltvChannel } from '../xmltv/types.js'; -import { textKey, unionBy, urlKey } from './programme.js'; +import { textKey, unionBy, urlKey } from './text.js'; /** * Merge two `` descriptions of the same channel from different diff --git a/src/merge/guide.ts b/src/merge/guide.ts index 669df0d..2accdcd 100644 --- a/src/merge/guide.ts +++ b/src/merge/guide.ts @@ -27,6 +27,7 @@ import { resolveMatch, type MergeWatch, } from './programme.js'; +import { resolveUnion } from './union.js'; import type { ChannelSource, RegistryEntry } from './registry.js'; import { channelSelection, configured, unmatched, unmatchedMessage } from './select.js'; import type { BuildGuideOptions, FillGapsContext, FillGapsOptions, GuideContext } from './types.js'; @@ -483,6 +484,28 @@ export async function* generateGuide(options: BuildGuideOptions): AsyncGenerator const listStrategy = channelStrategy === 'merge-programmes' ? programmeStrategy : 'concat'; const match = resolveMatch(options.merge?.match); + // Which texts of a programme say the same thing, applied as each is written. + // A loaded config's is settled already — refused, if it had to be, before + // anything was grabbed — and taken as it is; a caller's own is settled here. + const textUnion = resolveUnion(options.merge?.union, match.normalizeTitle); + /** + * A programme with its texts as the union leaves them, worked out once: one + * handed to a transform as `next` is finished itself straight after, and the + * union is asked of it once rather than twice. By identity, so one merged + * again in the meantime — held across midnight, and joined by the next day — + * is a programme of its own and asked anew. + */ + const unioned = new WeakMap(); + const written = (programme: XmltvProgramme): XmltvProgramme => { + let result = unioned.get(programme); + + if (result === undefined) { + result = textUnion.apply(programme); + unioned.set(programme, result); + } + + return result; + }; const { fillStop = true, clipOverlaps = true, @@ -496,10 +519,10 @@ export async function* generateGuide(options: BuildGuideOptions): AsyncGenerator ? undefined : ((typeof fillStop === 'object' ? fillStop.maxMs : undefined) ?? DEFAULT_FILL_STOP_MS); - // Checked rather than trusted, which nothing else in `merge` needs to be: a - // `blockMs` of zero or a NaN makes the loop that lays blocks out never reach - // the end of a gap, inside a generator a file is being written from. A wedged - // merge is worse than a rejected option. + // Checked rather than trusted, as `union` is: a `blockMs` of zero or a NaN + // makes the loop that lays blocks out never reach the end of a gap, inside a + // generator a file is being written from. A wedged merge is worse than a + // rejected option. const gaps = fillGaps === false ? undefined : resolveFillGaps(fillGaps === true ? {} : fillGaps); // Two entries share an `xmltvId` under `keep-all`, so each would fill the @@ -543,7 +566,7 @@ export async function* generateGuide(options: BuildGuideOptions): AsyncGenerator next: XmltvProgramme | undefined, xmltvId: string, ): XmltvProgramme | undefined => { - let result = programme; + let result = written(programme); if (next !== undefined) { const start = programme.start.getTime(); @@ -579,7 +602,10 @@ export async function* generateGuide(options: BuildGuideOptions): AsyncGenerator xmltvId, timeZone, ...mergeSays, - ...(next === undefined ? {} : { next }), + // With its texts as the union leaves them, as the guide will write + // them — unless it is the last of a day, held for the next one, which + // may yet merge more into it. + ...(next === undefined ? {} : { next: written(next) }), }) ?? undefined ); }; diff --git a/src/merge/main.ts b/src/merge/main.ts index f91eae7..339d877 100644 --- a/src/merge/main.ts +++ b/src/merge/main.ts @@ -5,10 +5,11 @@ export { mergeInto, mergeProgrammes, mergeProgrammeLists, - normalizeTitle, resolveMatch, titlesMatch, } from './programme.js'; +export { normalizeTitle } from './text.js'; +export { DEFAULT_UNION, ResolvedUnion, resolveUnion } from './union.js'; export type { BackfillOptions, ResolvedMatch } from './programme.js'; export { mergeChannels } from './channel.js'; export { generateGuide, writeGuide, defaultChannelInfo } from './guide.js'; diff --git a/src/merge/programme.ts b/src/merge/programme.ts index 9fa0691..8f9baf5 100644 --- a/src/merge/programme.ts +++ b/src/merge/programme.ts @@ -1,16 +1,13 @@ +import type { XmltvCredits, XmltvPersonValue, XmltvProgramme } from '../xmltv/types.js'; import type { - XmltvCredits, - XmltvPersonValue, - XmltvProgramme, - XmltvTextValue, - XmltvUrlValue, -} from '../xmltv/types.js'; -import type { + MergeUnion, ProgrammeMatch, ProgrammeMatcher, ProgrammeStrategy, WindowedMatcher, } from './types.js'; +import { normalizeTitle, textKey, unionBy, urlKey } from './text.js'; +import { resolveUnion } from './union.js'; /** * How two programmes are paired up when nothing says otherwise: five minutes @@ -23,46 +20,6 @@ export const DEFAULT_MATCH: Required = { normalizeTitle, }; -/** Union two optional arrays, keeping first occurrence per dedup key. */ -export function unionBy( - base: T[] | undefined, - extra: T[] | undefined, - key: (item: T) => string, -): T[] { - const result: T[] = []; - const seen = new Set(); - - // Iterate the two sources directly rather than spreading them into a - // throwaway concat array — this is the merge hot path. - for (let s = 0; s < 2; s++) { - const items = s === 0 ? base : extra; - - if (!items) { - continue; - } - - for (const item of items) { - const k = key(item); - - if (!seen.has(k)) { - seen.add(k); - result.push(item); - } - } - } - - return result; -} - -export function textKey(item: XmltvTextValue): string { - return `${item.lang ?? ''}|${item.value}`; -} - -/** Dedup key for `` values (string or object with `system`). */ -export function urlKey(item: XmltvUrlValue): string { - return typeof item === 'string' ? `|${item}` : `${item.system ?? ''}|${item.value}`; -} - function personKey(item: XmltvPersonValue): string { return typeof item === 'string' ? item : item.value; } @@ -244,25 +201,6 @@ export function mergeProgrammes(base: XmltvProgramme, extra: XmltvProgramme): Xm return merged; } -/** - * A title as it is compared: case folded, accents dropped and whitespace - * collapsed, so `Správy ` and `spravy` are one title. - * - * Accents go because the two sources being merged are very often one local - * feed and one international one, and the international one is the one that - * spells `Vecernicek` — while a source that drops accents rarely reinstates - * them for a different programme. It is a comparison only: what a programme - * carries is never rewritten. - */ -export function normalizeTitle(value: string): string { - return value - .normalize('NFKD') - .replace(/\p{M}+/gu, '') - .toLowerCase() - .replace(/\s+/g, ' ') - .trim(); -} - /** Do these two programmes share a title, in any language either offers? */ export function titlesMatch( a: XmltvProgramme, @@ -293,6 +231,12 @@ function duration(programme: XmltvProgramme): number | undefined { export interface ResolvedMatch { windowMs: number; matches: ProgrammeMatcher; + /** + * How it compares titles — a match's own `normalizeTitle`, or the default + * for a matcher of one's own, which says nothing about titles. What a + * `merge.union` of titles compares by too. + */ + normalizeTitle: (title: string) => string; } /** @@ -301,23 +245,31 @@ export interface ResolvedMatch { * pairs up, so every candidate is offered to it. */ export function resolveMatch( - match?: ProgrammeMatch | ProgrammeMatcher | WindowedMatcher, + match?: ProgrammeMatch | ProgrammeMatcher | WindowedMatcher | null, ): ResolvedMatch { if (typeof match === 'function') { // Everything, because a function that never said how far it looks might // look anywhere — and the cost of that is in `WindowedMatcher`, which is // how a matcher says. - return { windowMs: Number.POSITIVE_INFINITY, matches: match }; + return { windowMs: Number.POSITIVE_INFINITY, matches: match, normalizeTitle }; } - if (match !== undefined && 'matches' in match) { - return { windowMs: match.windowMs, matches: match.matches }; + if (match !== undefined && match !== null && 'matches' in match) { + return { windowMs: match.windowMs, matches: match.matches, normalizeTitle }; } - const { startToleranceMs, titles, normalizeTitle: normalize } = { ...DEFAULT_MATCH, ...match }; + const { + startToleranceMs, + titles, + normalizeTitle: normalize, + } = { + ...DEFAULT_MATCH, + ...match, + }; return { windowMs: startToleranceMs, + normalizeTitle: normalize, matches: (a, b) => { const shift = Math.abs(a.start.getTime() - b.start.getTime()); @@ -670,12 +622,22 @@ export function mergeProgrammeLists( strategy: ProgrammeStrategy, match?: ProgrammeMatch | ProgrammeMatcher | WindowedMatcher, fillStopMs: number | undefined = DEFAULT_FILL_STOP_MS, + /** + * Which texts of a programme say the same thing — see {@link MergeUnion}. + * Applied to each programme of the result, as the guide applies it to each + * one it writes. + */ + union?: MergeUnion | 'exact', ): XmltvProgramme[] { + const resolved = resolveMatch(match); + const texts = resolveUnion(union, resolved.normalizeTitle); + const written = (programmes: XmltvProgramme[]) => + programmes.map((programme) => texts.apply(programme)); + if (strategy === 'concat') { - return lists.flat().sort((a, b) => a.start.getTime() - b.start.getTime()); + return written(lists.flat().sort((a, b) => a.start.getTime() - b.start.getTime())); } - const resolved = resolveMatch(match); const merged: XmltvProgramme[] = []; // A branch of its own rather than falling through to the merge: they take the @@ -686,12 +648,12 @@ export function mergeProgrammeLists( backfillInto(merged, list, { match: resolved, fillStopMs }); } - return merged; + return written(merged); } for (const list of lists) { mergeInto(merged, list, resolved); } - return merged; + return written(merged); } diff --git a/src/merge/text.ts b/src/merge/text.ts new file mode 100644 index 0000000..501e507 --- /dev/null +++ b/src/merge/text.ts @@ -0,0 +1,75 @@ +/** + * Comparing and combining the texts a programme or a channel carries — leaf + * helpers, with nothing of the merge's own in them, so the merge's modules can + * share them without depending on each other. + */ + +import type { XmltvTextValue, XmltvUrlValue } from '../xmltv/types.js'; + +/** Union two optional arrays, keeping first occurrence per dedup key. */ +export function unionBy( + base: T[] | undefined, + extra: T[] | undefined, + key: (item: T) => string, +): T[] { + const result: T[] = []; + const seen = new Set(); + + // Iterate the two sources directly rather than spreading them into a + // throwaway concat array — this is the merge hot path. + for (let s = 0; s < 2; s++) { + const items = s === 0 ? base : extra; + + if (!items) { + continue; + } + + for (const item of items) { + const k = key(item); + + if (!seen.has(k)) { + seen.add(k); + result.push(item); + } + } + } + + return result; +} + +export function textKey(item: XmltvTextValue): string { + return `${item.lang ?? ''}|${item.value}`; +} + +/** Dedup key for `` values (string or object with `system`). */ +export function urlKey(item: XmltvUrlValue): string { + return typeof item === 'string' ? `|${item}` : `${item.system ?? ''}|${item.value}`; +} + +/** + * A title as it is compared: case folded, accents dropped and whitespace + * collapsed, so `Správy ` and `spravy` are one title. + * + * Accents go because the two sources being merged are very often one local + * feed and one international one, and the international one is the one that + * spells `Vecernicek` — while a source that drops accents rarely reinstates + * them for a different programme. It is a comparison only: what a programme + * carries is never rewritten. + */ +export function normalizeTitle(value: string): string { + return value + .normalize('NFKD') + .replace(/\p{M}+/gu, '') + .toLowerCase() + .replace(/\s+/g, ' ') + .trim(); +} + +/** + * A text as compared where its letters matter: in one Unicode form — a + * precomposed `á` and a decomposed one are one letter — with case and spacing + * aside. Accents stay, as {@link normalizeTitle} does not keep them. + */ +export function foldText(value: string): string { + return value.normalize('NFC').replace(/\s+/g, ' ').trim().toLowerCase(); +} diff --git a/src/merge/types.ts b/src/merge/types.ts index 4493738..e494db1 100644 --- a/src/merge/types.ts +++ b/src/merge/types.ts @@ -44,8 +44,9 @@ export type ChannelCoverage = 'all' | 'first' | CoverageRule; * (only relevant for `merge-programmes`). * * - `merge`: programmes with the same start time are merged into one — - * multi-language fields (title, desc, category, ...) are unioned by - * `(lang, value)`, scalar fields come from the highest-priority site. + * multi-language fields (title, desc, category, ...) are unioned within + * each language as {@link MergeUnion} says, scalar fields come from the + * highest-priority site. * - `concat`: all programmes are emitted, sorted by start time, without * deduplication. * - `backfill`: the highest-priority site contributes everything it has, and a @@ -248,6 +249,64 @@ export interface FillGapsContext { index: number; } +/** + * How one field's texts are told apart as a programme is written — within a + * language, the higher-priority source's text kept, and nothing rewritten: + * + * - `'exact'` — the texts as the sources gave them; only an identical text from + * two sources merged into one is ever one, as before. + * - `'caseless'` — the same but for case, spacing, hyphens and underscores, + * as a profile compares a category to a genre: "Film" and "film", "Sci-Fi" + * and "Sci Fi". + * - `'match'` — the same by the match's `normalizeTitle`, which by default + * folds case and accents and collapses spacing — "DJ Mixes premiere" and + * "Dj Mixes Premiere" — and which a config can loosen further: "Priatelia + * I." and "Priatelia" with one that drops a numeral. + * - `'contained'` — a text another one contains as whole words, compared but + * for case, spacing and a trailing "…", says nothing new and is dropped; the + * one containing it takes its place, so a fuller description from a + * lower-priority source stands where the teaser it completes stood. A teaser + * cut off with "…" may end part way through a word; "Krimi" beside + * "Krimiseriál o …" is not the same thing said again. + * - `{ contained: fn }` — the same, after `fn`: for a source that ends its + * descriptions with a suffix of its own to take off first. + * - a function — the same when it answers the same for both. + */ +export type TextUnion = + | 'exact' + | 'caseless' + | 'match' + | 'contained' + | { contained: (value: string) => string } + | ((value: string) => string); + +/** + * Which of each language-tagged field's texts the guide writes. A field left + * out has its default. + * + * Applied as each programme is written rather than as two are merged, so + * matching still sees every title: one a later source's matches is not gone + * because an earlier merge kept another spelling. + * + * ```ts + * merge: { union: { desc: { contained: (value) => value.replace(/\s*\(skryté titulky\)$/, '') } } } + * ``` + */ +export interface MergeUnion { + /** `'match'` unless said. */ + title?: TextUnion; + /** `'match'` unless said. */ + subTitle?: TextUnion; + /** `'contained'` unless said. */ + desc?: TextUnion; + /** `'caseless'` unless said. */ + category?: TextUnion; + /** `'caseless'` unless said. */ + keyword?: TextUnion; + /** `'caseless'` unless said. */ + country?: TextUnion; +} + export interface MergeOptions { channelStrategy?: ChannelStrategy; /** @@ -266,6 +325,14 @@ export interface MergeOptions { * describe. */ match?: ProgrammeMatch | ProgrammeMatcher | WindowedMatcher; + /** + * Which of a programme's texts say the same thing, field by field — see + * {@link MergeUnion}. By default a title by the match's `normalizeTitle`, a + * description another contains, and a category, keyword or country but for + * case, spacing, hyphens and underscores. `'exact'` on a field, or on its + * own for every field, is the old behaviour. + */ + union?: MergeUnion | 'exact'; /** * Give a programme with no `stop` the next one's start, capped at * {@link FillStopOptions.maxMs} (six hours). On by default. diff --git a/src/merge/union.ts b/src/merge/union.ts new file mode 100644 index 0000000..8e3e9d0 --- /dev/null +++ b/src/merge/union.ts @@ -0,0 +1,356 @@ +/** + * Which of a programme's texts say the same thing, and so which one the guide + * keeps. + * + * Two sources rarely write a title, a category or a description the same way + * to the letter — "DJ Mixes premiere" and "Dj Mixes Premiere", "Film" and + * "film", a teaser cut off with "…" beside the whole of it — and a programme + * merged from both carries every spelling, so a guide shows two titles and a + * description twice. + * + * Applied as each programme is written, not as two are merged. Merging decides + * which programmes are one broadcast by their titles, and a title dropped at + * the first merge would be one a third source's could no longer match: the + * merge keeps every text, so matching sees all of them, and the guide writes + * one of each. + */ + +import { genreKey } from '../xmltv/genres.js'; +import type { XmltvProgramme, XmltvTextValue } from '../xmltv/types.js'; +import { foldText, normalizeTitle as matchTitle, unionBy } from './text.js'; +import type { MergeUnion, TextUnion } from './types.js'; + +/** The fields a {@link MergeUnion} speaks for. */ +type UnionField = keyof MergeUnion; + +/** One field's texts, as the guide writes them — the same array where none is dropped. */ +type FieldUnion = (texts: XmltvTextValue[]) => XmltvTextValue[]; + +const FIELDS: readonly UnionField[] = [ + 'title', + 'subTitle', + 'desc', + 'category', + 'keyword', + 'country', +]; + +/** + * What each field does when a config says nothing: a title or a sub-title is + * the same as another by the match's own comparison, a description another + * contains says nothing new, and a category, keyword or country is the same + * but for case, spacing, hyphens and underscores. `'exact'` on a field is what + * every field did before. + */ +export const DEFAULT_UNION: Readonly> = Object.freeze({ + title: 'match', + subTitle: 'match', + desc: 'contained', + category: 'caseless', + keyword: 'caseless', + country: 'caseless', +}); + +/** + * What marks a settled union, holding the title comparison it was settled with + * — by the global registry, so a union another copy of this package settled is + * taken for one too, as two installs side by side would otherwise not. + * + * Not enumerable, so a copy — a spread, `structuredClone`, a deep clone — does + * not take it along: a copy has the fields a union was settled from and none + * of what it was settled into, and is a union to settle again. + */ +const SETTLED = Symbol.for('epg-tools.ResolvedUnion'); + +/** + * `merge.union`, settled: one union per field, and the match it was settled + * with — what {@link resolveUnion} makes, and what a loaded config carries in + * `merge.union` in place of what it was given. + * + * Its own fields are the union it was settled from, so it is a + * {@link MergeUnion} itself: a fingerprint of the options reads what it says, a + * spread of it is that union, and `merge.union`'s type stays what a config + * writes. What it was settled into is its own, behind {@link apply}. + */ +export class ResolvedUnion implements MergeUnion { + /** One of these, from this copy of the package or another — see {@link SETTLED}. */ + static [Symbol.hasInstance](value: unknown): value is ResolvedUnion { + return ( + typeof value === 'object' && + value !== null && + typeof (value as { [SETTLED]?: unknown })[SETTLED] === 'function' + ); + } + + declare readonly title?: TextUnion; + declare readonly subTitle?: TextUnion; + declare readonly desc?: TextUnion; + declare readonly category?: TextUnion; + declare readonly keyword?: TextUnion; + declare readonly country?: TextUnion; + readonly #fields: Readonly>; + + /** + * Settle a union, checked: a field or a mode it does not know is refused + * rather than read as the default — a misspelt one would otherwise keep + * every duplicate it was set to remove, or remove what it was set to keep, + * and nothing would say so. `'exact'` on its own is every field exact. + */ + constructor( + given: MergeUnion | 'exact' = {}, + normalizeTitle: (title: string) => string = matchTitle, + ) { + const union = read(given); + const fields = {} as Record; + + for (const field of FIELDS) { + const mode = union[field]; + + if (mode !== undefined) { + (this as { -readonly [F in UnionField]?: TextUnion })[field] = mode; + } + + // Left out is the default; anything else said, `null` included, is a + // mode to read — and refused when it is not one. + fields[field] = fieldUnion( + mode === undefined ? DEFAULT_UNION[field] : mode, + normalizeTitle, + field, + ); + } + + Object.defineProperty(this, SETTLED, { value: normalizeTitle }); + this.#fields = fields; + } + + /** + * A programme with its texts as the union says, field by field — the same + * programme where nothing changes, which is nearly always: a field with + * fewer than two texts has nothing to compare. + */ + apply(programme: XmltvProgramme): XmltvProgramme { + let result = programme; + + for (const field of FIELDS) { + const texts = programme[field]; + + if (texts === undefined || texts.length < 2) { + continue; + } + + const kept = this.#fields[field](texts); + + if (kept !== texts) { + result = result === programme ? { ...programme } : result; + result[field] = kept as XmltvProgramme['title']; + } + } + + return result; + } +} + +/** + * Settle `merge.union` into something to write with, for a match's + * `normalizeTitle`: what decides two programmes are one broadcast also decides + * their titles are one title, so a config that loosens one loosens both. + * + * One already settled for the same comparison — a loaded config's — is taken + * as it is; for another, it is settled again from the union it was settled + * from. + */ +export function resolveUnion( + union: MergeUnion | 'exact' | undefined = {}, + normalizeTitle: (title: string) => string = matchTitle, +): ResolvedUnion { + return union instanceof ResolvedUnion && + (union as { [SETTLED]?: unknown })[SETTLED] === normalizeTitle + ? union + : new ResolvedUnion(union, normalizeTitle); +} + +/** A union as given, checked for being one — a settled one by its own fields. */ +function read(given: unknown): MergeUnion { + if (given === 'exact') { + return Object.fromEntries(FIELDS.map((field) => [field, 'exact'])); + } + + const prototype: unknown = + typeof given === 'object' && given !== null ? Object.getPrototypeOf(given) : undefined; + + // A plain object, or a settled union read by the fields it was settled from. + // A `Map`, or an instance of another class, has no fields to read, and + // reading it as none would be the default unasked. + if (!(given instanceof ResolvedUnion) && prototype !== Object.prototype && prototype !== null) { + throw new TypeError( + `merge.union must be 'exact' or an object of fields — ${FIELDS.join(', ')} ` + + `(got ${described(given)})`, + ); + } + + const union = given as Record; + const unknown = Object.keys(union).filter((field) => !FIELDS.includes(field as UnionField)); + + if (unknown.length > 0) { + throw new TypeError( + `merge.union has no field ${unknown.map((field) => JSON.stringify(field)).join(', ')}: ` + + `its fields are ${FIELDS.join(', ')}`, + ); + } + + return Object.fromEntries( + FIELDS.map((field) => [field, union[field]]).filter(([, mode]) => mode !== undefined), + ); +} + +/** A value as an error can name it, whatever it is. */ +function described(value: unknown): string { + if (typeof value === 'function') { + return 'a function'; + } + + if (typeof value === 'object' && value !== null) { + const prototype: unknown = Object.getPrototypeOf(value); + + if (!Array.isArray(value) && prototype !== Object.prototype && prototype !== null) { + const name = (value as { constructor?: { name?: string } }).constructor?.name; + + return name ? `an instance of ${name}` : 'an object of a class'; + } + } + + try { + return JSON.stringify(value) ?? String(value); + } catch { + return Object.prototype.toString.call(value); + } +} + +/** One field's mode as a union. */ +function fieldUnion( + mode: TextUnion, + normalizeTitle: (title: string) => string, + field: UnionField, +): FieldUnion { + if (typeof mode === 'function') { + return byKey(mode); + } + + if (typeof mode === 'object' && mode !== null && typeof mode.contained === 'function') { + // The site's own comparison first — a suffix of its own taken off — and + // then the built-in one, so case, spacing and a teaser's "…" still count. + return byContainment((value) => comparable(mode.contained(value))); + } + + switch (mode) { + case 'exact': + // The texts as the sources gave them: merging two programmes already + // keeps one of two identical ones, and nothing more is asked of these. + return (texts) => texts; + case 'caseless': + return byKey(genreKey); + case 'match': + return byKey(normalizeTitle); + case 'contained': + return byContainment(comparable); + default: + throw new TypeError( + `merge.union.${field} must be 'exact', 'caseless', 'match', 'contained', ` + + `{ contained: fn } or a function (got ${described(mode)})`, + ); + } +} + +/** + * The language a text is in: a missing one and an empty one alike, and a tag in + * any case, as BCP 47 has them — `sk` and `SK` are one language. + */ +function language(text: XmltvTextValue): string { + return (text.lang ?? '').toLowerCase(); +} + +/** Texts the same by `key`, within a language: the first of them kept. */ +function byKey(key: (value: string) => string): FieldUnion { + return (texts) => { + const kept = unionBy(texts, undefined, (text) => `${language(text)}\0${key(text.value)}`); + + return kept.length === texts.length ? texts : kept; + }; +} + +/** A text as `contained` compares it, and whether it was cut off. */ +interface Comparable { + key: string; + /** It ended in "…", so it may stop part way through a word. */ + teaser: boolean; +} + +/** + * Texts said by another, within a language: one whose comparable form another + * contains, as whole words, is dropped, and the one containing it takes its + * place — so a fuller description from a lower-priority source stands where + * the teaser it completes stood, and priority order holds. Of two the same, the + * first stays. One that compares as nothing at all is dropped beside any other. + * + * Whole words, because a short text inside a longer one is often not the same + * thing said again: "Krimi" is not "Krimiseriál o vyšetrovateľovi". Only a + * teaser — cut off with "…" — may end part way through one. + */ +function byContainment(compare: (value: string) => Comparable): FieldUnion { + return (texts) => { + const compared = texts.map((text) => compare(text.value)); + const languages = texts.map(language); + const covers = (j: number, i: number): boolean => + j !== i && + languages[j] === languages[i] && + contains(compared[j]!.key, compared[i]!) && + (compared[j]!.key !== compared[i]!.key || j < i); + const kept = texts.map((_, i) => !texts.some((__, j) => covers(j, i))); + const order = new Set(); + + for (const i of texts.keys()) { + // Where a dropped one stood, the first kept text that covers it — once. + const at = kept[i] ? i : texts.findIndex((_, j) => kept[j] && covers(j, i)); + + if (at >= 0) { + order.add(at); + } + } + + return order.size === texts.length ? texts : [...order].map((i) => texts[i]!); + }; +} + +/** A letter or a digit — what a word is made of. */ +const WORD = /[\p{L}\p{N}]/u; + +/** Whether `inner` is in `outer` as whole words, its end free where it is a teaser. */ +function contains(outer: string, inner: Comparable): boolean { + if (inner.key === '') { + return true; + } + + for (let at = outer.indexOf(inner.key); at !== -1; at = outer.indexOf(inner.key, at + 1)) { + const end = at + inner.key.length; + const starts = at === 0 || !WORD.test(outer[at - 1]!); + const ends = inner.teaser || end === outer.length || !WORD.test(outer[end]!); + + if (starts && ends) { + return true; + } + } + + return false; +} + +/** + * A description as compared for `contained`: folded as text is, with a + * teaser's trailing "…" or "..." taken off, so the cut-off one is found inside + * the whole. + */ +function comparable(value: string): Comparable { + const folded = foldText(value); + const key = folded.replace(/(?:\.{3}|…)$/, '').trim(); + + return { key, teaser: key !== folded }; +} diff --git a/src/xmltv/genres.ts b/src/xmltv/genres.ts index 7470700..07e489f 100644 --- a/src/xmltv/genres.ts +++ b/src/xmltv/genres.ts @@ -473,7 +473,12 @@ export const DVB_GENRES: readonly Genre[] = [ * `'sci-fi'` in a hand-written table matches a source writing `Sci Fi`. */ export function genreKey(value: string): string { - return value.toLowerCase().replace(/[\s_-]+/gu, ''); + // In one Unicode form first: a source writing `Dráma` decomposed means the + // genre a precomposed one names. + return value + .normalize('NFC') + .toLowerCase() + .replace(/[\s_-]+/gu, ''); } /** Built once, at module load, since a guide asks this per category. */ diff --git a/src/xmltv/profile.ts b/src/xmltv/profile.ts index 88efff7..e23137a 100644 --- a/src/xmltv/profile.ts +++ b/src/xmltv/profile.ts @@ -11,9 +11,10 @@ * All of it applies as the document is **written**, never to what was grabbed. * One cache therefore serves every consumer, and changing profile costs no * refetch — the same property that makes `--no-extensions` cheap. It is also - * the only safe place for a genre code: the merge dedupes categories on - * `(lang, value)` and ignores their attributes, so an `eit` attached earlier - * would be discarded depending on which source happened to be the base. + * the only safe place for a genre code: the merge keeps one of two categories + * that are the same text and ignores their attributes, so an `eit` attached + * earlier would be discarded depending on which source happened to be the + * base. */ import { diff --git a/test/config.test.ts b/test/config.test.ts index 089cba9..2d35d6e 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; import { defineConfig, resolveConfigSource, type EpgConfig } from '../src/config.js'; +import { ResolvedUnion } from '../src/merge/union.js'; import { createConfigContext, defaultsReader, @@ -201,4 +202,36 @@ describe('a config as it is loaded', () => { 'timeZone must be an IANA time zone such as "Europe/Bratislava" (got "Mars/Olympus")', ); }); + + it('carries merge.union settled, with its own match, for the merge to take', async () => { + const loaded = await resolveConfigSource({ + ...plain, + merge: { + union: { title: 'match' }, + match: { normalizeTitle: (title) => title.replace(/\s+[IVX]+\.?$/, '') }, + }, + }); + const union = loaded.merge?.union; + + expect(union).toBeInstanceOf(ResolvedUnion); + expect( + (union as ResolvedUnion).apply({ + channel: 'X', + start: new Date(0), + title: [{ value: 'Priatelia I.' }, { value: 'Priatelia' }], + }).title, + ).toEqual([{ value: 'Priatelia I.' }]); + }); + + it('loads a config whose merge is null, as everything that reads it does', async () => { + const loaded = await resolveConfigSource({ ...plain, merge: null as never }); + + expect(loaded.merge).toBeNull(); + }); + + it('refuses a merge.union it cannot read, before a grab would run', async () => { + await expect( + resolveConfigSource({ ...plain, merge: { union: { desc: 'containd' as never } } }), + ).rejects.toThrow("merge.union.desc must be 'exact', 'caseless', 'match', 'contained'"); + }); }); diff --git a/test/genres.test.ts b/test/genres.test.ts index 3a7882c..4f00d79 100644 --- a/test/genres.test.ts +++ b/test/genres.test.ts @@ -98,6 +98,8 @@ describe('genreOf', () => { // Which is also why only one spelling of these needs to be listed. expect(genreOf('Sci-Fi')?.eit).toBe('0x13'); + // Written decomposed, as some sources write their accents: one genre still. + expect(genreKey('Dráma'.normalize('NFD'))).toBe(genreKey('Dráma')); expect(genreOf('SciFi')?.eit).toBe('0x13'); expect(genreOf('sci fi')?.eit).toBe('0x13'); }); diff --git a/test/merge.test.ts b/test/merge.test.ts index 22c1776..223960b 100644 --- a/test/merge.test.ts +++ b/test/merge.test.ts @@ -7,9 +7,11 @@ import type { CacheStore, ChannelDayKey } from '../src/cache/types.js'; import type { EpgEvent } from '../src/core/events.js'; import type { GrabberChannel, SiteConfig } from '../src/grabber/types.js'; import { mergeChannels } from '../src/merge/channel.js'; -import { mergeProgrammeLists, mergeProgrammes } from '../src/merge/programme.js'; +import { mergeProgrammeLists, mergeProgrammes, resolveMatch } from '../src/merge/programme.js'; +import { normalizeTitle } from '../src/merge/text.js'; +import { DEFAULT_UNION, ResolvedUnion, resolveUnion } from '../src/merge/union.js'; import { parseXmltvDate } from '../src/xmltv/main.js'; -import type { BuildGuideOptions } from '../src/merge/types.js'; +import type { BuildGuideOptions, MergeUnion } from '../src/merge/types.js'; import type { XmltvChannel, XmltvProgramme } from '../src/xmltv/types.js'; import { collect as collectEvents } from './reporting.js'; @@ -265,6 +267,372 @@ describe('mergeProgrammes', () => { }); }); +describe('merge.union', () => { + const at = new Date('2026-01-15T20:00:00Z'); + /** Two programmes for one broadcast, carrying `base` and `extra`'s texts. */ + const merge = ( + base: Partial, + extra: Partial, + union?: MergeUnion | 'exact', + ): XmltvProgramme => + resolveUnion(union).apply( + mergeProgrammes( + { channel: 'X', start: at, title: [{ value: 'A' }], ...base }, + { channel: 'X', start: at, title: [{ value: 'A' }], ...extra }, + ), + ); + const values = (texts: { value: string }[] | undefined) => texts?.map(({ value }) => value); + + it('takes titles the same by the match’s comparison for one, the first kept', () => { + const merged = merge( + { title: [{ value: 'Správy', lang: 'sk' }] }, + { + title: [ + { value: 'SPRAVY ', lang: 'sk' }, + { value: 'News', lang: 'en' }, + ], + }, + ); + + expect(merged.title).toEqual([ + { value: 'Správy', lang: 'sk' }, + { value: 'News', lang: 'en' }, + ]); + }); + + it('takes categories, keywords and countries the same but for case for one', () => { + const merged = merge( + { category: [{ value: 'Film' }], keyword: [{ value: 'Drama' }], country: [{ value: 'SK' }] }, + { category: [{ value: 'film' }], keyword: [{ value: 'drama ' }], country: [{ value: 'sk' }] }, + ); + + expect([values(merged.category), values(merged.keyword), values(merged.country)]).toEqual([ + ['Film'], + ['Drama'], + ['SK'], + ]); + }); + + it('drops a description another contains, the fuller one taking its place', () => { + const merged = merge( + { + desc: [ + { value: 'Príbeh o láske…', lang: 'sk' }, + { value: 'A story of love.', lang: 'en' }, + ], + }, + { + desc: [ + { value: 'Príbeh o láske a vojne na Slovensku.', lang: 'sk' }, + // In another language, so nothing to compare it with. + { value: 'Príbeh o láske', lang: 'cs' }, + ], + }, + ); + + expect(values(merged.desc)).toEqual([ + 'Príbeh o láske a vojne na Slovensku.', + 'A story of love.', + 'Príbeh o láske', + ]); + }); + + it('keeps every spelling a field is set to exact on', () => { + const merged = merge( + { category: [{ value: 'Film' }] }, + { category: [{ value: 'film' }] }, + { category: 'exact' }, + ); + + expect(values(merged.category)).toEqual(['Film', 'film']); + }); + + it('compares as a function of its own says', () => { + const merged = merge( + { title: [{ value: 'Priatelia I.' }] }, + { title: [{ value: 'Priatelia' }] }, + { title: (value) => value.replace(/\s+[IVX]+\.?$/, '') }, + ); + + expect(values(merged.title)).toEqual(['Priatelia I.']); + }); + + it('finds one description inside another as a function of its own compares them', () => { + const merged = merge( + { desc: [{ value: 'Správy dňa. (skryté titulky)' }] }, + { desc: [{ value: 'Správy dňa. Počasie a šport.' }] }, + { desc: { contained: (value) => value.replace(/\s*\(skryté titulky\)$/, '') } }, + ); + + expect(values(merged.desc)).toEqual(['Správy dňa. Počasie a šport.']); + }); + + it('takes titles for one by a match’s own normalizeTitle', () => { + const loose = (title: string) => title.toLowerCase().replace(/[^\p{L}]+/gu, ''); + const [merged] = mergeProgrammeLists( + [ + [prog('X', '2026-01-15T20:00:00Z', 'DJ Mixes premiere')], + [prog('X', '2026-01-15T20:00:00Z', 'Dj-Mixes Premiere!')], + ], + 'merge', + { normalizeTitle: loose }, + ); + + expect(values(merged?.title)).toEqual(['DJ Mixes premiere']); + }); + + it('applies to a source’s own rows for one broadcast, as configured', () => { + // Two rows at one instant in one source's list are pooled into one programme. + const [pooled] = mergeProgrammeLists( + [ + [ + prog('X', '2026-01-15T20:00:00Z', 'Film', undefined, { category: [{ value: 'Film' }] }), + prog('X', '2026-01-15T20:00:00Z', 'Film', undefined, { category: [{ value: 'film' }] }), + ], + ], + 'merge', + undefined, + undefined, + { category: 'exact' }, + ); + + expect(values(pooled?.category)).toEqual(['Film', 'film']); + }); + + it('leaves a numeral the default comparison keeps', () => { + // Folding case, accents and spacing is all the default does; a numeral + // takes a looser normalizeTitle, or a title function of its own. + const merged = merge( + { title: [{ value: 'Priatelia I.' }] }, + { title: [{ value: 'Priatelia' }] }, + ); + + expect(values(merged.title)).toEqual(['Priatelia I.', 'Priatelia']); + }); + + it('still merges a third source whose title only the dropped spelling matches', () => { + // A title rule looser than the match: had the second spelling been dropped + // as A and B merged, C's shifted start would have had nothing to match. + const merged = mergeProgrammeLists( + [ + [prog('X', '2026-01-15T20:00:00Z', 'Priatelia I.')], + [prog('X', '2026-01-15T20:00:00Z', 'Priatelia')], + [prog('X', '2026-01-15T20:03:00Z', 'Priatelia')], + ], + 'merge', + undefined, + undefined, + { title: (value) => value.replace(/\s+[IVX]+\.?$/, '') }, + ); + + expect(merged.map((programme) => values(programme.title))).toEqual([['Priatelia I.']]); + }); + + it('compares descriptions as contained does after a function of its own', () => { + const merged = merge( + { desc: [{ value: 'Správy dňa…' }] }, + { desc: [{ value: 'SPRÁVY DŇA. Počasie a šport. (skryté titulky)' }] }, + { desc: { contained: (value) => value.replace(/\s*\(skryté titulky\)$/, '') } }, + ); + + expect(values(merged.desc)).toEqual(['SPRÁVY DŇA. Počasie a šport. (skryté titulky)']); + }); + + it('takes a text with no language and one with an empty one as in one language', () => { + const merged = merge( + { desc: [{ value: 'Príbeh o láske…', lang: '' }] }, + { desc: [{ value: 'Príbeh o láske a vojne.' }] }, + ); + + expect(values(merged.desc)).toEqual(['Príbeh o láske a vojne.']); + }); + + it('compares letters written in either Unicode form as one', () => { + const merged = merge( + { category: [{ value: 'Dráma'.normalize('NFC') }] }, + { category: [{ value: 'Dráma'.normalize('NFD') }] }, + ); + + expect(merged.category).toHaveLength(1); + }); + + it('takes categories one genre is for one, as a profile does', () => { + const merged = merge({ category: [{ value: 'Sci-Fi' }] }, { category: [{ value: 'Sci Fi' }] }); + + expect(values(merged.category)).toEqual(['Sci-Fi']); + }); + + it('keeps every spelling of every field under exact on its own', () => { + const merged = merge( + { title: [{ value: 'Správy' }], category: [{ value: 'Film' }] }, + { title: [{ value: 'správy' }], category: [{ value: 'film' }] }, + 'exact', + ); + + expect([values(merged.title), values(merged.category)]).toEqual([ + ['Správy', 'správy'], + ['Film', 'film'], + ]); + }); + + it('refuses a field it does not have, and a union that is no object', () => { + expect(() => resolveUnion({ description: 'exact' } as never)).toThrow( + 'merge.union has no field "description": its fields are title, subTitle, desc, category, keyword, country', + ); + expect(() => resolveUnion(null as never)).toThrow("merge.union must be 'exact' or an object"); + }); + + it('takes one already settled as it is, for the match it was settled with', () => { + const loose = (title: string) => title.replace(/\s+[IVX]+\.?$/, ''); + const settled = resolveUnion({ title: 'match' }, loose); + + expect(resolveUnion(settled, loose)).toBe(settled); + expect( + settled.apply({ + ...prog('X', '2026-01-15T20:00:00Z', 'Priatelia I.'), + title: [{ value: 'Priatelia I.' }, { value: 'Priatelia' }], + }).title, + ).toEqual([{ value: 'Priatelia I.' }]); + }); + + it('takes its titles by a match of options, and by the default for a matcher of its own', () => { + const own = (title: string) => title; + + expect(resolveMatch({ normalizeTitle: own }).normalizeTitle).toBe(own); + expect(resolveMatch(() => true).normalizeTitle).toBe(normalizeTitle); + expect(resolveMatch({ windowMs: 1, matches: () => true }).normalizeTitle).toBe(normalizeTitle); + }); + + it('settles one already settled again for a match that has changed since', () => { + const settled = resolveUnion({ title: 'match' }); + const loose = (title: string) => title.replace(/\s+[IVX]+\.?$/, ''); + const again = resolveUnion(settled, loose); + + expect(again).not.toBe(settled); + expect( + again.apply({ + ...prog('X', '2026-01-15T20:00:00Z', 'Priatelia I.'), + title: [{ value: 'Priatelia I.' }, { value: 'Priatelia' }], + }).title, + ).toEqual([{ value: 'Priatelia I.' }]); + }); + + it('takes one settled by another copy of the package for one', () => { + // What another install's would look like from here: its own class, the + // same registered brand holding its comparison, and the fields it was + // settled from as its own. + class Theirs { + readonly desc = 'exact'; + + apply(programme: XmltvProgramme): XmltvProgramme { + return programme; + } + } + + const theirs = Object.defineProperty(new Theirs(), Symbol.for('epg-tools.ResolvedUnion'), { + value: normalizeTitle, + }); + + expect(theirs instanceof ResolvedUnion).toBe(true); + expect(resolveUnion(theirs as never)).toBe(theirs); + // Settled again for another comparison, from its own fields. + expect( + resolveUnion(theirs as never, (title) => title).apply({ + ...prog('X', '2026-01-15T20:00:00Z', 'A'), + desc: [{ value: 'Príbeh…' }, { value: 'Príbeh o láske.' }], + }).desc, + ).toHaveLength(2); + }); + + it('takes a copy of a settled one for the union it was settled from', () => { + const settled = resolveUnion({ category: 'exact' }); + const programme = { + ...prog('X', '2026-01-15T20:00:00Z', 'A'), + category: [{ value: 'Film' }, { value: 'film' }], + desc: [{ value: 'Príbeh…' }, { value: 'Príbeh o láske.' }], + }; + + // A spread keeps what it was settled from and none of what it was settled into. + const changed = resolveUnion({ ...settled, desc: 'exact' }); + + expect(values(changed.apply(programme).category)).toEqual(['Film', 'film']); + expect(values(changed.apply(programme).desc)).toEqual(['Príbeh…', 'Príbeh o láske.']); + expect(resolveUnion(structuredClone({ ...settled })).apply(programme).category).toHaveLength(2); + }); + + it('refuses what a union cannot be, naming it by what it is', () => { + expect(() => resolveUnion(new Map([['desc', 'exact']]) as never)).toThrow( + '(got an instance of Map)', + ); + expect(() => new ResolvedUnion({ description: 'exact' } as never)).toThrow( + 'merge.union has no field "description"', + ); + }); + + it('keeps its defaults as they are, whoever holds them', () => { + expect(Object.isFrozen(DEFAULT_UNION)).toBe(true); + }); + + it('keeps a short description that is a word of a longer one rather than the same', () => { + const merged = merge( + { desc: [{ value: 'Krimi' }] }, + { desc: [{ value: 'Krimiseriál o vyšetrovateľovi.' }] }, + ); + + expect(values(merged.desc)).toEqual(['Krimi', 'Krimiseriál o vyšetrovateľovi.']); + }); + + it('takes a teaser cut off part way through a word for the whole of it', () => { + const merged = merge( + { desc: [{ value: 'Príbeh o lás…' }] }, + { desc: [{ value: 'Príbeh o láske a vojne.' }] }, + ); + + expect(values(merged.desc)).toEqual(['Príbeh o láske a vojne.']); + }); + + it('leaves texts as the sources gave them under exact, a source’s own repeats included', () => { + const programme = { + ...prog('X', '2026-01-15T20:00:00Z', 'Film'), + category: [{ value: 'Film' }, { value: 'Film' }], + }; + + expect(resolveUnion('exact').apply(programme)).toBe(programme); + }); + + it('takes a language tag in any case for one language', () => { + const merged = merge( + { title: [{ value: 'Správy', lang: 'sk' }] }, + { title: [{ value: 'SPRÁVY', lang: 'SK' }] }, + ); + + expect(merged.title).toEqual([{ value: 'Správy', lang: 'sk' }]); + }); + + it('refuses a mode said as null, and names what it got whatever it is', () => { + expect(() => resolveUnion({ desc: null as never })).toThrow( + "merge.union.desc must be 'exact', 'caseless', 'match', 'contained', { contained: fn } or a function (got null)", + ); + expect(() => resolveUnion(((value: string) => value) as never)).toThrow('(got a function)'); + }); + + it('merges lists under concat with no match to speak of', () => { + const merged = mergeProgrammeLists( + [[prog('X', '2026-01-15T20:00:00Z', 'A')], [prog('X', '2026-01-15T19:00:00Z', 'B')]], + 'concat', + null as never, + ); + + expect(merged.map((programme) => values(programme.title))).toEqual([['B'], ['A']]); + }); + + it('refuses a mode it does not know, naming the field', () => { + expect(() => resolveUnion({ desc: 'containd' as never })).toThrow( + "merge.union.desc must be 'exact', 'caseless', 'match', 'contained', { contained: fn } or a function (got \"containd\")", + ); + }); +}); + describe('mergeChannels', () => { it('unions display names, icons and urls with base first', () => { const base: XmltvChannel = { @@ -659,6 +1027,85 @@ describe.skipIf(!xmltvReady)('generateGuide', () => { return collect(generateGuide(options)); } + it('hands a transform the next programme as it will be written', async () => { + const cache = createFakeCache({ + [`site-a.sk|X|${DAY}`]: [ + prog('X', '2026-01-15T10:00:00Z', 'Správy', 'sk'), + prog('X', '2026-01-15T11:00:00Z', 'Film', 'sk', { + category: [{ value: 'Film' }, { value: 'film' }], + }), + ], + }); + const seen: unknown[] = []; + + await generate({ + sites: [siteA], + cache, + days: 1, + startDay: DAY, + now: NOW, + merge: { + transform: (programme, { next }) => { + if (next) seen.push(next.category); + + return programme; + }, + }, + }); + + expect(seen).toEqual([[{ value: 'Film' }]]); + }); + + it('takes them for one under backfill, and across the day boundary', async () => { + const DAY2 = '2026-01-16'; + // One source, a broadcast across midnight in both days' entries, with its + // category spelt two ways: merged as the days meet, written once. + const cache = createFakeCache({ + [`site-a.sk|X|${DAY}`]: [ + prog('X', '2026-01-15T23:00:00Z', 'Film', 'sk', { category: [{ value: 'Film' }] }), + ], + [`site-a.sk|X|${DAY2}`]: [ + prog('X', '2026-01-15T23:00:00Z', 'Film', 'sk', { category: [{ value: 'film' }] }), + ], + }); + const categories = async (programmeStrategy: 'merge' | 'backfill') => + [ + ...( + await generate({ + sites: [siteA], + cache, + days: 2, + startDay: DAY, + now: NOW, + merge: { programmeStrategy }, + }) + ).matchAll(/]*>([^<]*)<\/category>/g), + ].map((found) => found[1]); + + expect(await categories('merge')).toEqual(['Film']); + expect(await categories('backfill')).toEqual(['Film']); + }); + + it('takes texts two sites spell differently for one, as merge.union says', async () => { + const cache = createFakeCache({ + [`site-a.sk|X|${DAY}`]: [ + prog('X', '2026-01-15T10:00:00Z', 'Správy', 'sk', { category: [{ value: 'News' }] }), + ], + [`site-b.com|X|${DAY}`]: [ + prog('X', '2026-01-15T10:00:00Z', 'Správy', 'sk', { category: [{ value: 'news' }] }), + ], + }); + const categories = async (merge: NonNullable) => + [ + ...( + await generate({ sites: [siteA, siteB], cache, days: 1, startDay: DAY, now: NOW, merge }) + ).matchAll(/]*>([^<]*)<\/category>/g), + ].map((found) => found[1]); + + expect(await categories({})).toEqual(['News']); + expect(await categories({ union: { category: 'exact' } })).toEqual(['News', 'news']); + }); + describe('dropContainers', () => { const site = makeSite('site-a.sk', [{ xmltvId: 'X', siteId: 'a-x' }]); diff --git a/test/profile.test.ts b/test/profile.test.ts index 01bee16..b3e1a27 100644 --- a/test/profile.test.ts +++ b/test/profile.test.ts @@ -264,8 +264,8 @@ describe('categories', () => { }); it('says the language it rewrote into, rather than keeping a false one', () => { - // A `lang="de"` left on English text is a lie the merge would believe: its - // dedup key is `(lang, value)`. + // A `lang="de"` left on English text is a lie the merge would believe: it + // compares texts within a language. expect(categories({ categories: true }, [{ value: 'Film', lang: 'de' }])).toEqual([ { value: 'Movie / Drama', lang: 'en' }, ]); @@ -280,8 +280,9 @@ describe('categories', () => { }); it('emits a genre once when two categories canonicalise onto it', () => { - // The merge deduped on `(lang, value)` before any of this ran, so the - // rewrite has to dedupe its own output. + // The merge took two categories for one only within a language and by + // their words, before any of this ran — `Movie` and `Film` are two to it — + // so the rewrite has to dedupe its own output. expect( categories({ categories: true }, [ { value: 'Movie', lang: 'en' }, diff --git a/test/readme-examples.ts b/test/readme-examples.ts index d98c03d..423c687 100644 --- a/test/readme-examples.ts +++ b/test/readme-examples.ts @@ -929,6 +929,20 @@ export const coveredPerChannel = defineConfig({ }, }); +// --- docs/configuration.md: Texts that say the same thing ------------------ +export const unionedTexts = defineConfig({ + sites: [example], + output: 'public/epg.xml', + merge: { + union: { + // A source that signs off every description with its own note. + desc: { contained: (value) => value.replace(/\s*\(skryté titulky\)$/, '') }, + // Two sources that number seasons differently: `Priatelia I.` and `Priatelia`. + title: (value) => value.replace(/\s+[IVX]+\.?$/, '').toLowerCase(), + }, + }, +}); + // --- docs/site-config.md: A source that is not UTF-8 ---------------------- export const notUtf8 = defineSiteConfig({ site: 'cp1250.example.tv', diff --git a/test/serve-handler.test.ts b/test/serve-handler.test.ts index 32d611b..9d2bf5b 100644 --- a/test/serve-handler.test.ts +++ b/test/serve-handler.test.ts @@ -92,6 +92,27 @@ async function handlerFor( return handler; } +describe('a guide’s validators and its merge.union', () => { + it('change when the union does, though it is carried settled', async () => { + const cache = await cacheWith({ one: [programme('one', 6)] }); + const etag = async (merge: EpgConfig['merge']) => { + const handler = await handlerFor( + { ...configFor(['one']), ...(merge ? { merge } : {}) }, + cache, + ); + + return (await handler.answer({ headers: {}, route: 'guide' })).headers['etag']; + }; + + const categories = await etag({ union: { category: 'exact' } }); + + // Two unions a server would write differently from, which a fingerprint of + // the settled union's own keys could not tell apart. + expect(await etag({ union: { desc: 'exact' } })).not.toBe(categories); + expect(await etag({ union: { category: 'exact' } })).toBe(categories); + }); +}); + /** An app of somebody's own: a few routes, one of which is the guide. */ async function mount(handler: GuideHandler, routes: (handler: GuideHandler) => Server) { const server = routes(handler);