From 23945c0b2227fdbe9207bcfcb40f479043b20f85 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:14:38 -0400 Subject: [PATCH 1/2] perf: paginate website catalog and load search data on demand --- README.md | 2 + docs/developerSetup.md | 6 +- gh-pages-template/assets/js/app.js | 130 +++++++---- gh-pages-template/assets/js/catalog.js | 192 ++++++++++++++++ gh-pages-template/assets/js/search-worker.js | 19 ++ gh-pages-template/index.html | 8 +- package.json | 2 +- src/build-site.js | 2 + src/site-catalog.js | 67 ++++++ tests/build-site.test.js | 4 + tests/presets.test.js | 8 +- tests/site-browser.test.js | 193 ++++++++++++++++ tests/site-catalog.test.js | 227 +++++++++++++++++++ 13 files changed, 809 insertions(+), 51 deletions(-) create mode 100644 gh-pages-template/assets/js/catalog.js create mode 100644 gh-pages-template/assets/js/search-worker.js create mode 100644 src/site-catalog.js create mode 100644 tests/site-browser.test.js create mode 100644 tests/site-catalog.test.js diff --git a/README.md b/README.md index 890ce80b48b..2c121cc83d4 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ Each GitHub issue requests **one preset**. Game requests must identify a [GameDB The [website](https://app.lizardbyte.dev/PresetDB/) shows games and apps with available presets. Each preset shows its launch method and host details as badges, its launch command, and a Sunshine application JSON export. The exported application name is the game or app name. Replace supported path placeholders with paths on your host. Review community commands before running them. Game cover images come from GameDB; app images are submitted as HTTPS URLs for the catalog. These image URLs are not used as Sunshine `image-path` values, which require local files. +Browsing shows 24 games and apps per page. Type at least three characters to search for any part of a name, then use the type and host OS filters to narrow the results. The website loads a small catalog manifest and only the chunks needed for the current page. Search runs in a web worker using a static substring index generated during the Pages build; it fetches only the relevant search files and the current page's summaries. Launch details load when you select an item or open a direct link. No search service is required. + The published JSON API contains `index.json`, `games/.json`, and `apps/.json`. GameDB uses the same numeric game IDs as IGDB. An entry contains all its presets. Issue-backed presets receive the originating issue number as a stable string ID, such as "4". GameDB-imported Steam presets use "steam" and have no issue history. Records use schema version 2 with a `command` field or a `commands_by_os` map and an optional `working_directory`; the website builds Sunshine JSON from these generic fields. The migration workflow updates version 1 records on the database branch; site and approval reads also convert them during the rollout. ## Database growth diff --git a/docs/developerSetup.md b/docs/developerSetup.md index 427c5e6fb67..b85ad6cc6e6 100644 --- a/docs/developerSetup.md +++ b/docs/developerSetup.md @@ -22,7 +22,11 @@ Build the static site from the local database layout with: node src/build-site.js --database database --output site-build ``` -The site builder writes `index.json`, `stats.json`, two SVG contribution charts, and game and app records. It uses the `gh-pages-template` directory as the Jekyll source. The Pages workflow packages that output for the shared LizardByte Jekyll workflow. +The site builder writes `index.json`, `stats.json`, two SVG contribution charts, game and app records, and a paged catalog under `catalog/`. It uses the `gh-pages-template` directory as the Jekyll source. The Pages workflow packages that output for the shared LizardByte Jekyll workflow. + +The browser reads `catalog/manifest.json` instead of the complete `index.json` API. The manifest contains counts for each type/OS filter and the sizes of the generated chunks. Catalog files live under a content-derived revision so browser caches cannot mix ordinals from different builds. Summary chunks contain 96 items; browse chunks contain 240 sorted item ordinals for a particular filter. The worker returns only the selected page of 24 summaries and retains at most 16 fetched chunks plus the most recent search result. + +Search requires three characters and preserves case-insensitive substring matching, including punctuation and Unicode. At build time, each consecutive three-character sequence maps to a sorted list of item ordinals and character positions. For a query, the worker loads enough sequences to cover every character and checks their relative positions, preventing matches assembled from unrelated parts of a name. Type and OS flags in the postings allow filtering and counting without loading other item summaries. Search and summary downloads use at most four concurrent requests; new requests cancel superseded work. Typing is debounced by 250 milliseconds. Existing `index.json`, `games/.json`, and `apps/.json` API endpoints remain available. Read the Docs pull request previews use `.readthedocs.yaml` and the shared `readthedocs_build.sh` script. Connect the repository to Read the Docs, enable pull request builds, and set these project environment variables: diff --git a/gh-pages-template/assets/js/app.js b/gh-pages-template/assets/js/app.js index e55d1ee5548..92d4d6a6dd3 100644 --- a/gh-pages-template/assets/js/app.js +++ b/gh-pages-template/assets/js/app.js @@ -5,18 +5,6 @@ const METHOD_NAMES = { gog: 'GOG', 'microsoft-store': 'Microsoft Store', emulator: 'Emulator' }; -function filterItems(index, query, kind, os) { - const needle = query.trim().toLowerCase(); - return [ - ...(index.games || []).map(item => ({ ...item, kind: 'game' })), - ...(index.apps || []).map(item => ({ ...item, kind: 'app' })) - ].filter(item => - (kind === 'all' || item.kind === kind) && - (os === 'all' || item.operating_systems.includes(os)) && - item.name.toLowerCase().includes(needle) - ).sort((a, b) => a.name.localeCompare(b.name)); -} - function commandForOs(preset, os = 'Windows') { const command = preset.commands_by_os?.[os] || preset.command; if (!command) throw new Error('Choose an available host OS'); @@ -203,13 +191,31 @@ function boot() { const list = document.getElementById('preset-list'); const status = document.getElementById('preset-status'); const detail = document.getElementById('preset-detail'); - let index; + const previous = document.getElementById('preset-previous'); + const next = document.getElementById('preset-next'); + const pageStatus = document.getElementById('preset-page'); + let page = 0; + let requestId = 0; + let debounce; + let detailController; + let detailId = 0; + let worker; - function renderList() { - if (!index) return; - const items = filterItems(index, search.value, kind.value, os.value); + function renderList(result) { + const { items, total, page_count: pageCount } = result; + page = result.page; list.replaceChildren(); - status.textContent = items.length ? `${items.length} games and apps found` : 'No matching presets yet.'; + list.setAttribute('aria-busy', 'false'); + previous.disabled = page === 0; + next.disabled = page + 1 >= pageCount; + pageStatus.textContent = pageCount ? `Page ${(page + 1).toLocaleString()} of ${pageCount.toLocaleString()}` : ''; + if (result.min_query_length) { + status.textContent = `Enter at least ${result.min_query_length} characters to search, or clear the search to browse.`; + } else { + status.textContent = total + ? `${total.toLocaleString()} games and apps found · Showing ${(result.start + 1).toLocaleString()}–${(result.start + items.length).toLocaleString()}` + : 'No matching presets yet.'; + } for (const item of items) { const column = element('div', 'col'); const button = element('button', 'card h-100 w-100 text-start border-0 shadow-sm rounded-0 p-4'); @@ -234,13 +240,18 @@ function boot() { } async function showRecord(item) { + detailController?.abort(); + detailController = new AbortController(); + const controller = detailController; + const id = ++detailId; detail.hidden = false; detail.replaceChildren(element('p', '', 'Loading launch options…')); try { const folder = item.kind === 'game' ? 'games' : 'apps'; - const response = await fetch(`${base}/${folder}/${encodeURIComponent(item.id)}.json`); + const response = await fetch(`${base}/${folder}/${encodeURIComponent(item.id)}.json`, { signal: controller.signal }); if (!response.ok) throw new Error(`HTTP ${response.status}`); const record = await response.json(); + if (id !== detailId) return; const heading = element('h2', 'fw-bold', record.name); detail.replaceChildren(heading, safeLink(record.source_url, item.kind === 'game' ? 'View on IGDB ↗' : 'Official app site ↗')); const image = safeImage(record.image_url); @@ -251,37 +262,72 @@ function boot() { detail.append(presets); detail.scrollIntoView({ behavior: 'smooth', block: 'start' }); } catch (error) { + if (controller.signal.aborted || id !== detailId) return; detail.replaceChildren(element('p', 'text-danger', `Could not load presets: ${error.message}`)); } } - async function load() { - try { - const response = await fetch(`${base}/index.json`); - if (!response.ok) throw new Error(`HTTP ${response.status}`); - index = await response.json(); - renderList(); - const params = new URLSearchParams(globalThis.location.search); - const requestedKind = params.get('kind'); - const requestedId = params.get('id'); - if (requestedId && ['game', 'app'].includes(requestedKind)) { - const item = filterItems(index, '', requestedKind, 'all').find(entry => String(entry.id) === requestedId); - if (item) showRecord(item); - } - } catch (error) { - status.textContent = `Could not load the preset index: ${error.message}`; + function loadRequestedRecord() { + const params = new URLSearchParams(globalThis.location.search); + const requestedKind = params.get('kind'); + const requestedId = params.get('id'); + if (requestedId && ['game', 'app'].includes(requestedKind)) { + showRecord({ kind: requestedKind, id: requestedId }); + } else { + detailController?.abort(); + detailId++; + detail.hidden = true; } } - for (const control of [search, kind, os]) control.addEventListener('input', renderList); - globalThis.addEventListener('popstate', () => { - const params = new URLSearchParams(globalThis.location.search); - const item = index && filterItems(index, '', params.get('kind'), 'all').find(entry => String(entry.id) === params.get('id')); - if (item) showRecord(item); - else detail.hidden = true; - }); - load(); + function loading() { + previous.disabled = true; + next.disabled = true; + pageStatus.textContent = ''; + status.textContent = 'Loading presets…'; + list.setAttribute('aria-busy', 'true'); + } + + function requestPage() { + clearTimeout(debounce); + loading(); + worker.postMessage({ id: ++requestId, base, query: search.value, kind: kind.value, os: os.value, page }); + } + + function catalogError(message) { + list.setAttribute('aria-busy', 'false'); + status.textContent = `Could not load the preset catalog: ${message}. Reload the page to try again.`; + } + + try { + worker = new Worker(`${base}/assets/js/search-worker.js`); + worker.addEventListener('message', event => { + if (event.data.id !== requestId) return; + if (event.data.error) catalogError(event.data.error); + else renderList(event.data.result); + }); + worker.addEventListener('error', () => catalogError('Search worker unavailable')); + search.addEventListener('input', () => { + page = 0; + requestId++; + loading(); + clearTimeout(debounce); + debounce = setTimeout(requestPage, 250); + }); + for (const control of [kind, os]) control.addEventListener('input', () => { + page = 0; + requestPage(); + }); + previous.addEventListener('click', () => { page--; requestPage(); }); + next.addEventListener('click', () => { page++; requestPage(); }); + requestPage(); + } catch (error) { + catalogError(error.message); + } + + globalThis.addEventListener('popstate', loadRequestedRecord); + loadRequestedRecord(); } if (typeof document !== 'undefined') document.addEventListener('DOMContentLoaded', boot); -if (typeof module !== 'undefined') module.exports = { filterItems, commandForOs, sunshineSnippet, renderPresetCard, browseMethodBadges, normalizeBasePath }; +if (typeof module !== 'undefined') module.exports = { boot, commandForOs, sunshineSnippet, renderPresetCard, browseMethodBadges, normalizeBasePath }; diff --git a/gh-pages-template/assets/js/catalog.js b/gh-pages-template/assets/js/catalog.js new file mode 100644 index 00000000000..3d27fab59c0 --- /dev/null +++ b/gh-pages-template/assets/js/catalog.js @@ -0,0 +1,192 @@ +'use strict'; + +const PAGE_SIZE = 24; +const CATALOG_CHUNK_SIZE = 96; +const BROWSE_CHUNK_SIZE = 240; +const MIN_QUERY_LENGTH = 3; +const OS_FLAGS = { Windows: 2, Linux: 4, macOS: 8 }; + +function normalizedName(value) { + return String(value).trim().toLowerCase(); +} + +function gramKey(gram) { + return Array.from(gram, character => character.codePointAt(0).toString(16).padStart(6, '0')).join(''); +} + +function itemFlags(item) { + return (item.kind === 'game' ? 1 : 0) | + item.operating_systems.reduce((flags, os) => flags | (OS_FLAGS[os] || 0), 0); +} + +function matchesFlags(code, kind, os) { + const flags = code % 16; + return (kind === 'all' || Boolean(flags & 1) === (kind === 'game')) && + (os === 'all' || Boolean(flags & OS_FLAGS[os])); +} + +// Non-overlapping grams plus the final gram cover every query character. The +// stored positions make their intersection an exact substring search. +function queryGrams(query) { + const characters = Array.from(query); + const grams = []; + for (let offset = 0; offset <= characters.length - 3; offset += 3) { + grams.push({ key: gramKey(characters.slice(offset, offset + 3).join('')), offset }); + } + const lastOffset = characters.length - 3; + if (grams.length && grams.at(-1).offset !== lastOffset) { + grams.push({ key: gramKey(characters.slice(lastOffset).join('')), offset: lastOffset }); + } + return grams; +} + +function containsPosition(postings, code, position) { + let low = 0; + let high = postings.length / 2; + while (low < high) { + const middle = Math.floor((low + high) / 2); + const item = postings[middle * 2]; + const offset = postings[middle * 2 + 1]; + if (item < code || (item === code && offset < position)) low = middle + 1; + else high = middle; + } + return postings[low * 2] === code && postings[low * 2 + 1] === position; +} + +function searchPositions(grams, postings, kind, os) { + const ordered = [...grams].sort((a, b) => postings.get(a.key).length - postings.get(b.key).length); + const anchor = ordered[0]; + const candidates = postings.get(anchor.key); + const results = []; + for (let i = 0; i < candidates.length; i += 2) { + const code = candidates[i]; + const ordinal = Math.floor(code / 16); + const start = candidates[i + 1] - anchor.offset; + if (start < 0 || results.at(-1) === ordinal || !matchesFlags(code, kind, os)) continue; + if (ordered.every(gram => containsPosition(postings.get(gram.key), code, start + gram.offset))) { + results.push(ordinal); + } + } + return results; +} + +class CatalogClient { + constructor(base, fetcher = globalThis.fetch.bind(globalThis)) { + this.base = base; + this.fetcher = fetcher; + this.cache = new Map(); + } + + async manifest(signal) { + if (this.metadata) return this.metadata; + const response = await this.fetcher(`${this.base}/catalog/manifest.json`, { signal, cache: 'no-cache' }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + const manifest = await response.json(); + if (manifest.schema_version !== 1 || !/^[a-f0-9]{16}$/.test(manifest.revision)) { + throw new Error('Unsupported catalog version'); + } + signal?.throwIfAborted(); + this.metadata = manifest; + return manifest; + } + + async json(file, signal, missingIsEmpty = false) { + if (this.cache.has(file)) { + const value = this.cache.get(file); + this.cache.delete(file); + this.cache.set(file, value); + return value; + } + const response = await this.fetcher(`${this.base}/catalog/${this.metadata.revision}/${file}`, { signal }); + let value; + if (missingIsEmpty && response.status === 404) value = []; + else { + if (!response.ok) throw new Error(`HTTP ${response.status}`); + value = await response.json(); + } + signal?.throwIfAborted(); + this.cache.set(file, value); + // Bound retained data even when someone searches or browses many pages. + if (this.cache.size > 16) this.cache.delete(this.cache.keys().next().value); + return value; + } + + async search(query, kind, os, signal) { + const grams = queryGrams(query); + const keys = [...new Set(grams.map(gram => gram.key))]; + const postings = new Map(); + let next = 0; + let empty = false; + const loadNext = async () => { + while (next < keys.length && !empty) { + signal?.throwIfAborted(); + const key = keys[next++]; + const values = await this.json(`search/${key}.json`, signal, true); + postings.set(key, values); + if (!values.length) empty = true; + } + }; + await Promise.all(Array.from({ length: Math.min(4, keys.length) }, loadNext)); + return empty ? [] : searchPositions(grams, postings, kind, os); + } + + async items(ordinals, signal) { + const size = this.metadata.catalog_chunk_size; + const chunks = [...new Set(ordinals.map(ordinal => Math.floor(ordinal / size)))]; + const loaded = new Map(); + let next = 0; + const loadNext = async () => { + while (next < chunks.length) { + signal?.throwIfAborted(); + const chunk = chunks[next++]; + loaded.set(chunk, await this.json(`items/${chunk}.json`, signal)); + } + }; + await Promise.all(Array.from({ length: Math.min(4, chunks.length) }, loadNext)); + return ordinals.map(ordinal => loaded.get(Math.floor(ordinal / size))[ordinal % size]); + } + + async page({ query = '', kind = 'all', os = 'all', page = 0 } = {}, signal) { + const manifest = await this.manifest(signal); + if (!['all', 'game', 'app'].includes(kind) || !['all', ...Object.keys(OS_FLAGS)].includes(os)) { + throw new Error('Invalid catalog filter'); + } + const needle = normalizedName(query); + if (needle && Array.from(needle).length < manifest.min_query_length) { + return { items: [], total: 0, page: 0, page_count: 0, min_query_length: manifest.min_query_length }; + } + let matches = null; + if (needle) { + const key = JSON.stringify([needle, kind, os]); + if (this.lastSearch?.key === key) matches = this.lastSearch.matches; + else { + matches = await this.search(needle, kind, os, signal); + signal?.throwIfAborted(); + this.lastSearch = { key, matches }; + } + } + const total = matches ? matches.length : manifest.counts[kind][os]; + const pageCount = Math.ceil(total / manifest.page_size); + const selectedPage = Math.min(Math.max(Number.isSafeInteger(page) ? page : 0, 0), Math.max(pageCount - 1, 0)); + const start = selectedPage * manifest.page_size; + const length = Math.min(manifest.page_size, total - start); + let ordinals; + if (matches) ordinals = matches.slice(start, start + length); + else if (kind === 'all' && os === 'all') { + ordinals = Array.from({ length }, (_, i) => start + i); + } else if (length) { + const chunk = Math.floor(start / manifest.browse_chunk_size); + const values = await this.json(`browse/${kind}/${os}/${chunk}.json`, signal); + const offset = start % manifest.browse_chunk_size; + ordinals = values.slice(offset, offset + length); + } else ordinals = []; + return { items: await this.items(ordinals, signal), total, start, page: selectedPage, page_count: pageCount }; + } +} + +const catalogApi = { + PAGE_SIZE, CATALOG_CHUNK_SIZE, BROWSE_CHUNK_SIZE, MIN_QUERY_LENGTH, + normalizedName, gramKey, itemFlags, CatalogClient +}; +if (typeof module !== 'undefined') module.exports = catalogApi; +else globalThis.PresetCatalog = catalogApi; diff --git a/gh-pages-template/assets/js/search-worker.js b/gh-pages-template/assets/js/search-worker.js new file mode 100644 index 00000000000..aaf2dbf1e64 --- /dev/null +++ b/gh-pages-template/assets/js/search-worker.js @@ -0,0 +1,19 @@ +'use strict'; + +importScripts('catalog.js'); + +let catalog; +let current; +globalThis.addEventListener('message', async event => { + current?.abort(); + const controller = new AbortController(); + current = controller; + const { id, base, ...options } = event.data; + if (!catalog || catalog.base !== base) catalog = new PresetCatalog.CatalogClient(base); + try { + const result = await catalog.page(options, controller.signal); + if (!controller.signal.aborted) globalThis.postMessage({ id, result }); + } catch (error) { + if (!controller.signal.aborted) globalThis.postMessage({ id, error: error.message }); + } +}); diff --git a/gh-pages-template/index.html b/gh-pages-template/index.html index 0b711f7d089..445430049e5 100644 --- a/gh-pages-template/index.html +++ b/gh-pages-template/index.html @@ -28,7 +28,8 @@

Browse presets

- + +
Enter at least 3 characters to search.
@@ -50,6 +51,11 @@

Browse presets

Loading presets…
+ diff --git a/package.json b/package.json index 9b56e8cdff2..64167907111 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ }, "scripts": { "test": "node --test tests/*.test.js", - "lint": "node --check src/presets.js && node --check src/record.js && node --check src/migrate-database.js && node --check src/database.js && node --check src/issue.js && node --check src/issue-comment.js && node --check src/gamedb-sync.js && node --check src/build-site.js && node --check src/statistics.js && node --check src/approval-queue.js && node --check src/comment-command.js && node --check src/verify-approval.js && node --check src/workflow-queue.js && node --check gh-pages-template/assets/js/app.js", + "lint": "node --check src/presets.js && node --check src/record.js && node --check src/migrate-database.js && node --check src/database.js && node --check src/issue.js && node --check src/issue-comment.js && node --check src/gamedb-sync.js && node --check src/build-site.js && node --check src/site-catalog.js && node --check src/statistics.js && node --check src/approval-queue.js && node --check src/comment-command.js && node --check src/verify-approval.js && node --check src/workflow-queue.js && node --check gh-pages-template/assets/js/app.js && node --check gh-pages-template/assets/js/catalog.js && node --check gh-pages-template/assets/js/search-worker.js", "test:ci": "node --test --experimental-test-coverage --test-reporter=junit --test-reporter=lcov --test-reporter-destination=junit.xml --test-reporter-destination=lcov.info tests/*.test.js" } } diff --git a/src/build-site.js b/src/build-site.js index 2f31333c4c8..547e511b2c2 100644 --- a/src/build-site.js +++ b/src/build-site.js @@ -4,6 +4,7 @@ const fs = require('node:fs'); const path = require('node:path'); const { buildStatistics } = require('./statistics'); const { normalizeRecord } = require('./record'); +const { buildCatalog } = require('./site-catalog'); function supportedOs(preset) { if (preset.os) return [preset.os]; @@ -130,6 +131,7 @@ async function buildSite(database, template, output, fetcher = globalThis.fetch, fs.writeFileSync(path.join(output, folder, `${item.id}.json`), JSON.stringify(item, null, 2) + '\n'); } fs.writeFileSync(path.join(output, 'index.json'), `${JSON.stringify(index, null, 2)}\n`); + buildCatalog(index, output); const statistics = buildStatistics(index, records); fs.writeFileSync(path.join(output, 'stats.json'), `${JSON.stringify(statistics.data, null, 2)}\n`); fs.writeFileSync(path.join(output, 'top_contributors.svg'), statistics.contributorsSvg); diff --git a/src/site-catalog.js b/src/site-catalog.js new file mode 100644 index 00000000000..0ca3c609169 --- /dev/null +++ b/src/site-catalog.js @@ -0,0 +1,67 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { createHash } = require('node:crypto'); +const { + PAGE_SIZE, CATALOG_CHUNK_SIZE, BROWSE_CHUNK_SIZE, MIN_QUERY_LENGTH, + normalizedName, gramKey, itemFlags +} = require('../gh-pages-template/assets/js/catalog'); + +function writeJson(root, file, value) { + const target = path.join(root, file); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, JSON.stringify(value) + '\n'); +} + +function writeChunks(root, folder, values, size) { + for (let start = 0; start < values.length; start += size) { + writeJson(root, `${folder}/${start / size}.json`, values.slice(start, start + size)); + } +} + +function buildCatalog(index, output) { + const items = [ + ...index.games.map(item => ({ ...item, kind: 'game' })), + ...index.apps.map(item => ({ ...item, kind: 'app' })) + ].sort((a, b) => a.name.localeCompare(b.name) || a.kind.localeCompare(b.kind) || + String(a.id).localeCompare(String(b.id))); + const manifest = { + schema_version: 1, page_size: PAGE_SIZE, catalog_chunk_size: CATALOG_CHUNK_SIZE, + browse_chunk_size: BROWSE_CHUNK_SIZE, min_query_length: MIN_QUERY_LENGTH, counts: {} + }; + manifest.revision = createHash('sha256').update(JSON.stringify({ manifest, items })).digest('hex').slice(0, 16); + const root = path.join(output, 'catalog', manifest.revision); + writeChunks(root, 'items', items, CATALOG_CHUNK_SIZE); + for (const kind of ['all', 'game', 'app']) { + manifest.counts[kind] = {}; + for (const os of ['all', 'Windows', 'Linux', 'macOS']) { + const ordinals = []; + for (let ordinal = 0; ordinal < items.length; ordinal++) { + const item = items[ordinal]; + if ((kind === 'all' || item.kind === kind) && + (os === 'all' || item.operating_systems.includes(os))) ordinals.push(ordinal); + } + manifest.counts[kind][os] = ordinals.length; + if (kind !== 'all' || os !== 'all') { + writeChunks(root, `browse/${kind}/${os}`, ordinals, BROWSE_CHUNK_SIZE); + } + } + } + const postings = new Map(); + for (let ordinal = 0; ordinal < items.length; ordinal++) { + const item = items[ordinal]; + const characters = Array.from(normalizedName(item.name)); + const code = ordinal * 16 + itemFlags(item); + for (let offset = 0; offset <= characters.length - MIN_QUERY_LENGTH; offset++) { + const key = gramKey(characters.slice(offset, offset + MIN_QUERY_LENGTH).join('')); + if (!postings.has(key)) postings.set(key, []); + postings.get(key).push(code, offset); + } + } + for (const [key, values] of postings) writeJson(root, `search/${key}.json`, values); + writeJson(output, 'catalog/manifest.json', manifest); + return manifest; +} + +module.exports = { buildCatalog }; diff --git a/tests/build-site.test.js b/tests/build-site.test.js index 38cda09f4b8..ca72a0e9a0c 100644 --- a/tests/build-site.test.js +++ b/tests/build-site.test.js @@ -69,6 +69,10 @@ test('PR cache-only builds keep the full database and stale ratings with no netw assert.equal(preset(output, 5).protondb, undefined); assert.equal(json(path.join(output, 'apps', 'app-one.json')).presets[0].command, 'app-one'); assert.equal(json(path.join(output, 'stats.json')).preset_count, 6); + assert.equal(json(path.join(output, 'catalog/manifest.json')).counts.all.all, 6); + for (const file of ['catalog.js', 'search-worker.js']) { + assert.ok(fs.existsSync(path.join(output, 'assets/js', file))); + } assert.equal(fs.readFileSync(cacheFile, 'utf8'), before); assert.equal(fs.readFileSync(path.join(database, 'games', '1.json'), 'utf8'), source); }); diff --git a/tests/presets.test.js b/tests/presets.test.js index da4f9853e82..5ed92a5d7a6 100644 --- a/tests/presets.test.js +++ b/tests/presets.test.js @@ -9,7 +9,7 @@ const { parseIssue, validateFields, validateGameDb, resolveIgdbSlug } = require( const { mergePreset } = require('../src/database'); const { processIssue } = require('../src/issue'); const { buildSite } = require('../src/build-site'); -const { filterItems, commandForOs, sunshineSnippet, normalizeBasePath } = require('../gh-pages-template/assets/js/app'); +const { commandForOs, sunshineSnippet, normalizeBasePath } = require('../gh-pages-template/assets/js/app'); const credentials = { clientId: 'client-id', clientSecret: 'client-secret' }; const gameValues = { @@ -241,11 +241,7 @@ test('issue processing and site build publish both game and app JSON', async t = assert.ok(!fs.readFileSync(path.join(output, 'index.json'), 'utf8').includes('{{ROM_PATH}}')); }); -test('site filters and serializes Sunshine application JSON', () => { - const index = { games: [{ id: 1, name: 'Halo', operating_systems: ['Windows'], preset_count: 2 }], - apps: [{ id: 'app', name: 'Media', operating_systems: ['Linux'], preset_count: 1 }] }; - assert.deepEqual(filterItems(index, 'hal', 'game', 'Windows').map(item => item.name), ['Halo']); - assert.equal(filterItems(index, '', 'all', 'macOS').length, 0); +test('site serializes Sunshine application JSON and supports preview base paths', () => { assert.equal(commandForOs({ command: 'game.exe' }), 'game.exe'); assert.deepEqual(JSON.parse(sunshineSnippet({ name: 'Halo', command: 'game.exe', working_directory: 'C:\\Games' })), { name: 'Halo', cmd: 'game.exe', 'working-dir': 'C:\\Games' }); diff --git a/tests/site-browser.test.js b/tests/site-browser.test.js new file mode 100644 index 00000000000..bc7344ae90f --- /dev/null +++ b/tests/site-browser.test.js @@ -0,0 +1,193 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { boot } = require('../gh-pages-template/assets/js/app'); + +class FakeElement { + constructor(tagName) { + this.tagName = tagName; + this.children = []; + this.handlers = {}; + this.attributes = {}; + this.textContent = ''; + this.style = {}; + } + + append(...nodes) { this.children.push(...nodes); } + prepend(...nodes) { this.children.unshift(...nodes); } + replaceChildren(...nodes) { this.children = nodes; } + addEventListener(name, handler) { this.handlers[name] = handler; } + setAttribute(name, value) { this.attributes[name] = value; } + scrollIntoView() { this.scrolled = true; } +} + +function fixture(t, search = '') { + const controls = {}; + for (const name of ['search', 'kind', 'os', 'list', 'status', 'detail', 'previous', 'next', 'page']) { + controls[`preset-${name}`] = new FakeElement('div'); + } + controls['preset-search'].value = ''; + controls['preset-kind'].value = 'all'; + controls['preset-os'].value = 'all'; + const calls = []; + const events = {}; + const timers = new Map(); + let timerId = 0; + let worker; + const location = { href: `https://example.org/en/pr-123/${search}`, search }; + const globals = { + document: { getElementById: id => controls[id], createElement: tag => new FakeElement(tag) }, + PRESET_BASE: '/en/pr-123', location, + history: { pushState: (state, title, url) => { location.href = url.href; location.search = url.search; } }, + addEventListener: (name, listener) => { events[name] = listener; }, + fetch: async (url, options) => { + calls.push({ url, options }); + return { ok: true, json: async () => ({ name: 'Halo', source_url: 'https://example.org/halo', + presets: [{ name: 'Halo', command: 'halo.exe' }] }) }; + }, + Worker: class { + constructor(url) { + this.url = url; + this.handlers = {}; + this.messages = []; + worker = this; + } + addEventListener(name, handler) { this.handlers[name] = handler; } + postMessage(message) { this.messages.push(message); } + }, + setTimeout: (handler, ms) => { const id = ++timerId; timers.set(id, { handler, ms }); return id; }, + clearTimeout: id => timers.delete(id) + }; + for (const [name, value] of Object.entries(globals)) { + const previous = Object.getOwnPropertyDescriptor(globalThis, name); + Object.defineProperty(globalThis, name, { configurable: true, writable: true, value }); + t.after(() => { + if (previous) Object.defineProperty(globalThis, name, previous); + else delete globalThis[name]; + }); + } + return { controls, calls, events, timers, worker: () => worker }; +} + +function result(page = 0, total = 50) { + const start = page * 24; + return { page, start, total, page_count: Math.ceil(total / 24), + items: Array.from({ length: Math.min(24, total - start) }, (_, i) => ({ + id: start + i + 1, name: `Game ${start + i + 1}`, kind: 'game', + preset_count: 1, operating_systems: ['Windows'], launch_methods: ['steam'] + })) }; +} + +test('browser renders one page, navigates it, and resets pagination when filters change', t => { + const { controls, calls, worker } = fixture(t); + boot(); + assert.equal(worker().url, '/en/pr-123/assets/js/search-worker.js'); + assert.equal(worker().messages[0].page, 0); + assert.equal(calls.length, 0); + worker().handlers.message({ data: { id: worker().messages.at(-1).id, result: result() } }); + assert.equal(controls['preset-list'].children.length, 24); + assert.match(controls['preset-status'].textContent, /50 games and apps found.*Showing 1–24/); + assert.equal(controls['preset-previous'].disabled, true); + assert.equal(controls['preset-next'].disabled, false); + assert.equal(controls['preset-page'].textContent, 'Page 1 of 3'); + controls['preset-next'].handlers.click(); + assert.equal(worker().messages.at(-1).page, 1); + assert.equal(controls['preset-next'].disabled, true); + worker().handlers.message({ data: { id: worker().messages.at(-1).id, result: result(1) } }); + assert.match(controls['preset-status'].textContent, /Showing 25–48/); + controls['preset-previous'].handlers.click(); + assert.equal(worker().messages.at(-1).page, 0); + controls['preset-kind'].value = 'app'; + controls['preset-kind'].handlers.input(); + assert.equal(worker().messages.at(-1).kind, 'app'); + assert.equal(worker().messages.at(-1).page, 0); + controls['preset-os'].value = 'Linux'; + controls['preset-os'].handlers.input(); + assert.equal(worker().messages.at(-1).os, 'Linux'); + worker().handlers.message({ data: { id: worker().messages.at(-1).id, result: result(0, 0) } }); + assert.equal(controls['preset-next'].disabled, true); + assert.equal(controls['preset-list'].children.length, 0); + assert.match(controls['preset-status'].textContent, /No matching/); +}); + +test('typing is debounced and invalidates results immediately, including during the delay', t => { + const { controls, timers, worker } = fixture(t); + boot(); + const first = worker().messages[0]; + controls['preset-search'].value = 'ha'; + controls['preset-search'].handlers.input(); + controls['preset-search'].value = 'halo'; + controls['preset-search'].handlers.input(); + assert.equal(timers.size, 1); + assert.equal(worker().messages.length, 1); + assert.equal(timers.values().next().value.ms, 250); + worker().handlers.message({ data: { id: first.id, result: result() } }); + assert.equal(controls['preset-list'].children.length, 0); + timers.values().next().value.handler(); + const current = worker().messages.at(-1); + assert.equal(current.query, 'halo'); + assert.equal(current.page, 0); + worker().handlers.message({ data: { id: first.id, error: 'stale failure' } }); + assert.ok(!controls['preset-status'].textContent.includes('stale failure')); + worker().handlers.message({ data: { id: current.id, + result: { ...result(0, 0), min_query_length: 3 } } }); + assert.match(controls['preset-status'].textContent, /at least 3 characters/); + assert.equal(controls['preset-list'].attributes['aria-busy'], 'false'); +}); + +test('direct links load records independently of catalog availability and back navigation hides details', async t => { + const { controls, calls, events, worker } = fixture(t, '?kind=game&id=42'); + boot(); + assert.equal(calls[0].url, '/en/pr-123/games/42.json'); + assert.equal(calls.length, 1); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(controls['preset-detail'].children[0].textContent, 'Halo'); + assert.equal(controls['preset-detail'].scrolled, true); + worker().handlers.message({ data: { id: worker().messages[0].id, error: 'HTTP 503' } }); + assert.match(controls['preset-status'].textContent, /HTTP 503/); + assert.equal(controls['preset-detail'].hidden, false); + location.search = ''; + events.popstate(); + assert.equal(controls['preset-detail'].hidden, true); + assert.equal(calls[0].options.signal.aborted, true); + location.search = '?kind=app&id=app-one'; + events.popstate(); + assert.equal(calls[1].url, '/en/pr-123/apps/app-one.json'); + location.search = '?kind=invalid&id=42'; + events.popstate(); + assert.equal(controls['preset-detail'].hidden, true); + worker().handlers.error(); + assert.match(controls['preset-status'].textContent, /Search worker unavailable/); +}); + +test('selecting a card keeps its direct link and superseded detail responses cannot overwrite it', async t => { + const { controls, calls, worker } = fixture(t); + const waiting = []; + globalThis.fetch = (url, options) => { + calls.push({ url, options }); + return new Promise(resolve => waiting.push(resolve)); + }; + boot(); + worker().handlers.message({ data: { id: worker().messages[0].id, result: result() } }); + controls['preset-list'].children[0].children[0].handlers.click(); + controls['preset-list'].children[1].children[0].handlers.click(); + assert.equal(new URL(location.href).searchParams.get('id'), '2'); + assert.equal(calls[0].options.signal.aborted, true); + const response = name => ({ ok: true, json: async () => ({ name, presets: [] }) }); + waiting[1](response('Second game')); + await new Promise(resolve => setImmediate(resolve)); + waiting[0](response('First game')); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(controls['preset-detail'].children[0].textContent, 'Second game'); +}); + +test('record fetch failures and worker construction failures show usable errors', async t => { + const { controls } = fixture(t, '?kind=game&id=42'); + globalThis.Worker = class { constructor() { throw new Error('blocked'); } }; + globalThis.fetch = async () => ({ ok: false, status: 404 }); + boot(); + assert.match(controls['preset-status'].textContent, /blocked.*Reload/); + await new Promise(resolve => setImmediate(resolve)); + assert.match(controls['preset-detail'].children[0].textContent, /HTTP 404/); +}); diff --git a/tests/site-catalog.test.js b/tests/site-catalog.test.js new file mode 100644 index 00000000000..af121add79b --- /dev/null +++ b/tests/site-catalog.test.js @@ -0,0 +1,227 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const vm = require('node:vm'); +const { buildCatalog } = require('../src/site-catalog'); +const { CatalogClient, PAGE_SIZE, gramKey } = require('../gh-pages-template/assets/js/catalog'); + +function entry(id, name, operatingSystems = ['Windows']) { + return { id, name, operating_systems: operatingSystems, preset_count: 1, + image_url: 'https://example.org/cover.png', launch_methods: ['steam'] }; +} + +function fixture(t, index) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'preset-catalog-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const manifest = buildCatalog(index, root); + const calls = []; + let bytes = 0; + const fetcher = async (url, options = {}) => { + options.signal?.throwIfAborted(); + calls.push({ url, options }); + const file = path.join(root, url.replace('/preview/PresetDB/', '')); + if (!fs.existsSync(file)) return { ok: false, status: 404 }; + const data = fs.readFileSync(file, 'utf8'); + bytes += Buffer.byteLength(data); + return { ok: true, json: async () => JSON.parse(data) }; + }; + return { root, manifest, calls, fetcher, bytes: () => bytes, + client: new CatalogClient('/preview/PresetDB', fetcher) }; +} + +function reference(index, query = '', kind = 'all', os = 'all') { + return [ + ...index.games.map(item => ({ ...item, kind: 'game' })), + ...index.apps.map(item => ({ ...item, kind: 'app' })) + ].filter(item => (kind === 'all' || item.kind === kind) && + (os === 'all' || item.operating_systems.includes(os)) && + item.name.toLowerCase().includes(query.trim().toLowerCase())) + .sort((a, b) => a.name.localeCompare(b.name) || a.kind.localeCompare(b.kind) || + String(a.id).localeCompare(String(b.id))); +} + +test('100,001 records need only a manifest and one small chunk on the first visit', async t => { + const index = { games: Array.from({ length: 100001 }, (_, id) => + entry(id + 1, `Game ${String(id).padStart(6, '0')}`, ['Windows', 'Linux', 'macOS'])), apps: [] }; + const { client, calls, bytes, manifest } = fixture(t, index); + const result = await client.page(); + assert.equal(manifest.counts.all.all, 100001); + assert.equal(result.total, 100001); + assert.equal(result.page_count, Math.ceil(100001 / PAGE_SIZE)); + assert.equal(result.items.length, PAGE_SIZE); + assert.deepEqual(result.items.map(item => item.id), Array.from({ length: PAGE_SIZE }, (_, id) => id + 1)); + assert.deepEqual(calls.map(call => call.url), [ + '/preview/PresetDB/catalog/manifest.json', + `/preview/PresetDB/catalog/${manifest.revision}/items/0.json` + ]); + assert.equal(calls[0].options.cache, 'no-cache'); + assert.ok(bytes() < 25000, `Initial transfer was ${bytes()} bytes`); + assert.ok(JSON.stringify(index).length > bytes() * 500); + t.diagnostic(`100,001 items: initial catalog ${bytes()} bytes; full index ${Buffer.byteLength(JSON.stringify(index, null, 2))} bytes`); + calls.length = 0; + const last = await client.page({ page: 99999 }); + assert.equal(last.items.length, 17); + assert.equal(last.items.at(-1).id, 100001); + assert.equal(last.start, 99984); + assert.equal(calls.length, 1); + calls.length = 0; + const found = await client.page({ query: 'Game 100000' }); + assert.equal(found.total, 1); + assert.equal(found.items[0].id, 100001); + assert.ok(calls.length <= 4); + assert.ok(calls.every(call => call.url.includes('/search/'))); +}); + +test('browse pages and counts agree with the full catalog for every type and OS filter', async t => { + const index = { + games: Array.from({ length: 481 }, (_, id) => entry(id + 1, `Game ${String(id).padStart(3, '0')}`, + id % 2 ? ['Windows', 'Linux', 'macOS'] : ['Windows'])), + apps: Array.from({ length: 29 }, (_, id) => entry(`app-${id}`, `App ${id}`, ['Linux'])) + }; + const { client } = fixture(t, index); + for (const kind of ['all', 'game', 'app']) { + for (const os of ['all', 'Windows', 'Linux', 'macOS']) { + const expected = reference(index, '', kind, os); + const actual = []; + for (let page = 0; page < Math.max(1, Math.ceil(expected.length / PAGE_SIZE)); page++) { + const result = await client.page({ kind, os, page }); + assert.equal(result.total, expected.length); + assert.equal(result.page, page); + assert.ok(result.items.length <= PAGE_SIZE); + actual.push(...result.items); + } + assert.deepEqual(actual, expected, `${kind}/${os}`); + } + } + assert.equal((await client.page({ page: -1 })).page, 0); + assert.equal((await client.page({ page: NaN })).page, 0); + await assert.rejects(client.page({ kind: '../bad' }), /Invalid catalog filter/); + await assert.rejects(client.page({ os: 'Other' }), /Invalid catalog filter/); +}); + +test('positional search preserves exact substrings, repeated grams, punctuation, and Unicode', async t => { + const names = ['abcdef', 'abcXXdef', 'aaaaaa', 'aaaaXaaa', 'HALO: Reach', + '星空🚀冒険', '🚀星空冒険', 'Résumé: Édition', 'short', 'A', 'abcdef']; + const index = { + games: names.map((name, id) => entry(id + 1, name, id % 2 ? ['Linux'] : ['Windows', 'macOS'])), + apps: [entry('abc', 'abcdef', ['Linux']), entry('halo', 'HALO: Reach', ['Windows'])] + }; + const { client, calls } = fixture(t, index); + const queries = new Set([' abcdef ', 'AAAaaa', 'HAL', 'o: re', '🚀冒険', 'absent', 'abc def']); + for (const name of names) { + const characters = Array.from(name); + for (let start = 0; start < characters.length; start++) { + for (let length = 3; length <= characters.length - start; length++) { + queries.add(characters.slice(start, start + length).join('')); + } + } + } + for (const query of queries) { + if (Array.from(query.trim()).length < 3) continue; + for (const kind of ['all', 'game', 'app']) { + for (const os of ['all', 'Windows', 'Linux', 'macOS']) { + const result = await client.page({ query, kind, os }); + const expected = reference(index, query, kind, os); + assert.equal(result.total, expected.length, `${query}/${kind}/${os}`); + assert.deepEqual(result.items, expected); + } + } + } + assert.ok(calls.every(call => !call.url.endsWith('/index.json'))); + assert.ok(calls.some(call => call.url.includes(`/search/${gramKey('🚀冒険')}.json`))); +}); + +test('short queries fetch no search data; empty catalogs and absent grams give empty results', async t => { + const { client, calls } = fixture(t, { games: [entry(1, 'Halo')], apps: [] }); + for (const query of ['a', 'ab', '🚀星']) { + assert.equal((await client.page({ query })).min_query_length, 3); + } + assert.equal(calls.length, 1); + assert.equal((await client.page({ query: 'zzz' })).total, 0); + assert.equal(calls.length, 2); + const empty = fixture(t, { games: [], apps: [] }); + assert.deepEqual((await empty.client.page()).items, []); + assert.equal((await empty.client.page()).total, 0); + assert.equal(empty.calls.length, 1); +}); + +test('search pages reuse the last match set and bound cached chunks', async t => { + const index = { games: Array.from({ length: 2000 }, (_, id) => + entry(id + 1, `Halo ${String(id).padStart(4, '0')}`)), apps: [] }; + const { client, calls } = fixture(t, index); + const first = await client.page({ query: 'halo' }); + assert.equal(first.total, 2000); + assert.equal(first.items.length, PAGE_SIZE); + calls.length = 0; + const second = await client.page({ query: 'HALO', page: 1 }); + assert.equal(second.items[0].id, 25); + assert.equal(calls.length, 0); + for (let page = 0; page < 84; page += 4) await client.page({ page }); + assert.equal(client.cache.size, 16); + assert.ok(!client.cache.has('items/0.json')); +}); + +test('catalog revisions track metadata, while errors and aborted requests can be retried', async t => { + const index = { games: [entry(1, 'Halo')], apps: [] }; + const { root, client, manifest, fetcher } = fixture(t, index); + assert.equal(buildCatalog(index, root).revision, manifest.revision); + assert.notEqual(buildCatalog({ ...index, games: [entry(1, 'Halo Updated')] }, root).revision, manifest.revision); + let failed = false; + client.fetcher = async (...args) => { + if (!failed) { failed = true; return { ok: false, status: 503 }; } + return fetcher(...args); + }; + await assert.rejects(client.page(), /HTTP 503/); + assert.equal((await client.page()).items[0].name, 'Halo Updated'); + client.fetcher = async () => ({ ok: false, status: 503 }); + await assert.rejects(client.page({ query: 'alo' }), /HTTP 503/); + client.fetcher = fetcher; + const controller = new AbortController(); + controller.abort(); + await assert.rejects(client.page({ query: 'alo' }, controller.signal), { name: 'AbortError' }); + assert.equal((await client.page({ query: 'alo' })).total, 1); + const invalid = new CatalogClient('', async () => ({ ok: true, json: async () => ({ schema_version: 2 }) })); + await assert.rejects(invalid.page(), /Unsupported catalog version/); +}); + +test('worker aborts superseded requests and reports only the latest result or error', async t => { + const { fetcher } = fixture(t, { games: [entry(1, 'Halo')], apps: [] }); + const messages = []; + let handler; + let release; + let first = true; + let oldSignal; + const context = vm.createContext({ + AbortController, + addEventListener: (name, listener) => { assert.equal(name, 'message'); handler = listener; }, + postMessage: message => messages.push(message), + fetch: async function (url, options) { + assert.equal(this, vm.runInContext('globalThis', context)); + if (first) { + first = false; + oldSignal = options.signal; + await new Promise(resolve => { release = resolve; }); + return fetcher(url); // Deliberately ignore cancellation to exercise the stale-result guard. + } + return fetcher(url, options); + } + }); + context.importScripts = file => vm.runInContext(fs.readFileSync( + path.join(__dirname, '..', 'gh-pages-template/assets/js', file), 'utf8'), context); + vm.runInContext(fs.readFileSync(path.join(__dirname, '..', + 'gh-pages-template/assets/js/search-worker.js'), 'utf8'), context); + const old = handler({ data: { id: 1, base: '/preview/PresetDB', query: 'zzz' } }); + await handler({ data: { id: 2, base: '/preview/PresetDB', query: 'hal' } }); + assert.equal(oldSignal.aborted, true); + release(); + await old; + assert.deepEqual(messages.map(message => message.id), [2]); + assert.equal(messages[0].result.items[0].name, 'Halo'); + context.fetch = async () => { throw new Error('offline'); }; + await handler({ data: { id: 3, base: '/other', query: '' } }); + assert.equal(messages[1].error, 'offline'); +}); From 328add9c1c492d7667369a839e13ae5abebb6355 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:31:32 -0400 Subject: [PATCH 2/2] fix: address website catalog Sonar findings --- gh-pages-template/assets/js/app.js | 3 +- gh-pages-template/assets/js/catalog.js | 24 +++---- gh-pages-template/assets/js/search-worker.js | 2 + src/site-catalog.js | 55 +++++++++------ tests/site-catalog.test.js | 72 ++++++++++++++++++-- 5 files changed, 118 insertions(+), 38 deletions(-) diff --git a/gh-pages-template/assets/js/app.js b/gh-pages-template/assets/js/app.js index 92d4d6a6dd3..b66b211496b 100644 --- a/gh-pages-template/assets/js/app.js +++ b/gh-pages-template/assets/js/app.js @@ -272,7 +272,8 @@ function boot() { const requestedKind = params.get('kind'); const requestedId = params.get('id'); if (requestedId && ['game', 'app'].includes(requestedKind)) { - showRecord({ kind: requestedKind, id: requestedId }); + // Record loading reports failures in the detail panel. + void showRecord({ kind: requestedKind, id: requestedId }); } else { detailController?.abort(); detailId++; diff --git a/gh-pages-template/assets/js/catalog.js b/gh-pages-template/assets/js/catalog.js index 3d27fab59c0..7b2454b35ee 100644 --- a/gh-pages-template/assets/js/catalog.js +++ b/gh-pages-template/assets/js/catalog.js @@ -118,13 +118,13 @@ class CatalogClient { let next = 0; let empty = false; const loadNext = async () => { - while (next < keys.length && !empty) { - signal?.throwIfAborted(); - const key = keys[next++]; - const values = await this.json(`search/${key}.json`, signal, true); - postings.set(key, values); - if (!values.length) empty = true; - } + if (next >= keys.length || empty) return; + signal?.throwIfAborted(); + const key = keys[next++]; + const values = await this.json(`search/${key}.json`, signal, true); + postings.set(key, values); + if (!values.length) empty = true; + return loadNext(); }; await Promise.all(Array.from({ length: Math.min(4, keys.length) }, loadNext)); return empty ? [] : searchPositions(grams, postings, kind, os); @@ -136,11 +136,11 @@ class CatalogClient { const loaded = new Map(); let next = 0; const loadNext = async () => { - while (next < chunks.length) { - signal?.throwIfAborted(); - const chunk = chunks[next++]; - loaded.set(chunk, await this.json(`items/${chunk}.json`, signal)); - } + if (next >= chunks.length) return; + signal?.throwIfAborted(); + const chunk = chunks[next++]; + loaded.set(chunk, await this.json(`items/${chunk}.json`, signal)); + return loadNext(); }; await Promise.all(Array.from({ length: Math.min(4, chunks.length) }, loadNext)); return ordinals.map(ordinal => loaded.get(Math.floor(ordinal / size))[ordinal % size]); diff --git a/gh-pages-template/assets/js/search-worker.js b/gh-pages-template/assets/js/search-worker.js index aaf2dbf1e64..aab7dc4dde8 100644 --- a/gh-pages-template/assets/js/search-worker.js +++ b/gh-pages-template/assets/js/search-worker.js @@ -5,6 +5,8 @@ importScripts('catalog.js'); let catalog; let current; globalThis.addEventListener('message', async event => { + // Dedicated worker messages use an empty origin; reject other foreign origins. + if (event.origin !== '' && event.origin !== globalThis.location.origin) return; current?.abort(); const controller = new AbortController(); current = controller; diff --git a/src/site-catalog.js b/src/site-catalog.js index 0ca3c609169..12d3fcbbb6a 100644 --- a/src/site-catalog.js +++ b/src/site-catalog.js @@ -20,34 +20,30 @@ function writeChunks(root, folder, values, size) { } } -function buildCatalog(index, output) { - const items = [ - ...index.games.map(item => ({ ...item, kind: 'game' })), - ...index.apps.map(item => ({ ...item, kind: 'app' })) - ].sort((a, b) => a.name.localeCompare(b.name) || a.kind.localeCompare(b.kind) || - String(a.id).localeCompare(String(b.id))); - const manifest = { - schema_version: 1, page_size: PAGE_SIZE, catalog_chunk_size: CATALOG_CHUNK_SIZE, - browse_chunk_size: BROWSE_CHUNK_SIZE, min_query_length: MIN_QUERY_LENGTH, counts: {} - }; - manifest.revision = createHash('sha256').update(JSON.stringify({ manifest, items })).digest('hex').slice(0, 16); - const root = path.join(output, 'catalog', manifest.revision); - writeChunks(root, 'items', items, CATALOG_CHUNK_SIZE); +function browseOrdinals(items, kind, os) { + const ordinals = []; + for (let ordinal = 0; ordinal < items.length; ordinal++) { + const item = items[ordinal]; + if ((kind === 'all' || item.kind === kind) && + (os === 'all' || item.operating_systems.includes(os))) ordinals.push(ordinal); + } + return ordinals; +} + +function writeBrowseChunks(root, items, counts) { for (const kind of ['all', 'game', 'app']) { - manifest.counts[kind] = {}; + counts[kind] = {}; for (const os of ['all', 'Windows', 'Linux', 'macOS']) { - const ordinals = []; - for (let ordinal = 0; ordinal < items.length; ordinal++) { - const item = items[ordinal]; - if ((kind === 'all' || item.kind === kind) && - (os === 'all' || item.operating_systems.includes(os))) ordinals.push(ordinal); - } - manifest.counts[kind][os] = ordinals.length; + const ordinals = browseOrdinals(items, kind, os); + counts[kind][os] = ordinals.length; if (kind !== 'all' || os !== 'all') { writeChunks(root, `browse/${kind}/${os}`, ordinals, BROWSE_CHUNK_SIZE); } } } +} + +function writeSearchIndex(root, items) { const postings = new Map(); for (let ordinal = 0; ordinal < items.length; ordinal++) { const item = items[ordinal]; @@ -60,6 +56,23 @@ function buildCatalog(index, output) { } } for (const [key, values] of postings) writeJson(root, `search/${key}.json`, values); +} + +function buildCatalog(index, output) { + const items = [ + ...index.games.map(item => ({ ...item, kind: 'game' })), + ...index.apps.map(item => ({ ...item, kind: 'app' })) + ].sort((a, b) => a.name.localeCompare(b.name) || a.kind.localeCompare(b.kind) || + String(a.id).localeCompare(String(b.id))); + const manifest = { + schema_version: 1, page_size: PAGE_SIZE, catalog_chunk_size: CATALOG_CHUNK_SIZE, + browse_chunk_size: BROWSE_CHUNK_SIZE, min_query_length: MIN_QUERY_LENGTH, counts: {} + }; + manifest.revision = createHash('sha256').update(JSON.stringify({ manifest, items })).digest('hex').slice(0, 16); + const root = path.join(output, 'catalog', manifest.revision); + writeChunks(root, 'items', items, CATALOG_CHUNK_SIZE); + writeBrowseChunks(root, items, manifest.counts); + writeSearchIndex(root, items); writeJson(output, 'catalog/manifest.json', manifest); return manifest; } diff --git a/tests/site-catalog.test.js b/tests/site-catalog.test.js index af121add79b..db1539f95e8 100644 --- a/tests/site-catalog.test.js +++ b/tests/site-catalog.test.js @@ -188,7 +188,7 @@ test('catalog revisions track metadata, while errors and aborted requests can be await assert.rejects(invalid.page(), /Unsupported catalog version/); }); -test('worker aborts superseded requests and reports only the latest result or error', async t => { +test('worker checks origins, aborts superseded requests, and reports only the latest result or error', async t => { const { fetcher } = fixture(t, { games: [entry(1, 'Halo')], apps: [] }); const messages = []; let handler; @@ -197,6 +197,7 @@ test('worker aborts superseded requests and reports only the latest result or er let oldSignal; const context = vm.createContext({ AbortController, + location: { origin: 'https://example.org' }, addEventListener: (name, listener) => { assert.equal(name, 'message'); handler = listener; }, postMessage: message => messages.push(message), fetch: async function (url, options) { @@ -214,14 +215,77 @@ test('worker aborts superseded requests and reports only the latest result or er path.join(__dirname, '..', 'gh-pages-template/assets/js', file), 'utf8'), context); vm.runInContext(fs.readFileSync(path.join(__dirname, '..', 'gh-pages-template/assets/js/search-worker.js'), 'utf8'), context); - const old = handler({ data: { id: 1, base: '/preview/PresetDB', query: 'zzz' } }); - await handler({ data: { id: 2, base: '/preview/PresetDB', query: 'hal' } }); + const old = handler({ origin: '', data: { id: 1, base: '/preview/PresetDB', query: 'zzz' } }); + await handler({ origin: 'https://foreign.example', data: { id: 99, base: '/other', query: '' } }); + await handler({ data: { id: 100, base: '/other', query: '' } }); + assert.equal(oldSignal.aborted, false); + assert.equal(messages.length, 0); + await handler({ origin: 'https://example.org', data: { id: 2, base: '/preview/PresetDB', query: 'hal' } }); assert.equal(oldSignal.aborted, true); release(); await old; assert.deepEqual(messages.map(message => message.id), [2]); assert.equal(messages[0].result.items[0].name, 'Halo'); context.fetch = async () => { throw new Error('offline'); }; - await handler({ data: { id: 3, base: '/other', query: '' } }); + await handler({ origin: '', data: { id: 3, base: '/other', query: '' } }); assert.equal(messages[1].error, 'offline'); }); + +async function checkRequestLimit(start, client, fetcher, folder) { + const pending = new Map(); + const calls = []; + let active = 0; + let maximum = 0; + let released = false; + client.fetcher = async (url, options) => { + if (!url.includes(folder)) return fetcher(url, options); + calls.push(url); + active++; + maximum = Math.max(maximum, active); + await new Promise(resolve => { + if (released) resolve(); + else pending.set(url, resolve); + }); + pending.delete(url); + active--; + return fetcher(url, options); + }; + const loading = start(); + let result; + try { + await new Promise(resolve => setImmediate(resolve)); + assert.equal(calls.length, 4); + assert.equal(active, 4); + pending.values().next().value(); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(calls.length, 5); + assert.equal(active, 4); + } finally { + released = true; + for (const resolve of pending.values()) resolve(); + result = await loading; + } + assert.equal(maximum, 4); + assert.equal(active, 0); + return { result, calls }; +} + +test('search refills each available request slot while keeping at most four requests active', async t => { + const query = 'abcdefghijklmnopqrstuvwxyz0123456789'; + const { client, fetcher } = fixture(t, { games: [entry(1, query)], apps: [] }); + const { result, calls } = await checkRequestLimit(() => client.page({ query }), client, fetcher, '/search/'); + assert.equal(result.total, 1); + assert.equal(result.items[0].id, 1); + assert.equal(calls.length, 12); + assert.equal(new Set(calls).size, 12); +}); + +test('summary loading refills each slot without fetching more than four chunks at a time', async t => { + const { client, fetcher, manifest } = fixture(t, { games: Array.from({ length: 600 }, (_, i) => + entry(i + 1, `Game ${String(i).padStart(3, '0')}`)), apps: [] }); + await client.manifest(); + const ordinals = Array.from({ length: 6 }, (_, i) => i * manifest.catalog_chunk_size); + const { result, calls } = await checkRequestLimit(() => client.items(ordinals), client, fetcher, '/items/'); + assert.deepEqual(result.map(item => item.id), ordinals.map(i => i + 1)); + assert.equal(calls.length, 6); +});