diff --git a/.github/workflows/build-pages.yml b/.github/workflows/build-pages.yml index 5d696fd5568..16b30f189ef 100644 --- a/.github/workflows/build-pages.yml +++ b/.github/workflows/build-pages.yml @@ -36,6 +36,14 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} + - name: Restore ProtonDB ratings + id: protondb-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .cache/protondb.json + key: protondb-v1-${{ runner.os }}-${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: protondb-v1-${{ runner.os }}- + - name: Preview database migrations if: github.event_name == 'pull_request' shell: bash @@ -46,7 +54,18 @@ jobs: --backup-branch "database-backup-$source_commit" --source-commit "$source_commit" - name: Assemble site - run: node src/build-site.js --database database-branch/database --output site-build + env: + PROTONDB_MODE: ${{ github.event_name == 'pull_request' && 'cache-only' || 'refresh' }} + run: >- + node src/build-site.js --database database-branch/database --output site-build + --protondb-cache .cache/protondb.json --protondb-mode "$PROTONDB_MODE" + + - name: Save ProtonDB ratings + if: github.event_name != 'pull_request' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .cache/protondb.json + key: ${{ steps.protondb-cache.outputs.cache-primary-key }} - name: Archive site run: | diff --git a/.gitignore b/.gitignore index c18fedade22..31d550bcc55 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ node_modules/ coverage/ junit.xml lcov.info +.cache/ diff --git a/README.md b/README.md index e48bbb7e870..890ce80b48b 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ The [GameDB Steam sync workflow](.github/workflows/sync-gamedb-steam.yml) runs d On each push to master, the Migrate Database workflow checks `database/migrations.json` for completed migration IDs. Before applying pending migrations, it creates a `database-backup-` branch pointing to the untouched database. It then commits the migrated records and history file to the active `database` branch and triggers a fresh Pages build. Re-runs skip completed migrations. The backup branch is never used as the working database. -For pull requests, the Pages workflow applies pending migrations in its temporary database checkout before building the preview. It does not commit or push those preview changes. The Pages workflow creates an archive from the database and site template, then calls the same [LizardByte Jekyll build workflow](https://github.com/LizardByte/LizardByte.github.io/blob/master/.github/workflows/jekyll-build.yml) used by GameDB and ThemerrDB. It deploys to `gh-pages` after changes to `master` or an approved database update. +For pull requests, the Pages workflow applies pending migrations in its temporary database checkout before building the preview. It does not commit or push those preview changes. The full database is built for previews, but ProtonDB ratings are read from the latest production build cache without making live requests. Missing cached ratings remain unknown and their ProtonDB links stay available. Production builds reuse ratings for seven days, fetch new or expired Steam IDs, and retain the last known rating if a refresh is unavailable. The site builder accepts `--protondb-cache` for the cache file and `--protondb-mode cache-only` for builds without live rating lookups. The Pages workflow creates an archive from the database and site template, then calls the same [LizardByte Jekyll build workflow](https://github.com/LizardByte/LizardByte.github.io/blob/master/.github/workflows/jekyll-build.yml) used by GameDB and ThemerrDB. It deploys to `gh-pages` after changes to `master` or an approved database update. The empty `database` branch is initialized with `database/apps` and `database/games`. GitHub Pages serves the empty orphan `gh-pages` branch. Configure `GH_BOT_TOKEN`, `GH_BOT_EMAIL`, `GH_BOT_NAME`, `TWITCH_CLIENT_ID`, and `TWITCH_CLIENT_SECRET` as in the existing LizardByte database projects. `GH_BOT_TOKEN` needs issue, contents, and Actions write access to publish the database and dispatch the Pages build. The Twitch credentials resolve IGDB slugs; GameDB itself needs no credentials. The request, approval, and method labels are configured in the repository. Set the GitHub repository description to the heading description above. The `database` branch retains the `database/` directory; approved records are written there. diff --git a/src/build-site.js b/src/build-site.js index 689b8cfacb3..2f31333c4c8 100644 --- a/src/build-site.js +++ b/src/build-site.js @@ -11,6 +11,26 @@ function supportedOs(preset) { } const PROTON_TIERS = new Set(['borked', 'bronze', 'silver', 'gold', 'platinum', 'native']); +const PROTON_CACHE_MAX_AGE = 7 * 24 * 60 * 60 * 1000; + +function readProtonDbCache(file) { + if (!file || !fs.existsSync(file)) return {}; + try { + const cache = JSON.parse(fs.readFileSync(file, 'utf8')); + if (cache?.schema_version === 1 && cache.apps && typeof cache.apps === 'object' && + !Array.isArray(cache.apps)) return cache.apps; + } catch { + // Ratings are optional; a missing or invalid cache must not block a preview. + } + return {}; +} + +function validProtonDbCacheEntry(entry, now) { + if (!Number.isSafeInteger(entry?.fetched_at) || entry.fetched_at < 0 || entry.fetched_at > now) return false; + const rating = entry.rating; + return rating === null || (rating && PROTON_TIERS.has(rating.tier) && + (rating.reports === null || (Number.isInteger(rating.reports) && rating.reports >= 0))); +} async function protonDbRating(appId, fetcher) { try { @@ -29,28 +49,55 @@ async function protonDbRating(appId, fetcher) { } } -async function addProtonDb(records, fetcher) { - const ids = [...new Set(records.flatMap(item => item.presets - .filter(preset => preset.method === 'steam' && /^[1-9]\d{0,9}$/.test(preset.launch_id || '')) - .map(preset => preset.launch_id)))]; - const cache = new Map(); - let next = 0; - await Promise.all(Array.from({ length: Math.min(32, ids.length) }, async () => { - while (next < ids.length) { - const appId = ids[next++]; - cache.set(appId, await protonDbRating(appId, fetcher)); - } - })); +function applyProtonDbRatings(records, cache) { for (const item of records) { for (const preset of item.presets) { if (!cache.has(preset.launch_id)) continue; preset.protondb_url = 'https://www.protondb.com/app/' + preset.launch_id; - preset.protondb = cache.get(preset.launch_id); + preset.protondb = cache.get(preset.launch_id)?.rating ?? null; + } + } +} + +async function addProtonDb(records, fetcher, { cacheFile, mode, now }) { + const ids = [...new Set(records.flatMap(item => item.presets + .filter(preset => preset.method === 'steam' && /^[1-9]\d{0,9}$/.test(preset.launch_id || '')) + .map(preset => preset.launch_id)))]; + const prior = readProtonDbCache(cacheFile); + const cache = new Map(); + const pending = []; + let cached = 0; + for (const appId of ids) { + const entry = validProtonDbCacheEntry(prior[appId], now) ? prior[appId] : null; + if (mode === 'cache-only' || (entry && now - entry.fetched_at < PROTON_CACHE_MAX_AGE)) { + if (entry) cached++; + cache.set(appId, entry); + } else { + pending.push({ appId, entry }); } } + let next = 0; + async function refreshNext() { + if (next >= pending.length) return; + const { appId, entry } = pending[next++]; + const rating = await protonDbRating(appId, fetcher); + cache.set(appId, { fetched_at: now, rating: rating ?? entry?.rating ?? null }); + return refreshNext(); + } + // Each worker takes another ID only after its current request finishes. + await Promise.all(Array.from({ length: Math.min(32, pending.length) }, refreshNext)); + applyProtonDbRatings(records, cache); + if (cacheFile && mode === 'refresh') { + fs.mkdirSync(path.dirname(cacheFile), { recursive: true }); + fs.writeFileSync(cacheFile, JSON.stringify({ schema_version: 1, apps: Object.fromEntries(cache) }) + '\n'); + } + console.log(`ProtonDB ratings: ${ids.length} Steam IDs, ${cached} cached, ${pending.length} fetched`); } -async function buildSite(database, template, output, fetcher = globalThis.fetch) { +async function buildSite(database, template, output, fetcher = globalThis.fetch, { + cacheFile, mode = 'refresh', now = Date.now() +} = {}) { + if (!['refresh', 'cache-only'].includes(mode)) throw new Error('ProtonDB mode must be refresh or cache-only'); fs.mkdirSync(output, { recursive: true }); fs.cpSync(template, output, { recursive: true }); const index = { schema_version: 1, games: [], apps: [] }; @@ -77,7 +124,7 @@ async function buildSite(database, template, output, fetcher = globalThis.fetch) } index[folder].sort((a, b) => a.name.localeCompare(b.name) || String(a.id).localeCompare(String(b.id))); } - await addProtonDb(records, fetcher); + await addProtonDb(records, fetcher, { cacheFile, mode, now }); for (const item of records) { const folder = item.kind === 'game' ? 'games' : 'apps'; fs.writeFileSync(path.join(output, folder, `${item.id}.json`), JSON.stringify(item, null, 2) + '\n'); @@ -90,13 +137,15 @@ async function buildSite(database, template, output, fetcher = globalThis.fetch) return index; } -async function main(args = process.argv.slice(2)) { +async function main(args = process.argv.slice(2), fetcher = globalThis.fetch) { const values = {}; for (let i = 0; i < args.length; i += 2) values[args[i]] = args[i + 1]; if (!values['--database'] || !values['--output']) throw new Error('Use --database and --output'); - await buildSite(values['--database'], values['--template'] || 'gh-pages-template', values['--output']); + return buildSite(values['--database'], values['--template'] || 'gh-pages-template', values['--output'], fetcher, { + cacheFile: values['--protondb-cache'], mode: values['--protondb-mode'] || 'refresh' + }); } if (require.main === module) main().catch(error => { console.error(error); process.exitCode = 1; }); -module.exports = { buildSite, protonDbRating }; +module.exports = { buildSite, protonDbRating, main }; diff --git a/tests/build-site.test.js b/tests/build-site.test.js new file mode 100644 index 00000000000..38cda09f4b8 --- /dev/null +++ b/tests/build-site.test.js @@ -0,0 +1,224 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { buildSite, main } = require('../src/build-site'); + +const template = path.join(__dirname, '..', 'gh-pages-template'); +const now = Date.UTC(2026, 9, 9); +const day = 24 * 60 * 60 * 1000; +const gold = { tier: 'gold', reports: 73 }; + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'preset-build-cache-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const database = path.join(root, 'database'); + const output = path.join(root, 'site'); + const cacheFile = path.join(root, 'cache', 'protondb.json'); + for (const folder of ['games', 'apps']) fs.mkdirSync(path.join(database, folder), { recursive: true }); + fs.mkdirSync(path.dirname(cacheFile)); + for (const [id, launchId] of [[1, '1001'], [2, '1002'], [3, '1003'], [4, '1001'], [5, null]]) { + fs.writeFileSync(path.join(database, 'games', `${id}.json`), JSON.stringify({ + schema_version: 2, kind: 'game', id, name: `Game ${id}`, presets: [{ + id: launchId ? 'steam' : '42', name: `Game ${id}`, method: launchId ? 'steam' : 'native', + ...(launchId ? { launch_id: launchId, commands_by_os: { Windows: `steam://rungameid/${launchId}` } } + : { os: 'Windows', command: 'game.exe' }) + }] + })); + } + fs.writeFileSync(path.join(database, 'apps', 'app-one.json'), JSON.stringify({ + schema_version: 2, kind: 'app', id: 'app-one', name: 'App One', + presets: [{ id: '43', name: 'App One', method: 'native', os: 'Linux', command: 'app-one' }] + })); + return { database, output, cacheFile }; +} + +function writeCache(file, apps) { + fs.writeFileSync(file, JSON.stringify({ schema_version: 1, apps })); +} + +function json(file) { + return JSON.parse(fs.readFileSync(file, 'utf8')); +} + +function preset(output, id) { + return json(path.join(output, 'games', `${id}.json`)).presets[0]; +} + +test('PR cache-only builds keep the full database and stale ratings with no network or cache writes', async t => { + const { database, output, cacheFile } = fixture(t); + writeCache(cacheFile, { 1001: { fetched_at: now - 10 * day, rating: gold } }); + const before = fs.readFileSync(cacheFile, 'utf8'); + const source = fs.readFileSync(path.join(database, 'games', '1.json'), 'utf8'); + let calls = 0; + const index = await buildSite(database, template, output, async () => { calls++; }, + { cacheFile, mode: 'cache-only', now }); + assert.equal(calls, 0); + assert.equal(index.games.length, 5); + assert.equal(index.apps.length, 1); + assert.deepEqual(preset(output, 1).protondb, gold); + assert.deepEqual(preset(output, 4).protondb, gold); + for (const id of [2, 3]) { + assert.equal(preset(output, id).protondb, null); + assert.equal(preset(output, id).protondb_url, `https://www.protondb.com/app/${1000 + id}`); + } + assert.equal(preset(output, 5).command, 'game.exe'); + assert.equal(preset(output, 5).protondb, undefined); + assert.equal(json(path.join(output, 'apps', 'app-one.json')).presets[0].command, 'app-one'); + assert.equal(json(path.join(output, 'stats.json')).preset_count, 6); + assert.equal(fs.readFileSync(cacheFile, 'utf8'), before); + assert.equal(fs.readFileSync(path.join(database, 'games', '1.json'), 'utf8'), source); +}); + +test('production reuses six-day-old ratings and refreshes unique Steam IDs at seven days', async t => { + const { database, output, cacheFile } = fixture(t); + writeCache(cacheFile, { + 1001: { fetched_at: now - 6 * day, rating: gold }, + 1002: { fetched_at: now - 7 * day, rating: { tier: 'bronze', reports: 5 } } + }); + const calls = []; + await buildSite(database, template, output, async url => { + calls.push(url); + return { ok: true, json: async () => ({ tier: 'platinum', total: 90 }) }; + }, { cacheFile, now }); + assert.deepEqual(calls.sort(), ['1002', '1003'].map(id => + `https://www.protondb.com/api/v1/reports/summaries/${id}.json`)); + assert.deepEqual(preset(output, 1).protondb, gold); + assert.deepEqual(preset(output, 2).protondb, { tier: 'platinum', reports: 90 }); + const cache = json(cacheFile); + assert.equal(cache.schema_version, 1); + assert.deepEqual(Object.keys(cache.apps), ['1001', '1002', '1003']); + assert.equal(cache.apps['1001'].fetched_at, now - 6 * day); + assert.equal(cache.apps['1002'].fetched_at, now); + assert.deepEqual(cache.apps['1003'].rating, { tier: 'platinum', reports: 90 }); + + calls.length = 0; + await buildSite(database, template, output, async url => { + calls.push(url); + return { ok: true, json: async () => ({ tier: 'native', total: 100 }) }; + }, { cacheFile, now: now + day }); + assert.deepEqual(calls, ['https://www.protondb.com/api/v1/reports/summaries/1001.json']); + assert.deepEqual(preset(output, 1).protondb, { tier: 'native', reports: 100 }); + assert.deepEqual(preset(output, 4).protondb, { tier: 'native', reports: 100 }); + assert.equal(json(cacheFile).apps['1001'].fetched_at, now + day); + assert.equal(json(cacheFile).apps['1002'].fetched_at, now); +}); + +test('failed refreshes retain known ratings and missing ratings are cached to avoid repeated lookups', async t => { + const { database, output, cacheFile } = fixture(t); + writeCache(cacheFile, { + 1001: { fetched_at: now - 8 * day, rating: gold }, + 1002: { fetched_at: now - 2 * day, rating: null } + }); + const calls = []; + await buildSite(database, template, output, async url => { + calls.push(url); + if (url.endsWith('/1001.json')) throw new Error('offline'); + return { ok: false, status: 404 }; + }, { cacheFile, now }); + assert.equal(calls.length, 2); + assert.deepEqual(preset(output, 1).protondb, gold); + assert.equal(preset(output, 2).protondb, null); + assert.equal(preset(output, 3).protondb, null); + assert.deepEqual(json(cacheFile).apps['1001'], { fetched_at: now, rating: gold }); + assert.deepEqual(json(cacheFile).apps['1003'], { fetched_at: now, rating: null }); + let repeatedCalls = 0; + await buildSite(database, template, output, async () => { repeatedCalls++; }, + { cacheFile, now: now + 1 }); + assert.equal(repeatedCalls, 0); +}); + +test('production keeps at most 32 requests active and refills each available worker', async t => { + const { database, output, cacheFile } = fixture(t); + const game = json(path.join(database, 'games', '1.json')); + for (let id = 6; id <= 70; id++) { + const launchId = String(1000 + id); + fs.writeFileSync(path.join(database, 'games', `${id}.json`), JSON.stringify({ + ...game, id, name: `Game ${id}`, presets: [{ + ...game.presets[0], launch_id: launchId, + commands_by_os: { Windows: `steam://rungameid/${launchId}` } + }] + })); + } + writeCache(cacheFile, { 1001: { fetched_at: now, rating: gold } }); + const calls = []; + const waiting = new Map(); + let released = false; + let active = 0; + let maximum = 0; + const building = buildSite(database, template, output, async url => { + calls.push(url); + active++; + maximum = Math.max(maximum, active); + await new Promise(resolve => { + if (released) resolve(); + else waiting.set(url, resolve); + }); + waiting.delete(url); + active--; + return { ok: true, json: async () => ({ tier: 'gold', total: 73 }) }; + }, { cacheFile, now }); + try { + await new Promise(resolve => setImmediate(resolve)); + assert.equal(calls.length, 32); + assert.equal(active, 32); + waiting.values().next().value(); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(calls.length, 33); + assert.equal(active, 32); + assert.equal(waiting.size, 32); + } finally { + released = true; + for (const resolve of waiting.values()) resolve(); + await building; + } + assert.equal(maximum, 32); + assert.equal(active, 0); + assert.equal(calls.length, 67); + assert.equal(new Set(calls).size, 67); + assert.ok(calls.every(url => !url.endsWith('/1001.json'))); + assert.equal(json(cacheFile).apps['1001'].fetched_at, now); + for (const id of [1, 2, 3, 4, 6, 70]) assert.deepEqual(preset(output, id).protondb, gold); +}); + +test('cold PR builds and invalid caches still publish presets and links without network requests', async t => { + const { database, output, cacheFile } = fixture(t); + const inputs = [undefined, '{', 'null', JSON.stringify({ schema_version: 2, apps: {} }), + JSON.stringify({ schema_version: 1, apps: [] }), JSON.stringify({ schema_version: 1, apps: { + 1001: { fetched_at: now + 1, rating: gold }, + 1002: { fetched_at: now, rating: { tier: 'unknown', reports: 0 } }, + 1003: { fetched_at: now, rating: { tier: 'gold', reports: -1 } } + } })]; + let calls = 0; + for (const input of inputs) { + if (input !== undefined) fs.writeFileSync(cacheFile, input); + await buildSite(database, template, output, async () => { calls++; }, + { cacheFile, mode: 'cache-only', now }); + for (const id of [1, 2, 3, 4]) { + assert.equal(preset(output, id).protondb, null); + assert.match(preset(output, id).protondb_url, /^https:\/\/www\.protondb\.com\/app\//); + } + if (input === undefined) assert.equal(fs.existsSync(cacheFile), false); + else assert.equal(fs.readFileSync(cacheFile, 'utf8'), input); + } + assert.equal(calls, 0); +}); + +test('site CLI forwards cache-only options and rejects an invalid rating mode before any requests', async t => { + const { database, output, cacheFile } = fixture(t); + writeCache(cacheFile, { 1001: { fetched_at: now, rating: gold } }); + let calls = 0; + const fetcher = async () => { calls++; }; + const args = ['--database', database, '--template', template, '--output', output, + '--protondb-cache', cacheFile, '--protondb-mode', 'cache-only']; + await assert.rejects(main([], fetcher), /Use --database and --output/); + await assert.rejects(main([...args.slice(0, -1), 'invalid'], fetcher), /ProtonDB mode/); + assert.equal(fs.existsSync(output), false); + const index = await main(args, fetcher); + assert.equal(index.games.length, 5); + assert.equal(calls, 0); + assert.deepEqual(preset(output, 1).protondb, gold); +});