From 72aa1c55df4c6090d1cef60d66fb90575877a684 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:21:04 -0400 Subject: [PATCH] perf: batch GameDB sync pushes --- .github/workflows/sync-gamedb-steam.yml | 2 +- README.md | 2 +- src/gamedb-sync.js | 53 ++++-- tests/gamedb-sync.test.js | 215 ++++++++++++++++++++---- 4 files changed, 224 insertions(+), 48 deletions(-) diff --git a/.github/workflows/sync-gamedb-steam.yml b/.github/workflows/sync-gamedb-steam.yml index 70801abcfc8..b9951ce3c7d 100644 --- a/.github/workflows/sync-gamedb-steam.yml +++ b/.github/workflows/sync-gamedb-steam.yml @@ -59,7 +59,7 @@ jobs: set -euo pipefail git -C database-branch config user.name "$BOT_NAME" git -C database-branch config user.email "$BOT_EMAIL" - node src/gamedb-sync.js --gamedb gamedb --database database-branch + node src/gamedb-sync.js --gamedb gamedb --database database-branch --batch-size 500 - name: Build updated site if: always() && steps.import.outputs.changed == 'true' diff --git a/README.md b/README.md index 6c927b39035..e48bbb7e870 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Game submissions resolve the submitted slug through [IGDB authenticated API](htt - `auto_approved_users.json`: trusted approvers and game submitters eligible for automatic queueing. - `.readthedocs.yaml`: pull request preview builds using the shared LizardByte Jekyll script. -The [GameDB Steam sync workflow](.github/workflows/sync-gamedb-steam.yml) runs daily at 07:00 UTC and can also be started manually. GameDB publishes its database on `gh-pages`, so the workflow reads that branch's PC game records and Steam external IDs. It writes eligible game files directly to the PresetDB `database` branch, with one file per commit and no issue or approval queue. Existing issue-backed Steam presets retain their numeric IDs, issue history, and contributor-managed launch values. GameDB changes to game metadata are synchronized; direct imports also receive Steam ID and command updates. Ambiguous Steam IDs are skipped. The job has a six-hour timeout; the next daily run skips unchanged files and continues the import. +The [GameDB Steam sync workflow](.github/workflows/sync-gamedb-steam.yml) runs daily at 07:00 UTC and can also be started manually. GameDB publishes its database on `gh-pages`, so the workflow reads that branch's PC game records and Steam external IDs. It writes eligible game files directly to the PresetDB `database` branch, with one file per commit and no issue or approval queue. Commits are pushed in batches of 500, with any remaining commits pushed at the end; the importer also accepts `--batch-size` to adjust this limit. Each batch fetches the database once, and rejected pushes retry against the latest records to preserve concurrent approvals. Existing issue-backed Steam presets retain their numeric IDs, issue history, and contributor-managed launch values. GameDB changes to game metadata are synchronized; direct imports also receive Steam ID and command updates. Ambiguous Steam IDs are skipped. The job has a six-hour timeout; the next daily run skips unchanged files and continues the import. On each push to master, the Migrate Database workflow checks `database/migrations.json` for completed migration IDs. Before applying pending migrations, it creates a `database-backup-` branch pointing to the untouched database. It then commits the migrated records and history file to the active `database` branch and triggers a fresh Pages build. Re-runs skip completed migrations. The backup branch is never used as the working database. diff --git a/src/gamedb-sync.js b/src/gamedb-sync.js index 59b1305710d..676bd1348ac 100644 --- a/src/gamedb-sync.js +++ b/src/gamedb-sync.js @@ -77,32 +77,51 @@ function git(checkout, args) { return execFileSync('/usr/bin/git', args, { cwd: checkout, encoding: 'utf8', stdio: 'pipe' }); } -function publishGame({ checkout, game, file, onPublished, gitCommand = git }) { - const relative = `database/games/${game.id}.json`; +function publishBatch({ checkout, games, onPublished, gitCommand = git }) { for (let attempt = 1; attempt <= 5; attempt++) { gitCommand(checkout, ['fetch', 'origin', 'database']); gitCommand(checkout, ['reset', '--hard', 'origin/database']); - const desired = mergeGame(game, readRecord(file)); - if (!desired) return false; - fs.mkdirSync(path.dirname(file), { recursive: true }); - fs.writeFileSync(file, `${JSON.stringify(desired, null, 2)}\n`); - gitCommand(checkout, ['add', '--', relative]); - gitCommand(checkout, ['commit', '-m', `chore: sync GameDB game ${game.id}`, '--', relative]); + let published = 0; + for (const game of games) { + const relative = `database/games/${game.id}.json`; + const file = path.join(checkout, relative); + const desired = mergeGame(game, readRecord(file)); + if (!desired) continue; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(desired, null, 2)}\n`); + gitCommand(checkout, ['add', '--', relative]); + gitCommand(checkout, ['commit', '-m', `chore: sync GameDB game ${game.id}`, '--', relative]); + published++; + } + if (!published) return 0; try { gitCommand(checkout, ['push', 'origin', 'HEAD:database']); - onPublished?.(); - return true; } catch (error) { if (attempt === 5) throw error; - console.log(`Database branch moved while publishing game ${game.id}; retrying`); + console.log('Database branch moved while publishing GameDB batch; retrying'); + continue; } + onPublished?.(); + return published; } } -function run({ gameDbDir, checkout, onPublished, gitCommand = git }) { +function run({ gameDbDir, checkout, batchSize = 500, onPublished, gitCommand = git }) { + if (!Number.isSafeInteger(batchSize) || batchSize < 1) { + throw new Error('Batch size must be a positive safe integer'); + } const database = path.join(checkout, 'database'); const index = JSON.parse(fs.readFileSync(path.join(gameDbDir, 'platforms', '6.json'), 'utf8')); const counts = { scanned: 0, eligible: 0, published: 0, current: 0 }; + let pending = []; + function flushBatch() { + if (!pending.length) return; + const published = publishBatch({ checkout, games: pending, onPublished, gitCommand }); + counts.published += published; + counts.current += pending.length - published; + console.log(`Published GameDB batch: ${published} games`); + pending = []; + } for (const id of pcGameIds(index)) { counts.scanned++; const source = JSON.parse(fs.readFileSync(path.join(gameDbDir, 'games', `${id}.json`), 'utf8')); @@ -112,11 +131,10 @@ function run({ gameDbDir, checkout, onPublished, gitCommand = git }) { counts.eligible++; const file = path.join(database, 'games', `${id}.json`); if (!mergeGame(game, readRecord(file))) { counts.current++; continue; } - if (publishGame({ checkout, game, file, onPublished, gitCommand })) { - counts.published++; - console.log(`Published game ${id}`); - } else counts.current++; + pending.push(game); + if (pending.length === batchSize) flushBatch(); } + flushBatch(); console.log(`GameDB Steam sync: ${JSON.stringify(counts)}`); return counts; } @@ -131,6 +149,7 @@ function main(args = process.argv.slice(2), env = process.env, gitCommand = git) return run({ gameDbDir: path.resolve(option('--gamedb')), checkout: path.resolve(option('--database')), + batchSize: args.includes('--batch-size') ? Number(option('--batch-size')) : 500, gitCommand, onPublished: () => { if (!reported && env.GITHUB_OUTPUT) { @@ -141,6 +160,6 @@ function main(args = process.argv.slice(2), env = process.env, gitCommand = git) }); } -module.exports = { pcGameIds, steamCandidate, mergeGame, publishGame, run, main }; +module.exports = { pcGameIds, steamCandidate, mergeGame, publishBatch, run, main }; if (require.main === module) main(); diff --git a/tests/gamedb-sync.test.js b/tests/gamedb-sync.test.js index ead7400d99b..ade7061c924 100644 --- a/tests/gamedb-sync.test.js +++ b/tests/gamedb-sync.test.js @@ -6,7 +6,7 @@ const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); const { execFileSync } = require('node:child_process'); -const { pcGameIds, steamCandidate, mergeGame, run, main } = require('../src/gamedb-sync'); +const { pcGameIds, steamCandidate, mergeGame, publishBatch, run, main } = require('../src/gamedb-sync'); const { mergePreset } = require('../src/database'); const { buildSite } = require('../src/build-site'); const { comparePresetIds } = require('../src/record'); @@ -23,6 +23,34 @@ function command(cwd, ...args) { return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: 'pipe' }).trim(); } +function syncFixture(t, ids, { withGit = true } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'preset-gamedb-sync-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const remote = path.join(root, 'remote.git'); + const checkout = path.join(root, 'checkout'); + const gameDbDir = path.join(root, 'gamedb'); + fs.mkdirSync(path.join(checkout, 'database', 'games'), { recursive: true }); + fs.mkdirSync(path.join(gameDbDir, 'platforms'), { recursive: true }); + fs.mkdirSync(path.join(gameDbDir, 'games')); + if (withGit) { + command(root, 'init', '--bare', remote); + command(checkout, 'init', '-b', 'database'); + command(checkout, 'remote', 'add', 'origin', remote); + command(checkout, 'config', 'user.name', 'LizardByte-bot'); + command(checkout, 'config', 'user.email', 'bot@example.com'); + fs.writeFileSync(path.join(checkout, 'database', 'games', '.gitkeep'), ''); + command(checkout, 'add', 'database/games/.gitkeep'); + command(checkout, 'commit', '-m', 'seed database'); + command(checkout, 'push', '-u', 'origin', 'database'); + } + fs.writeFileSync(path.join(gameDbDir, 'platforms', '6.json'), + JSON.stringify({ id: 6, games: ids.map(id => ({ id })) })); + for (const id of ids) { + fs.writeFileSync(path.join(gameDbDir, 'games', `${id}.json`), JSON.stringify(sourceGame(id))); + } + return { root, remote, checkout, gameDbDir }; +} + test('PC index and Steam source select one unambiguous launch ID', () => { assert.deepEqual(pcGameIds({ id: 6, games: [{ id: 2 }, { id: 1 }, { id: 2 }] }), [1, 2]); assert.throws(() => pcGameIds({ id: 5, games: [] }), /PC platform index/); @@ -70,43 +98,29 @@ test('direct import has no issue identity and refreshes only imported Steam laun }); test('sync publishes each changed game file in its own commit and skips unchanged files', async t => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'preset-gamedb-sync-')); - t.after(() => fs.rmSync(root, { recursive: true, force: true })); - const remote = path.join(root, 'remote.git'); - const checkout = path.join(root, 'checkout'); - const gameDbDir = path.join(root, 'gamedb'); - fs.mkdirSync(checkout); - fs.mkdirSync(path.join(gameDbDir, 'platforms'), { recursive: true }); - fs.mkdirSync(path.join(gameDbDir, 'games')); - command(root, 'init', '--bare', remote); - command(checkout, 'init', '-b', 'database'); - command(checkout, 'remote', 'add', 'origin', remote); - command(checkout, 'config', 'user.name', 'LizardByte-bot'); - command(checkout, 'config', 'user.email', 'bot@example.com'); - fs.mkdirSync(path.join(checkout, 'database', 'games'), { recursive: true }); - fs.writeFileSync(path.join(checkout, 'database', 'games', '.gitkeep'), ''); - command(checkout, 'add', 'database/games/.gitkeep'); - command(checkout, 'commit', '-m', 'seed database'); - command(checkout, 'push', '-u', 'origin', 'database'); - - fs.writeFileSync(path.join(gameDbDir, 'platforms', '6.json'), - JSON.stringify({ id: 6, games: [{ id: 1 }, { id: 2 }] })); - for (const id of [1, 2]) { - fs.writeFileSync(path.join(gameDbDir, 'games', `${id}.json`), JSON.stringify(sourceGame(id))); - } - - const args = { gameDbDir, checkout }; + const { root, checkout, gameDbDir } = syncFixture(t, [1, 2]); + const calls = []; + const gitCommand = (directory, gitArgs) => { + calls.push(gitArgs); + return command(directory, ...gitArgs); + }; + const args = { gameDbDir, checkout, gitCommand }; const outputFile = path.join(root, 'github-output'); assert.throws(() => main([], {}, () => {}), /Missing --gamedb/); assert.equal(main(['--gamedb', gameDbDir, '--database', checkout], - { GITHUB_OUTPUT: outputFile }, (directory, gitArgs) => command(directory, ...gitArgs)).published, 2); + { GITHUB_OUTPUT: outputFile }, gitCommand).published, 2); assert.equal(fs.readFileSync(outputFile, 'utf8'), 'changed=true\n'); + assert.equal(calls.filter(args => args[0] === 'fetch').length, 1); + assert.equal(calls.filter(args => args[0] === 'reset').length, 1); + assert.equal(calls.filter(args => args[0] === 'push').length, 1); assert.equal(command(checkout, 'rev-list', '--count', 'HEAD'), '3'); assert.equal(command(checkout, 'diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD'), 'database/games/2.json'); assert.equal(command(checkout, 'diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD^'), 'database/games/1.json'); - assert.equal(run(args).published, 0); + calls.length = 0; + assert.deepEqual(run(args), { scanned: 2, eligible: 2, published: 0, current: 2 }); + assert.deepEqual(calls, []); assert.equal(command(checkout, 'rev-list', '--count', 'HEAD'), '3'); const imported = JSON.parse(fs.readFileSync(path.join(checkout, 'database/games/1.json'))); assert.throws(() => mergePreset(path.join(checkout, 'database'), { @@ -126,6 +140,149 @@ test('sync publishes each changed game file in its own commit and skips unchange assert.equal(siteRecord.presets[0].protondb_url, 'https://www.protondb.com/app/1001'); }); +test('default batches push 500 commits and flush the remainder, excluding current and ineligible games', t => { + const ids = Array.from({ length: 1004 }, (_, index) => index + 1); + const { checkout, gameDbDir } = syncFixture(t, ids, { withGit: false }); + for (const id of [2, 501]) { + fs.writeFileSync(path.join(checkout, 'database', 'games', `${id}.json`), + JSON.stringify(mergeGame(steamCandidate(sourceGame(id)), null))); + } + fs.writeFileSync(path.join(gameDbDir, 'games', '1004.json'), + JSON.stringify({ ...sourceGame(1004), platforms: [3] })); + let commits = 0; + let fetches = 0; + let resets = 0; + let notifications = 0; + const pushes = []; + const counts = run({ + gameDbDir, checkout, + onPublished: () => notifications++, + gitCommand: (directory, args) => { + assert.equal(directory, checkout); + if (args[0] === 'fetch') fetches++; + if (args[0] === 'reset') resets++; + if (args[0] === 'commit') commits++; + if (args[0] === 'push') { pushes.push(commits); commits = 0; } + } + }); + assert.deepEqual(counts, { scanned: 1004, eligible: 1003, published: 1001, current: 2 }); + assert.deepEqual(pushes, [500, 500, 1]); + assert.equal(fetches, 3); + assert.equal(resets, 3); + assert.equal(notifications, 3); +}); + +test('CLI batch size controls pushes and publishes all commits to a real remote', t => { + const { root, remote, checkout, gameDbDir } = syncFixture(t, [1, 2, 3, 4, 5, 6, 7]); + const outputFile = path.join(root, 'github-output'); + const remoteCounts = []; + const gitCommand = (directory, args) => { + const result = command(directory, ...args); + if (args[0] === 'push') { + remoteCounts.push(Number(command(root, '--git-dir', remote, 'rev-list', '--count', 'database'))); + } + return result; + }; + const args = ['--gamedb', gameDbDir, '--database', checkout]; + assert.throws(() => main([...args, '--batch-size'], {}, gitCommand), /Missing --batch-size/); + for (const batchSize of ['0', '-1', '1.5', 'invalid', 'Infinity', '9007199254740992']) { + assert.throws(() => main([...args, '--batch-size', batchSize], {}, gitCommand), + /Batch size must be a positive safe integer/); + } + assert.equal(main([...args, '--batch-size', '3'], { GITHUB_OUTPUT: outputFile }, gitCommand).published, 7); + assert.deepEqual(remoteCounts, [4, 7, 8]); + assert.equal(fs.readFileSync(outputFile, 'utf8'), 'changed=true\n'); + assert.equal(command(checkout, 'rev-parse', 'HEAD'), + command(root, '--git-dir', remote, 'rev-parse', 'database')); +}); + +test('rejected batch is rebuilt from concurrent approvals and skips games already imported remotely', t => { + const { root, remote, checkout, gameDbDir } = syncFixture(t, [1, 2, 3]); + const peer = path.join(root, 'peer'); + command(root, 'clone', '--branch', 'database', remote, peer); + command(peer, 'config', 'user.name', 'Contributor'); + command(peer, 'config', 'user.email', 'contributor@example.com'); + const approved = mergeGame(steamCandidate(sourceGame(2)), null); + approved.name = 'Contributor title'; + approved.presets[0] = { + id: '42', name: approved.name, os: null, method: 'steam', launch_id: '9999', + commands_by_os: { Windows: 'custom command' }, origin_issue: 42, source_issue: 42, + history: [{ issue: 42, action: 'add' }] + }; + const unrelated = { contributor: 'retained' }; + const outputFile = path.join(root, 'github-output'); + let pushes = 0; + let fetches = 0; + const gitCommand = (directory, args) => { + if (args[0] === 'fetch') fetches++; + if (args[0] === 'push' && ++pushes === 1) { + fs.writeFileSync(path.join(peer, 'database', 'games', '1.json'), + JSON.stringify(mergeGame(steamCandidate(sourceGame(1)), null), null, 2) + '\n'); + fs.writeFileSync(path.join(peer, 'database', 'games', '2.json'), JSON.stringify(approved)); + fs.writeFileSync(path.join(peer, 'database', 'contributor.json'), JSON.stringify(unrelated)); + command(peer, 'add', 'database'); + command(peer, 'commit', '-m', 'concurrent approval'); + command(peer, 'push', 'origin', 'database'); + } + return command(directory, ...args); + }; + assert.deepEqual(main(['--gamedb', gameDbDir, '--database', checkout], + { GITHUB_OUTPUT: outputFile }, gitCommand), + { scanned: 3, eligible: 3, published: 2, current: 1 }); + assert.equal(fetches, 2); + assert.equal(pushes, 2); + assert.equal(command(checkout, 'rev-list', '--count', 'HEAD'), '4'); + assert.equal(command(checkout, 'log', '-2', '--format=%s'), + 'chore: sync GameDB game 3\nchore: sync GameDB game 2'); + const record = JSON.parse(fs.readFileSync(path.join(checkout, 'database', 'games', '2.json'))); + assert.deepEqual(record, { + ...approved, name: 'Game 2', presets: [{ ...approved.presets[0], name: 'Game 2' }] + }); + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(checkout, 'database', 'contributor.json'))), + unrelated); + assert.equal(fs.readFileSync(outputFile, 'utf8'), 'changed=true\n'); +}); + +test('a retry that finds the entire batch current avoids another push and changed output', t => { + const { root, checkout, gameDbDir } = syncFixture(t, [1], { withGit: false }); + const file = path.join(checkout, 'database', 'games', '1.json'); + const outputFile = path.join(root, 'github-output'); + let pushes = 0; + let resets = 0; + const counts = main(['--gamedb', gameDbDir, '--database', checkout], + { GITHUB_OUTPUT: outputFile }, (directory, args) => { + if (args[0] === 'reset' && ++resets === 2) { + fs.writeFileSync(file, JSON.stringify(mergeGame(steamCandidate(sourceGame(1)), null))); + } + if (args[0] === 'push') { pushes++; throw new Error('rejected push'); } + }); + assert.deepEqual(counts, { scanned: 1, eligible: 1, published: 0, current: 1 }); + assert.equal(pushes, 1); + assert.equal(resets, 2); + assert.equal(fs.existsSync(outputFile), false); +}); + +test('failed pushes stop after five attempts without reporting unpublished changes', t => { + const { checkout } = syncFixture(t, [], { withGit: false }); + const file = path.join(checkout, 'database', 'games', '1.json'); + const error = new Error('push failed'); + let pushes = 0; + let commits = 0; + let notifications = 0; + assert.throws(() => publishBatch({ + checkout, games: [steamCandidate(sourceGame(1))], + onPublished: () => notifications++, + gitCommand: (directory, args) => { + if (args[0] === 'reset' && fs.existsSync(file)) fs.unlinkSync(file); + if (args[0] === 'commit') commits++; + if (args[0] === 'push') { pushes++; throw error; } + } + }), error); + assert.equal(pushes, 5); + assert.equal(commits, 5); + assert.equal(notifications, 0); +}); + test('preset ordering keeps numeric issue IDs before named imports', () => { assert.equal(comparePresetIds('2', '10'), -1); assert.equal(comparePresetIds('10', '2'), 1);