Skip to content

perf: batch GameDB sync pushes #13

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ReenigneArcher merged 1 commit into from
Oct 10, 2026
+224 −48
Merged

perf: batch GameDB sync pushes #13

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/sync-gamedb-steam.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ jobs:
set -euo pipefail
git -C database-branch config user.name "$BOT_NAME"
git -C database-branch config user.email "$BOT_EMAIL"
node src/gamedb-sync.js --gamedb gamedb --database database-branch
node src/gamedb-sync.js --gamedb gamedb --database database-branch --batch-size 500

- name: Build updated site
if: always() && steps.import.outputs.changed == 'true'
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ Game submissions resolve the submitted slug through [IGDB authenticated API](htt
- `auto_approved_users.json`: trusted approvers and game submitters eligible for automatic queueing.
- `.readthedocs.yaml`: pull request preview builds using the shared LizardByte Jekyll script.

The [GameDB Steam sync workflow](.github/workflows/sync-gamedb-steam.yml) runs daily at 07:00 UTC and can also be started manually. GameDB publishes its database on `gh-pages`, so the workflow reads that branch's PC game records and Steam external IDs. It writes eligible game files directly to the PresetDB `database` branch, with one file per commit and no issue or approval queue. Existing issue-backed Steam presets retain their numeric IDs, issue history, and contributor-managed launch values. GameDB changes to game metadata are synchronized; direct imports also receive Steam ID and command updates. Ambiguous Steam IDs are skipped. The job has a six-hour timeout; the next daily run skips unchanged files and continues the import.
The [GameDB Steam sync workflow](.github/workflows/sync-gamedb-steam.yml) runs daily at 07:00 UTC and can also be started manually. GameDB publishes its database on `gh-pages`, so the workflow reads that branch's PC game records and Steam external IDs. It writes eligible game files directly to the PresetDB `database` branch, with one file per commit and no issue or approval queue. Commits are pushed in batches of 500, with any remaining commits pushed at the end; the importer also accepts `--batch-size` to adjust this limit. Each batch fetches the database once, and rejected pushes retry against the latest records to preserve concurrent approvals. Existing issue-backed Steam presets retain their numeric IDs, issue history, and contributor-managed launch values. GameDB changes to game metadata are synchronized; direct imports also receive Steam ID and command updates. Ambiguous Steam IDs are skipped. The job has a six-hour timeout; the next daily run skips unchanged files and continues the import.

On each push to master, the Migrate Database workflow checks `database/migrations.json` for completed migration IDs. Before applying pending migrations, it creates a `database-backup-<pre-migration commit SHA>` branch pointing to the untouched database. It then commits the migrated records and history file to the active `database` branch and triggers a fresh Pages build. Re-runs skip completed migrations. The backup branch is never used as the working database.

Expand Down
53 changes: 36 additions & 17 deletions src/gamedb-sync.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,32 +77,51 @@ function git(checkout, args) {
return execFileSync('/usr/bin/git', args, { cwd: checkout, encoding: 'utf8', stdio: 'pipe' });
}

function publishGame({ checkout, game, file, onPublished, gitCommand = git }) {
const relative = `database/games/${game.id}.json`;
function publishBatch({ checkout, games, onPublished, gitCommand = git }) {
for (let attempt = 1; attempt <= 5; attempt++) {
gitCommand(checkout, ['fetch', 'origin', 'database']);
gitCommand(checkout, ['reset', '--hard', 'origin/database']);
const desired = mergeGame(game, readRecord(file));
if (!desired) return false;
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, `${JSON.stringify(desired, null, 2)}\n`);
gitCommand(checkout, ['add', '--', relative]);
gitCommand(checkout, ['commit', '-m', `chore: sync GameDB game ${game.id}`, '--', relative]);
let published = 0;
for (const game of games) {
const relative = `database/games/${game.id}.json`;
const file = path.join(checkout, relative);
const desired = mergeGame(game, readRecord(file));
if (!desired) continue;
fs.mkdirSync(path.dirname(file), { recursive: true });
fs.writeFileSync(file, `${JSON.stringify(desired, null, 2)}\n`);
gitCommand(checkout, ['add', '--', relative]);
gitCommand(checkout, ['commit', '-m', `chore: sync GameDB game ${game.id}`, '--', relative]);
published++;
}
if (!published) return 0;
try {
gitCommand(checkout, ['push', 'origin', 'HEAD:database']);
onPublished?.();
return true;
} catch (error) {
if (attempt === 5) throw error;
console.log(`Database branch moved while publishing game ${game.id}; retrying`);
console.log('Database branch moved while publishing GameDB batch; retrying');
continue;
}
onPublished?.();
return published;
}
}

function run({ gameDbDir, checkout, onPublished, gitCommand = git }) {
function run({ gameDbDir, checkout, batchSize = 500, onPublished, gitCommand = git }) {
if (!Number.isSafeInteger(batchSize) || batchSize < 1) {
throw new Error('Batch size must be a positive safe integer');
}
const database = path.join(checkout, 'database');
const index = JSON.parse(fs.readFileSync(path.join(gameDbDir, 'platforms', '6.json'), 'utf8'));
const counts = { scanned: 0, eligible: 0, published: 0, current: 0 };
let pending = [];
function flushBatch() {
if (!pending.length) return;
const published = publishBatch({ checkout, games: pending, onPublished, gitCommand });
counts.published += published;
counts.current += pending.length - published;
console.log(`Published GameDB batch: ${published} games`);
pending = [];
}
for (const id of pcGameIds(index)) {
counts.scanned++;
const source = JSON.parse(fs.readFileSync(path.join(gameDbDir, 'games', `${id}.json`), 'utf8'));
Expand All @@ -112,11 +131,10 @@ function run({ gameDbDir, checkout, onPublished, gitCommand = git }) {
counts.eligible++;
const file = path.join(database, 'games', `${id}.json`);
if (!mergeGame(game, readRecord(file))) { counts.current++; continue; }
if (publishGame({ checkout, game, file, onPublished, gitCommand })) {
counts.published++;
console.log(`Published game ${id}`);
} else counts.current++;
pending.push(game);
if (pending.length === batchSize) flushBatch();
}
flushBatch();
console.log(`GameDB Steam sync: ${JSON.stringify(counts)}`);
return counts;
}
Expand All @@ -131,6 +149,7 @@ function main(args = process.argv.slice(2), env = process.env, gitCommand = git)
return run({
gameDbDir: path.resolve(option('--gamedb')),
checkout: path.resolve(option('--database')),
batchSize: args.includes('--batch-size') ? Number(option('--batch-size')) : 500,
gitCommand,
onPublished: () => {
if (!reported && env.GITHUB_OUTPUT) {
Expand All @@ -141,6 +160,6 @@ function main(args = process.argv.slice(2), env = process.env, gitCommand = git)
});
}

module.exports = { pcGameIds, steamCandidate, mergeGame, publishGame, run, main };
module.exports = { pcGameIds, steamCandidate, mergeGame, publishBatch, run, main };

if (require.main === module) main();
215 changes: 186 additions & 29 deletions tests/gamedb-sync.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { execFileSync } = require('node:child_process');
const { pcGameIds, steamCandidate, mergeGame, run, main } = require('../src/gamedb-sync');
const { pcGameIds, steamCandidate, mergeGame, publishBatch, run, main } = require('../src/gamedb-sync');
const { mergePreset } = require('../src/database');
const { buildSite } = require('../src/build-site');
const { comparePresetIds } = require('../src/record');
Expand All @@ -23,6 +23,34 @@ function command(cwd, ...args) {
return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: 'pipe' }).trim();
}

function syncFixture(t, ids, { withGit = true } = {}) {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'preset-gamedb-sync-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
const remote = path.join(root, 'remote.git');
const checkout = path.join(root, 'checkout');
const gameDbDir = path.join(root, 'gamedb');
fs.mkdirSync(path.join(checkout, 'database', 'games'), { recursive: true });
fs.mkdirSync(path.join(gameDbDir, 'platforms'), { recursive: true });
fs.mkdirSync(path.join(gameDbDir, 'games'));
if (withGit) {
command(root, 'init', '--bare', remote);
command(checkout, 'init', '-b', 'database');
command(checkout, 'remote', 'add', 'origin', remote);
command(checkout, 'config', 'user.name', 'LizardByte-bot');
command(checkout, 'config', 'user.email', 'bot@example.com');
fs.writeFileSync(path.join(checkout, 'database', 'games', '.gitkeep'), '');
command(checkout, 'add', 'database/games/.gitkeep');
command(checkout, 'commit', '-m', 'seed database');
command(checkout, 'push', '-u', 'origin', 'database');
}
fs.writeFileSync(path.join(gameDbDir, 'platforms', '6.json'),
JSON.stringify({ id: 6, games: ids.map(id => ({ id })) }));
for (const id of ids) {
fs.writeFileSync(path.join(gameDbDir, 'games', `${id}.json`), JSON.stringify(sourceGame(id)));
}
return { root, remote, checkout, gameDbDir };
}

test('PC index and Steam source select one unambiguous launch ID', () => {
assert.deepEqual(pcGameIds({ id: 6, games: [{ id: 2 }, { id: 1 }, { id: 2 }] }), [1, 2]);
assert.throws(() => pcGameIds({ id: 5, games: [] }), /PC platform index/);
Expand Down Expand Up @@ -70,43 +98,29 @@ test('direct import has no issue identity and refreshes only imported Steam laun
});

test('sync publishes each changed game file in its own commit and skips unchanged files', async t => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'preset-gamedb-sync-'));
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
const remote = path.join(root, 'remote.git');
const checkout = path.join(root, 'checkout');
const gameDbDir = path.join(root, 'gamedb');
fs.mkdirSync(checkout);
fs.mkdirSync(path.join(gameDbDir, 'platforms'), { recursive: true });
fs.mkdirSync(path.join(gameDbDir, 'games'));
command(root, 'init', '--bare', remote);
command(checkout, 'init', '-b', 'database');
command(checkout, 'remote', 'add', 'origin', remote);
command(checkout, 'config', 'user.name', 'LizardByte-bot');
command(checkout, 'config', 'user.email', 'bot@example.com');
fs.mkdirSync(path.join(checkout, 'database', 'games'), { recursive: true });
fs.writeFileSync(path.join(checkout, 'database', 'games', '.gitkeep'), '');
command(checkout, 'add', 'database/games/.gitkeep');
command(checkout, 'commit', '-m', 'seed database');
command(checkout, 'push', '-u', 'origin', 'database');

fs.writeFileSync(path.join(gameDbDir, 'platforms', '6.json'),
JSON.stringify({ id: 6, games: [{ id: 1 }, { id: 2 }] }));
for (const id of [1, 2]) {
fs.writeFileSync(path.join(gameDbDir, 'games', `${id}.json`), JSON.stringify(sourceGame(id)));
}

const args = { gameDbDir, checkout };
const { root, checkout, gameDbDir } = syncFixture(t, [1, 2]);
const calls = [];
const gitCommand = (directory, gitArgs) => {
calls.push(gitArgs);
return command(directory, ...gitArgs);
};
const args = { gameDbDir, checkout, gitCommand };
const outputFile = path.join(root, 'github-output');
assert.throws(() => main([], {}, () => {}), /Missing --gamedb/);
assert.equal(main(['--gamedb', gameDbDir, '--database', checkout],
{ GITHUB_OUTPUT: outputFile }, (directory, gitArgs) => command(directory, ...gitArgs)).published, 2);
{ GITHUB_OUTPUT: outputFile }, gitCommand).published, 2);
assert.equal(fs.readFileSync(outputFile, 'utf8'), 'changed=true\n');
assert.equal(calls.filter(args => args[0] === 'fetch').length, 1);
assert.equal(calls.filter(args => args[0] === 'reset').length, 1);
assert.equal(calls.filter(args => args[0] === 'push').length, 1);
assert.equal(command(checkout, 'rev-list', '--count', 'HEAD'), '3');
assert.equal(command(checkout, 'diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD'),
'database/games/2.json');
assert.equal(command(checkout, 'diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD^'),
'database/games/1.json');
assert.equal(run(args).published, 0);
calls.length = 0;
assert.deepEqual(run(args), { scanned: 2, eligible: 2, published: 0, current: 2 });
assert.deepEqual(calls, []);
assert.equal(command(checkout, 'rev-list', '--count', 'HEAD'), '3');
const imported = JSON.parse(fs.readFileSync(path.join(checkout, 'database/games/1.json')));
assert.throws(() => mergePreset(path.join(checkout, 'database'), {
Expand All @@ -126,6 +140,149 @@ test('sync publishes each changed game file in its own commit and skips unchange
assert.equal(siteRecord.presets[0].protondb_url, 'https://www.protondb.com/app/1001');
});

test('default batches push 500 commits and flush the remainder, excluding current and ineligible games', t => {
const ids = Array.from({ length: 1004 }, (_, index) => index + 1);
const { checkout, gameDbDir } = syncFixture(t, ids, { withGit: false });
for (const id of [2, 501]) {
fs.writeFileSync(path.join(checkout, 'database', 'games', `${id}.json`),
JSON.stringify(mergeGame(steamCandidate(sourceGame(id)), null)));
}
fs.writeFileSync(path.join(gameDbDir, 'games', '1004.json'),
JSON.stringify({ ...sourceGame(1004), platforms: [3] }));
let commits = 0;
let fetches = 0;
let resets = 0;
let notifications = 0;
const pushes = [];
const counts = run({
gameDbDir, checkout,
onPublished: () => notifications++,
gitCommand: (directory, args) => {
assert.equal(directory, checkout);
if (args[0] === 'fetch') fetches++;
if (args[0] === 'reset') resets++;
if (args[0] === 'commit') commits++;
if (args[0] === 'push') { pushes.push(commits); commits = 0; }
}
});
assert.deepEqual(counts, { scanned: 1004, eligible: 1003, published: 1001, current: 2 });
assert.deepEqual(pushes, [500, 500, 1]);
assert.equal(fetches, 3);
assert.equal(resets, 3);
assert.equal(notifications, 3);
});

test('CLI batch size controls pushes and publishes all commits to a real remote', t => {
const { root, remote, checkout, gameDbDir } = syncFixture(t, [1, 2, 3, 4, 5, 6, 7]);
const outputFile = path.join(root, 'github-output');
const remoteCounts = [];
const gitCommand = (directory, args) => {
const result = command(directory, ...args);
if (args[0] === 'push') {
remoteCounts.push(Number(command(root, '--git-dir', remote, 'rev-list', '--count', 'database')));
}
return result;
};
const args = ['--gamedb', gameDbDir, '--database', checkout];
assert.throws(() => main([...args, '--batch-size'], {}, gitCommand), /Missing --batch-size/);
for (const batchSize of ['0', '-1', '1.5', 'invalid', 'Infinity', '9007199254740992']) {
assert.throws(() => main([...args, '--batch-size', batchSize], {}, gitCommand),
/Batch size must be a positive safe integer/);
}
assert.equal(main([...args, '--batch-size', '3'], { GITHUB_OUTPUT: outputFile }, gitCommand).published, 7);
assert.deepEqual(remoteCounts, [4, 7, 8]);
assert.equal(fs.readFileSync(outputFile, 'utf8'), 'changed=true\n');
assert.equal(command(checkout, 'rev-parse', 'HEAD'),
command(root, '--git-dir', remote, 'rev-parse', 'database'));
});

test('rejected batch is rebuilt from concurrent approvals and skips games already imported remotely', t => {
const { root, remote, checkout, gameDbDir } = syncFixture(t, [1, 2, 3]);
const peer = path.join(root, 'peer');
command(root, 'clone', '--branch', 'database', remote, peer);
command(peer, 'config', 'user.name', 'Contributor');
command(peer, 'config', 'user.email', 'contributor@example.com');
const approved = mergeGame(steamCandidate(sourceGame(2)), null);
approved.name = 'Contributor title';
approved.presets[0] = {
id: '42', name: approved.name, os: null, method: 'steam', launch_id: '9999',
commands_by_os: { Windows: 'custom command' }, origin_issue: 42, source_issue: 42,
history: [{ issue: 42, action: 'add' }]
};
const unrelated = { contributor: 'retained' };
const outputFile = path.join(root, 'github-output');
let pushes = 0;
let fetches = 0;
const gitCommand = (directory, args) => {
if (args[0] === 'fetch') fetches++;
if (args[0] === 'push' && ++pushes === 1) {
fs.writeFileSync(path.join(peer, 'database', 'games', '1.json'),
JSON.stringify(mergeGame(steamCandidate(sourceGame(1)), null), null, 2) + '\n');
fs.writeFileSync(path.join(peer, 'database', 'games', '2.json'), JSON.stringify(approved));
fs.writeFileSync(path.join(peer, 'database', 'contributor.json'), JSON.stringify(unrelated));
command(peer, 'add', 'database');
command(peer, 'commit', '-m', 'concurrent approval');
command(peer, 'push', 'origin', 'database');
}
return command(directory, ...args);
};
assert.deepEqual(main(['--gamedb', gameDbDir, '--database', checkout],
{ GITHUB_OUTPUT: outputFile }, gitCommand),
{ scanned: 3, eligible: 3, published: 2, current: 1 });
assert.equal(fetches, 2);
assert.equal(pushes, 2);
assert.equal(command(checkout, 'rev-list', '--count', 'HEAD'), '4');
assert.equal(command(checkout, 'log', '-2', '--format=%s'),
'chore: sync GameDB game 3\nchore: sync GameDB game 2');
const record = JSON.parse(fs.readFileSync(path.join(checkout, 'database', 'games', '2.json')));
assert.deepEqual(record, {
...approved, name: 'Game 2', presets: [{ ...approved.presets[0], name: 'Game 2' }]
});
assert.deepEqual(JSON.parse(fs.readFileSync(path.join(checkout, 'database', 'contributor.json'))),
unrelated);
assert.equal(fs.readFileSync(outputFile, 'utf8'), 'changed=true\n');
});

test('a retry that finds the entire batch current avoids another push and changed output', t => {
const { root, checkout, gameDbDir } = syncFixture(t, [1], { withGit: false });
const file = path.join(checkout, 'database', 'games', '1.json');
const outputFile = path.join(root, 'github-output');
let pushes = 0;
let resets = 0;
const counts = main(['--gamedb', gameDbDir, '--database', checkout],
{ GITHUB_OUTPUT: outputFile }, (directory, args) => {
if (args[0] === 'reset' && ++resets === 2) {
fs.writeFileSync(file, JSON.stringify(mergeGame(steamCandidate(sourceGame(1)), null)));
}
if (args[0] === 'push') { pushes++; throw new Error('rejected push'); }
});
assert.deepEqual(counts, { scanned: 1, eligible: 1, published: 0, current: 1 });
assert.equal(pushes, 1);
assert.equal(resets, 2);
assert.equal(fs.existsSync(outputFile), false);
});

test('failed pushes stop after five attempts without reporting unpublished changes', t => {
const { checkout } = syncFixture(t, [], { withGit: false });
const file = path.join(checkout, 'database', 'games', '1.json');
const error = new Error('push failed');
let pushes = 0;
let commits = 0;
let notifications = 0;
assert.throws(() => publishBatch({
checkout, games: [steamCandidate(sourceGame(1))],
onPublished: () => notifications++,
gitCommand: (directory, args) => {
if (args[0] === 'reset' && fs.existsSync(file)) fs.unlinkSync(file);
if (args[0] === 'commit') commits++;
if (args[0] === 'push') { pushes++; throw error; }
}
}), error);
assert.equal(pushes, 5);
assert.equal(commits, 5);
assert.equal(notifications, 0);
});

test('preset ordering keeps numeric issue IDs before named imports', () => {
assert.equal(comparePresetIds('2', '10'), -1);
assert.equal(comparePresetIds('10', '2'), 1);
Expand Down
Loading