diff --git a/.gitignore b/.gitignore index 9db812842d..79159e264b 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ src/assets/ *debug*.txt eslint_out.txt tests/perf/output/ +tests/load/results/ # Locally built Go binary (gen-maps uses `go run .`) map-generator/map-generator diff --git a/eslint.config.js b/eslint.config.js index 3ac29c3e3a..d0e3f8090f 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -33,6 +33,7 @@ export default [ "__mocks__/fileMock.js", "eslint.config.js", "scripts/sync-assets.mjs", + "tests/load/*.mjs", "tests/matchmaking/*.mjs", ], }, diff --git a/package.json b/package.json index d33064e650..3f35a9cf7b 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,10 @@ "build-prod": "concurrently --kill-others-on-fail \"tsc --noEmit\" \"vite build\"", "start:client": "vite", "start:server": "tsx src/server/Server.ts", + "start:server:bun": "bun src/server/Server.ts", "start:server-dev": "cross-env GAME_ENV=dev NUM_WORKERS=2 TURNSTILE_SITE_KEY=1x00000000000000000000AA API_KEY=WARNING_DEV_API_KEY_DO_NOT_USE_IN_PRODUCTION ADMIN_BOT_API_KEY=WARNING_DEV_ADMIN_BOT_KEY_DO_NOT_USE_IN_PRODUCTION DOMAIN=localhost GIT_COMMIT=DEV tsx src/server/Server.ts", + "start:server-dev:bun": "cross-env GAME_ENV=dev NUM_WORKERS=2 TURNSTILE_SITE_KEY=1x00000000000000000000AA API_KEY=WARNING_DEV_API_KEY_DO_NOT_USE_IN_PRODUCTION ADMIN_BOT_API_KEY=WARNING_DEV_ADMIN_BOT_KEY_DO_NOT_USE_IN_PRODUCTION DOMAIN=localhost GIT_COMMIT=DEV bun src/server/Server.ts", + "dev:bun": "cross-env GAME_ENV=dev concurrently \"npm run start:client\" \"npm run start:server-dev:bun\"", "dev": "cross-env GAME_ENV=dev concurrently \"npm run start:client\" \"npm run start:server-dev\"", "dev:host": "cross-env GAME_ENV=dev VITE_HOST=lan concurrently \"npm run start:client\" \"npm run start:server-dev\"", "dev:staging": "cross-env GAME_ENV=dev API_DOMAIN=api.openfront.dev concurrently \"npm run start:client\" \"npm run start:server-dev\"", @@ -13,6 +16,9 @@ "docs:map-generator": "cd map-generator && go doc -cmd -u -all", "tunnel": "npm run build-prod && npm run start:server", "test": "vitest run && vitest run tests/server", + "test:e2e": "vitest run --config vitest.e2e.config.ts", + "test:e2e:bun": "cross-env E2E_RUNTIME=bun vitest run --config vitest.e2e.config.ts", + "perf:server": "node tests/load/loadtest.mjs", "test:matchmaking": "node tests/matchmaking/contained.mjs", "test:matchmaking:e2e": "node tests/matchmaking/e2e.mjs", "test:matchmaking:cancel": "node tests/matchmaking/e2e-cancel.mjs", diff --git a/src/server/Worker.ts b/src/server/Worker.ts index aa17d86230..576fc443e4 100644 --- a/src/server/Worker.ts +++ b/src/server/Worker.ts @@ -47,6 +47,16 @@ const playlist = new MapPlaylist(); export async function startWorker() { log.info(`Worker starting...`); + // Exit when the IPC channel to the master closes. Node's cluster workers + // already die with the master; Bun's do not — an orphaned worker would + // keep its port bound (SO_REUSEPORT) and serve stale state next to the + // restarted server's workers. Explicit exit makes the lifecycle identical + // on both runtimes. + process.on("disconnect", () => { + log.info("IPC channel to master closed, shutting down worker"); + process.exit(0); + }); + const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); diff --git a/src/server/WorkerLobbyService.ts b/src/server/WorkerLobbyService.ts index 01e34572dd..ebbd69cd7f 100644 --- a/src/server/WorkerLobbyService.ts +++ b/src/server/WorkerLobbyService.ts @@ -127,7 +127,13 @@ export class WorkerLobbyService { } private sendToMaster(msg: WorkerReady | WorkerLobbyList) { - process.send?.(msg); + // On Node a closed IPC channel makes process.send return false; on Bun + // it throws instead. Treat both as "master is gone, drop the message". + try { + process.send?.(msg); + } catch (error) { + this.log.warn(`Failed to send IPC message to master: ${error}`); + } } private sendMyLobbiesToMaster() { diff --git a/tests/e2e/GameFlow.test.ts b/tests/e2e/GameFlow.test.ts new file mode 100644 index 0000000000..0664285a4b --- /dev/null +++ b/tests/e2e/GameFlow.test.ts @@ -0,0 +1,238 @@ +// End-to-end test of the game server over real HTTP + WebSocket: boots the +// actual master + cluster workers (Node via tsx, or Bun with +// E2E_RUNTIME=bun) and drives the full lobby -> start -> turn-relay -> +// rejoin -> kick flow that production clients exercise. + +import { randomUUID } from "crypto"; +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { + createGame, + gameInfo, + RUNTIME, + sleep, + TestClient, + TestServer, + waitFor, +} from "./util"; + +describe(`game server e2e (runtime: ${RUNTIME})`, () => { + const server = new TestServer(); + const creatorToken = randomUUID(); + let game: { gameID: string; workerIndex: number; port: number }; + let creator: TestClient; + let playerB: TestClient; + let playerC: TestClient; + + beforeAll(async () => { + await server.start(); + game = await createGame(creatorToken); + creator = new TestClient(game.port, game.gameID, "creator", creatorToken); + playerB = new TestClient(game.port, game.gameID, "playerB"); + playerC = new TestClient(game.port, game.gameID, "playerC"); + }); + + afterAll(async () => { + for (const c of [creator, playerB, playerC]) c?.close(); + await server.stop(); + }); + + test("health endpoint reports ok once workers are ready", async () => { + const res = await fetch("http://127.0.0.1:3000/api/health"); + expect(res.ok).toBe(true); + expect(await res.json()).toEqual({ status: "ok" }); + }); + + test("create_game requires an auth token", async () => { + const res = await fetch(`http://127.0.0.1:${game.port}/api/create_game`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + }); + expect(res.status).toBe(400); + }); + + test("created game is queryable on its worker", async () => { + const info = await gameInfo(game.port, game.gameID); + expect(info).not.toBeNull(); + expect(info.gameID).toBe(game.gameID); + expect(info.gameConfig.gameType).toBe("Private"); + }); + + test("clients join over WebSocket and get server-assigned clientIDs", async () => { + await creator.join(); + await playerB.join(); + await playerC.join(); + expect(creator.clientID).toBeTruthy(); + expect(playerB.clientID).toBeTruthy(); + expect(playerC.clientID).toBeTruthy(); + // All three ids are distinct. + expect( + new Set([creator.clientID, playerB.clientID, playerC.clientID]).size, + ).toBe(3); + + // Lobby info converges to 3 clients for everyone. + await waitFor( + async () => { + const info = await gameInfo(game.port, game.gameID); + return info?.clients?.length === 3; + }, + 10_000, + "lobby to report 3 clients", + ); + }); + + test("lobby identifies the creator", async () => { + const info = await gameInfo(game.port, game.gameID); + expect(info.lobbyCreatorClientID).toBe(creator.clientID); + }); + + test("non-creator cannot start the game", async () => { + playerB.sendIntent({ type: "toggle_game_start_timer" }); + await sleep(1500); + const info = await gameInfo(game.port, game.gameID); + expect(info.startsAt ?? undefined).toBeUndefined(); + }); + + test("creator starts the game; every client receives prestart and start", async () => { + creator.sendIntent({ type: "toggle_game_start_timer" }); + for (const c of [creator, playerB, playerC]) { + await c.waitForMessage((m) => m.type === "prestart", 15_000); + const start = (await c.waitForMessage( + (m) => m.type === "start", + 15_000, + )) as any; + expect(start.gameStartInfo.gameID).toBe(game.gameID); + expect(start.gameStartInfo.players).toHaveLength(3); + expect(start.myClientID).toBe(c.clientID); + const usernames = start.gameStartInfo.players.map((p: any) => p.username); + expect(usernames).toEqual( + expect.arrayContaining(["creator", "playerB", "playerC"]), + ); + } + }); + + test("server broadcasts turns at the 100ms tick", async () => { + const before = creator.turns().length; + await sleep(1200); + const after = creator.turns().length; + const gained = after - before; + // ~12 expected; allow generous slack for CI jitter. + expect(gained).toBeGreaterThanOrEqual(8); + expect(gained).toBeLessThanOrEqual(16); + // Turn numbers are consecutive. + const numbers = creator.turns().map((t) => t.turn.turnNumber); + for (let i = 1; i < numbers.length; i++) { + expect(numbers[i]).toBe(numbers[i - 1] + 1); + } + }); + + test("an intent is relayed to every client, stamped with the sender's clientID", async () => { + const marker = 987654321; // distinctive troops value to find the intent + playerB.sendIntent({ type: "attack", targetID: null, troops: marker }); + for (const c of [creator, playerB, playerC]) { + const turnMsg = (await c.waitForMessage( + (m) => + m.type === "turn" && + (m as any).turn.intents.some((i: any) => i.troops === marker), + 5_000, + )) as any; + const intent = turnMsg.turn.intents.find((i: any) => i.troops === marker); + // The clientID comes from the authenticated connection, not the payload. + expect(intent.clientID).toBe(playerB.clientID); + } + }); + + test("a client that vanishes can rejoin and receives the missed turns", async () => { + // Hard-drop C's socket (no close frame ≈ network loss). + playerC.ws!.terminate(); + await sleep(600); + const lastTurn = + playerC.turns().length === 0 + ? 0 + : playerC.turns()[playerC.turns().length - 1].turn.turnNumber + 1; + + const rejoined = new TestClient( + game.port, + game.gameID, + "playerC", + playerC.token, + ); + await rejoined.connect(); + rejoined.send({ + type: "rejoin", + gameID: game.gameID, + lastTurn, + token: playerC.token, + }); + const start = (await rejoined.waitForMessage( + (m) => m.type === "start", + 10_000, + )) as any; + // The catch-up slice starts exactly where the client left off. + expect(start.gameStartInfo.gameID).toBe(game.gameID); + if (start.turns.length > 0) { + expect(start.turns[0].turnNumber).toBe(lastTurn); + } + // Same identity as before the drop. + expect(start.myClientID).toBe(playerC.clientID); + playerC = rejoined; + playerC.clientID = start.myClientID; + }); + + test("a client sending garbage is kicked with a reason", async () => { + const victim = new TestClient(game.port, game.gameID, "victim"); + await victim.join(); + victim.ws!.send("this is not json"); + await victim.waitForMessage( + (m) => m.type === "error" && (m as any).error.includes("invalid_message"), + 5_000, + ); + await waitFor( + () => victim.closeCode !== null, + 5_000, + "victim socket to close", + ); + expect(victim.closeCode).toBe(1000); + + // Kicked identity cannot rejoin. + const comeback = new TestClient( + game.port, + game.gameID, + "victim", + victim.token, + ); + await comeback.connect(); + comeback.send({ + type: "join", + token: comeback.token, + gameID: game.gameID, + username: "victim", + clanTag: null, + turnstileToken: null, + }); + await waitFor( + () => comeback.closeCode !== null, + 5_000, + "kicked rejoin to be rejected", + ); + expect(comeback.closeCode).toBe(1002); + }); + + test("games on the wrong worker are rejected", async () => { + // Join a game that lives on worker A via worker B's port: the message + // is dropped (no lobby_info ever arrives). + const wrongPort = game.port === 3001 ? 3002 : 3001; + const lost = new TestClient(wrongPort, game.gameID, "lostsoul"); + await lost.connect(); + lost.send({ + type: "join", + token: lost.token, + gameID: game.gameID, + username: "lostsoul", + clanTag: null, + turnstileToken: null, + }); + await sleep(1500); + expect(lost.messages.find((m) => m.type === "lobby_info")).toBeUndefined(); + lost.close(); + }); +}); diff --git a/tests/e2e/WorkerLifecycle.test.ts b/tests/e2e/WorkerLifecycle.test.ts new file mode 100644 index 0000000000..5f31a31f2c --- /dev/null +++ b/tests/e2e/WorkerLifecycle.test.ts @@ -0,0 +1,50 @@ +// Regression test for the cluster worker lifecycle: when the master dies, +// every worker must exit too. Node's cluster does this implicitly; Bun's +// does not, which orphaned workers that kept their ports bound (via +// SO_REUSEPORT) and served stale state next to a restarted server. Worker.ts +// now exits explicitly on IPC disconnect — this test pins that behavior on +// both runtimes. + +import { afterAll, beforeAll, describe, expect, test } from "vitest"; +import { RUNTIME, TestServer, waitFor } from "./util"; + +function pidAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +describe(`worker lifecycle (runtime: ${RUNTIME})`, () => { + const server = new TestServer(); + + beforeAll(async () => { + await server.start(); + }); + + afterAll(async () => { + await server.stop(); + }); + + test("workers exit when the master is killed", async () => { + const workerPids = server.workerPids(); + expect(workerPids.length).toBeGreaterThanOrEqual(2); + for (const pid of workerPids) { + expect(pidAlive(pid)).toBe(true); + } + + const masterPid = server.masterPid(); + expect(masterPid).not.toBeNull(); + // SIGKILL: the harshest case — no signal handler can run, only the IPC + // channel closing tells the workers their master is gone. + process.kill(masterPid!, "SIGKILL"); + + await waitFor( + () => workerPids.every((pid) => !pidAlive(pid)), + 10_000, + `workers ${workerPids.join(",")} to exit after master SIGKILL`, + ); + }); +}); diff --git a/tests/e2e/util.ts b/tests/e2e/util.ts new file mode 100644 index 0000000000..de656710fc --- /dev/null +++ b/tests/e2e/util.ts @@ -0,0 +1,281 @@ +// Helpers for e2e tests that boot the real game server (master + cluster +// workers) as a child process and drive it over HTTP + WebSocket. +// +// The server runtime is selectable via E2E_RUNTIME=node|bun (default node), +// so the same suite verifies both the tsx/Node deployment and the Bun one. + +import { ChildProcess, spawn } from "child_process"; +import { randomUUID } from "crypto"; +import path from "path"; +import WebSocket from "ws"; +import type { ServerMessage } from "../../src/core/Schemas"; + +export const MASTER_PORT = 3000; +export const NUM_WORKERS = 2; +export const RUNTIME = process.env.E2E_RUNTIME === "bun" ? "bun" : "node"; + +const repoRoot = path.resolve(__dirname, "../.."); + +export const workerPort = (i: number) => 3001 + i; + +const serverEnv = { + ...process.env, + GAME_ENV: "dev", + NUM_WORKERS: String(NUM_WORKERS), + TURNSTILE_SITE_KEY: "1x00000000000000000000AA", + API_KEY: "WARNING_DEV_API_KEY_DO_NOT_USE_IN_PRODUCTION", + ADMIN_BOT_API_KEY: "WARNING_DEV_ADMIN_BOT_KEY_DO_NOT_USE_IN_PRODUCTION", + DOMAIN: "localhost", + GIT_COMMIT: "DEV", +}; + +function serverCommand(): [string, string[]] { + if (RUNTIME === "bun") { + return ["bun", ["src/server/Server.ts"]]; + } + return ["npx", ["tsx", "src/server/Server.ts"]]; +} + +export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +export async function waitFor( + fn: () => boolean | Promise, + timeoutMs: number, + label: string, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + if (await fn()) return; + if (Date.now() > deadline) { + throw new Error(`Timed out after ${timeoutMs}ms waiting for: ${label}`); + } + await sleep(200); + } +} + +async function portServesHttp(port: number): Promise { + try { + await fetch(`http://127.0.0.1:${port}/`, { + signal: AbortSignal.timeout(1000), + }); + return true; + } catch { + return false; + } +} + +export class TestServer { + proc: ChildProcess | null = null; + logs: string[] = []; + + async start(): Promise { + // Fail fast if a stray server is already bound to our ports — under + // Bun's cluster (SO_REUSEPORT) a stray would silently absorb traffic. + for (let p = MASTER_PORT; p <= workerPort(NUM_WORKERS - 1); p++) { + if (await portServesHttp(p)) { + throw new Error(`port ${p} already serving HTTP; kill strays first`); + } + } + const [cmd, args] = serverCommand(); + this.proc = spawn(cmd, args, { + cwd: repoRoot, + env: serverEnv, + detached: true, // own process group so stop() can kill the whole tree + stdio: ["ignore", "pipe", "pipe"], + }); + this.proc.stdout!.on("data", (d) => this.logs.push(String(d))); + this.proc.stderr!.on("data", (d) => this.logs.push(String(d))); + + await waitFor( + async () => { + try { + const res = await fetch( + `http://127.0.0.1:${MASTER_PORT}/api/health`, + { signal: AbortSignal.timeout(1000) }, + ); + return res.ok; + } catch { + return false; + } + }, + 60_000, + "server health", + ); + } + + // PIDs (from the server's own logs) of the cluster worker processes. + workerPids(): number[] { + const pids: number[] = []; + for (const line of this.logs) { + for (const m of line.matchAll(/Started worker \d+ \(PID: (\d+)\)/g)) { + pids.push(Number(m[1])); + } + } + return pids; + } + + masterPid(): number | null { + for (const line of this.logs) { + const m = line.match(/Primary (\d+) is running/); + if (m) return Number(m[1]); + } + return null; + } + + async stop(): Promise { + if (this.proc?.pid) { + try { + process.kill(-this.proc.pid, "SIGTERM"); + } catch { + // already dead + } + } + // Wait until every port is released so the next suite can bind. + await waitFor( + async () => { + for (let p = MASTER_PORT; p <= workerPort(NUM_WORKERS - 1); p++) { + if (await portServesHttp(p)) return false; + } + return true; + }, + 15_000, + "server ports released", + ).catch(() => { + // Last resort: SIGKILL the group. + if (this.proc?.pid) { + try { + process.kill(-this.proc.pid, "SIGKILL"); + } catch { + // group already gone + } + } + }); + this.proc = null; + } +} + +export async function createGame( + creatorToken: string, + config: object = {}, + onWorker = 0, +): Promise<{ gameID: string; workerIndex: number; port: number }> { + const res = await fetch( + `http://127.0.0.1:${workerPort(onWorker)}/api/create_game`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${creatorToken}`, + }, + body: JSON.stringify(config), + }, + ); + if (!res.ok) { + throw new Error(`create_game failed: ${res.status} ${await res.text()}`); + } + const info = await res.json(); + return { + gameID: info.gameID, + workerIndex: info.workerIndex, + port: workerPort(info.workerIndex), + }; +} + +export async function gameInfo(port: number, gameID: string): Promise { + const res = await fetch(`http://127.0.0.1:${port}/api/game/${gameID}`); + if (!res.ok) return null; + return res.json(); +} + +// A WebSocket game client that records every server message and exposes +// promise-based waits, so tests read as: join → waitFor("start") → assert. +export class TestClient { + token: string; + ws: WebSocket | null = null; + messages: ServerMessage[] = []; + clientID: string | null = null; + closeCode: number | null = null; + closeReason = ""; + + constructor( + public port: number, + public gameID: string, + public username: string, + token?: string, + ) { + this.token = token ?? randomUUID(); + } + + async join(): Promise { + await this.connect(); + this.send({ + type: "join", + token: this.token, + gameID: this.gameID, + username: this.username, + clanTag: null, + turnstileToken: null, + }); + // The server-assigned clientID arrives in lobby_info (pre-start joins) + // or in the start message (late joins into a running game). + const msg = await this.waitForMessage( + (m) => m.type === "lobby_info" || m.type === "start", + 10_000, + ); + this.clientID = (msg as any).myClientID; + } + + connect(): Promise { + return new Promise((resolve, reject) => { + const ws = new WebSocket(`ws://127.0.0.1:${this.port}/`); + this.ws = ws; + ws.on("open", () => resolve()); + ws.on("error", (err) => reject(err)); + ws.on("message", (data) => { + try { + this.messages.push(JSON.parse(data.toString())); + } catch { + // not JSON; ignore + } + }); + ws.on("close", (code, reason) => { + this.closeCode = code; + this.closeReason = reason.toString(); + }); + }); + } + + send(obj: unknown): void { + this.ws!.send(JSON.stringify(obj)); + } + + sendIntent(intent: object): void { + this.send({ type: "intent", intent }); + } + + async waitForMessage( + pred: (m: ServerMessage) => boolean, + timeoutMs = 10_000, + ): Promise { + let found: ServerMessage | undefined; + await waitFor( + () => { + found = this.messages.find(pred); + return found !== undefined; + }, + timeoutMs, + `message matching predicate (got ${this.messages.length} messages)`, + ); + return found!; + } + + turns(): any[] { + return this.messages.filter((m) => m.type === "turn"); + } + + close(): void { + if (this.ws && this.ws.readyState === WebSocket.OPEN) { + this.ws.close(1000); + } + } +} diff --git a/tests/load/README.md b/tests/load/README.md new file mode 100644 index 0000000000..507de8b010 --- /dev/null +++ b/tests/load/README.md @@ -0,0 +1,80 @@ +# Server WebSocket load test + +Measures the game server's hot path — WebSocket intent ingestion and the +100ms turn broadcast — against the **real** server (master + cluster +workers), with a selectable server runtime. + +```bash +npm run perf:server # Node (tsx) server, default profile +npm run perf:server -- --runtime bun # same profile, Bun server +npm run perf:server -- --games 10 --clients 40 --duration 60 --intent-rate 2 +npm run perf:server -- --attach --workers 1 # drive an already-running server +``` + +The load generator always runs under plain Node so the measuring instrument +stays constant across server runtimes. Results are printed and written to +`tests/load/results/