From 568848b8f71e723c79b889c122eb2426b50d477d Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Sat, 26 Sep 2026 09:49:17 -0400 Subject: [PATCH 1/2] bug: fix panel bugs #626, #633 - #626: only rate players who actually played; benched subs of a full side are no longer marked as no-shows - #633: keep the tv stream and relay up until viewers have seen the end of the match --- hasura/functions/match/match_player_elo.sql | 64 +++++++++- .../game-streamer/game-streamer.service.ts | 17 ++- src/matches/jobs/CancelExpiredMatches.spec.ts | 30 +++++ src/matches/jobs/CancelExpiredMatches.ts | 23 +++- src/matches/jobs/StopMatchBroadcast.spec.ts | 79 +++++++++++++ src/matches/jobs/StopMatchBroadcast.ts | 44 +++++++ .../match-assistant.service.ts | 2 +- .../matches.controller.match-events.spec.ts | 110 ++++++++++++++++-- src/matches/matches.controller.ts | 109 ++++++++++++++--- src/matches/matches.module.ts | 2 + test/elo.spec.ts | 109 ++++++++++++++++- 11 files changed, 552 insertions(+), 37 deletions(-) create mode 100644 src/matches/jobs/StopMatchBroadcast.spec.ts create mode 100644 src/matches/jobs/StopMatchBroadcast.ts diff --git a/hasura/functions/match/match_player_elo.sql b/hasura/functions/match/match_player_elo.sql index 50d56e9f..0fe34c23 100644 --- a/hasura/functions/match/match_player_elo.sql +++ b/hasura/functions/match/match_player_elo.sql @@ -18,6 +18,60 @@ BEGIN END; $$ LANGUAGE plpgsql STABLE; +-- A lineup carries its substitutes whether or not they ever joined the server, +-- so lineup membership alone cannot tell who played. Anything the game server +-- recorded against a player in this match can. +CREATE OR REPLACE FUNCTION public.player_has_match_activity( + _match_id UUID, + _steam_id BIGINT +) RETURNS BOOLEAN AS $$ + SELECT EXISTS (SELECT 1 FROM player_kills WHERE match_id = _match_id AND attacker_steam_id = _steam_id) + OR EXISTS (SELECT 1 FROM player_kills WHERE match_id = _match_id AND attacked_steam_id = _steam_id) + OR EXISTS (SELECT 1 FROM player_damages WHERE match_id = _match_id AND attacker_steam_id = _steam_id) + OR EXISTS (SELECT 1 FROM player_damages WHERE match_id = _match_id AND attacked_steam_id = _steam_id) + OR EXISTS (SELECT 1 FROM player_assists WHERE match_id = _match_id AND attacker_steam_id = _steam_id) + OR EXISTS (SELECT 1 FROM player_flashes WHERE match_id = _match_id AND attacker_steam_id = _steam_id) + OR EXISTS (SELECT 1 FROM player_utility WHERE match_id = _match_id AND attacker_steam_id = _steam_id) + OR EXISTS (SELECT 1 FROM player_objectives WHERE match_id = _match_id AND player_steam_id = _steam_id) + OR EXISTS (SELECT 1 FROM player_unused_utility WHERE match_id = _match_id AND player_steam_id = _steam_id); +$$ LANGUAGE sql STABLE; + +-- The players a match is rated over: everyone who played, plus anyone who +-- abandoned it so the leaver penalty still lands on a player who never +-- connected. A lineup with no recorded activity at all (a no-show forfeit, a +-- match whose events never arrived) keeps every member, since there is nothing +-- to tell its players apart by. +CREATE OR REPLACE FUNCTION public.match_elo_participants( + _match_id UUID +) RETURNS TABLE (steam_id BIGINT, match_lineup_id UUID) AS $$ + WITH lineup_players AS ( + SELECT + mlp.steam_id AS player_steam_id, + mlp.match_lineup_id AS lineup_id, + public.player_has_match_activity(_match_id, mlp.steam_id) AS played + FROM matches m + JOIN match_lineup_players mlp + ON mlp.match_lineup_id IN (m.lineup_1_id, m.lineup_2_id) + WHERE m.id = _match_id + AND mlp.steam_id IS NOT NULL + ) + SELECT DISTINCT lp.player_steam_id, lp.lineup_id + FROM lineup_players lp + WHERE lp.played + OR EXISTS ( + SELECT 1 + FROM abandoned_matches am + WHERE am.match_id = _match_id + AND am.steam_id = lp.player_steam_id + ) + OR NOT EXISTS ( + SELECT 1 + FROM lineup_players teammate + WHERE teammate.lineup_id = lp.lineup_id + AND teammate.played + ); +$$ LANGUAGE sql STABLE; + CREATE OR REPLACE FUNCTION get_player_elo_for_match( match_record public.matches, player_record public.players, @@ -62,9 +116,14 @@ DECLARE _player_map_wins INT := 0; _player_map_losses INT := 0; _series_multiplier INT := 1; + + _participants BIGINT[]; BEGIN SELECT "type" INTO match_type FROM match_options WHERE id = match_record.match_options_id; + SELECT array_agg(mep.steam_id) INTO _participants + FROM public.match_elo_participants(match_record.id) mep; + _seasons_enabled := seasons_enabled(); -- Get the player's current ELO value from the most recent record @@ -187,6 +246,7 @@ BEGIN match_lineup_players mlp WHERE mlp.match_lineup_id = _player_lineup_id + AND (mlp.steam_id IS NULL OR mlp.steam_id = ANY(_participants)) GROUP BY mlp.steam_id ) AS team_elos; @@ -241,6 +301,7 @@ BEGIN match_lineup_players mlp WHERE mlp.match_lineup_id = _opponent_lineup_id + AND (mlp.steam_id IS NULL OR mlp.steam_id = ANY(_participants)) GROUP BY mlp.steam_id ) AS team_elos; @@ -482,8 +543,7 @@ BEGIN FOR player_record IN SELECT DISTINCT p.* FROM players p - JOIN match_lineup_players mlp ON p.steam_id = mlp.steam_id - WHERE mlp.match_lineup_id IN (match_record.lineup_1_id, match_record.lineup_2_id) + JOIN public.match_elo_participants(_match_id) mep ON p.steam_id = mep.steam_id LOOP -- Calculate ELO change for this player in this match elo_data := get_player_elo_for_match(match_record, player_record, _season_id, _is_tournament); diff --git a/src/matches/game-streamer/game-streamer.service.ts b/src/matches/game-streamer/game-streamer.service.ts index 37f3c319..d0df380f 100644 --- a/src/matches/game-streamer/game-streamer.service.ts +++ b/src/matches/game-streamer/game-streamer.service.ts @@ -1865,7 +1865,9 @@ export class GameStreamerService { } } - public async stopLiveIfRunning(matchId: string) { + public async getLiveStreamMode( + matchId: string, + ): Promise<"live" | "tv" | null> { const { match_streams } = await this.hasura.query({ match_streams: { __args: { @@ -1875,15 +1877,24 @@ export class GameStreamerService { }, limit: 1, }, - id: true, + mode: true, }, }); if (!match_streams?.length) { - return; + return null; + } + + return match_streams[0].mode === "tv" ? "tv" : "live"; + } + + public async stopLiveIfRunning(matchId: string): Promise { + if (!(await this.getLiveStreamMode(matchId))) { + return false; } await this.stopLive(matchId); + return true; } public async switchLive( diff --git a/src/matches/jobs/CancelExpiredMatches.spec.ts b/src/matches/jobs/CancelExpiredMatches.spec.ts index 87c2d1ad..341f2d78 100644 --- a/src/matches/jobs/CancelExpiredMatches.spec.ts +++ b/src/matches/jobs/CancelExpiredMatches.spec.ts @@ -373,6 +373,36 @@ describe("CancelExpiredMatches", () => { expect(penalised).not.toContain("showed-up"); }); + it("does not penalise the benched substitute of a side that showed up in full", async () => { + tournamentMatches = [ + expiredTournamentMatch({ + is_tournament_match: false, + options: { match_mode: "auto", type: "Wingman" }, + lineup_1: { + id: "lineup-1", + is_ready: true, + lineup_players: [ + { steam_id: "starter-a", is_connected: true }, + { steam_id: "starter-b", is_connected: true }, + { steam_id: "benched-sub", is_connected: false }, + ], + }, + lineup_2: { + id: "lineup-2", + is_ready: false, + lineup_players: [ + { steam_id: "showed-up", is_connected: true }, + { steam_id: "no-show", is_connected: false }, + ], + }, + }), + ]; + + await job.process(); + + expect(abandonedFor()).toEqual(["no-show"]); + }); + it("penalises nobody when no server was ever assigned", async () => { tournamentMatches = [ expiredTournamentMatch({ diff --git a/src/matches/jobs/CancelExpiredMatches.ts b/src/matches/jobs/CancelExpiredMatches.ts index 18e9b545..81626829 100644 --- a/src/matches/jobs/CancelExpiredMatches.ts +++ b/src/matches/jobs/CancelExpiredMatches.ts @@ -9,6 +9,7 @@ import { AppConfig } from "../../configs/types/AppConfig"; import { RconService } from "../../rcon/rcon.service"; import { DISCORD_COLORS } from "../../notifications/utilities/constants"; import { MatchAssistantService } from "../match-assistant/match-assistant.service"; +import { ExpectedPlayers } from "../../discord-bot/enums/ExpectedPlayers"; @UseQueue("Matches", MatchQueues.ScheduledMatches) export class CancelExpiredMatches extends WorkerHost { @@ -310,12 +311,21 @@ export class CancelExpiredMatches extends WorkerHost { return; } - const noShows = [ - ...(match.lineup_1.lineup_players ?? []), - ...(match.lineup_2.lineup_players ?? []), - ].filter( - (lineupPlayer) => lineupPlayer.steam_id && !lineupPlayer.is_connected, - ); + // A side that had a full team connected never needed its substitutes, so + // a benched sub who stayed away did not no-show. + const fullSide = ExpectedPlayers[match.options.type] / 2; + const noShows = [match.lineup_1, match.lineup_2].flatMap((lineup) => { + const lineupPlayers = lineup.lineup_players ?? []; + const connected = lineupPlayers.filter( + (lineupPlayer) => lineupPlayer.is_connected, + ).length; + if (connected >= fullSide) { + return []; + } + return lineupPlayers.filter( + (lineupPlayer) => lineupPlayer.steam_id && !lineupPlayer.is_connected, + ); + }); if (noShows.length === 0) { return; @@ -529,6 +539,7 @@ export class CancelExpiredMatches extends WorkerHost { winning_lineup_id: true, }, options: { + type: true, match_mode: true, }, lineup_1: { diff --git a/src/matches/jobs/StopMatchBroadcast.spec.ts b/src/matches/jobs/StopMatchBroadcast.spec.ts new file mode 100644 index 00000000..899f383a --- /dev/null +++ b/src/matches/jobs/StopMatchBroadcast.spec.ts @@ -0,0 +1,79 @@ +jest.mock("@kubernetes/client-node", () => ({ + BatchV1Api: class BatchV1Api {}, + CoreV1Api: class CoreV1Api {}, + KubeConfig: class KubeConfig {}, + Exec: class Exec {}, +})); + +import { StopMatchBroadcast } from "./StopMatchBroadcast"; + +describe("StopMatchBroadcast", () => { + let matchAssistant: { hasMatchEnded: jest.Mock }; + let gameStreamer: Record; + let matchRelay: { removeBroadcast: jest.Mock }; + let clips: { resumeAllPausedBatches: jest.Mock }; + let job: StopMatchBroadcast; + + beforeEach(() => { + matchAssistant = { hasMatchEnded: jest.fn(async () => true) }; + gameStreamer = { + stopLiveIfRunning: jest.fn(async () => true), + promotePendingLiveStreams: jest.fn(async () => ({ + promoted: [] as string[], + })), + }; + matchRelay = { removeBroadcast: jest.fn() }; + clips = { resumeAllPausedBatches: jest.fn() }; + job = new StopMatchBroadcast( + { log: jest.fn() } as any, + matchAssistant as any, + gameStreamer as any, + matchRelay as any, + clips as any, + ); + }); + + const run = () => job.process({ data: { matchId: "match-1" } } as any); + + it("stops the stream and drops the relay broadcast once the match is over", async () => { + await run(); + + expect(matchRelay.removeBroadcast).toHaveBeenCalledWith("match-1"); + expect(gameStreamer.stopLiveIfRunning).toHaveBeenCalledWith("match-1"); + }); + + it("hands the freed GPU to a waiting stream first", async () => { + gameStreamer.promotePendingLiveStreams.mockResolvedValue({ + promoted: ["match-2"], + }); + + await run(); + + expect(gameStreamer.promotePendingLiveStreams).toHaveBeenCalled(); + expect(clips.resumeAllPausedBatches).not.toHaveBeenCalled(); + }); + + it("hands the freed GPU to paused renders when no stream is waiting", async () => { + await run(); + + expect(clips.resumeAllPausedBatches).toHaveBeenCalled(); + }); + + it("frees nothing when the stream was already gone", async () => { + gameStreamer.stopLiveIfRunning.mockResolvedValue(false); + + await run(); + + expect(gameStreamer.promotePendingLiveStreams).not.toHaveBeenCalled(); + expect(clips.resumeAllPausedBatches).not.toHaveBeenCalled(); + }); + + it("leaves a match that was started again during the delay alone", async () => { + matchAssistant.hasMatchEnded.mockResolvedValue(false); + + await run(); + + expect(matchRelay.removeBroadcast).not.toHaveBeenCalled(); + expect(gameStreamer.stopLiveIfRunning).not.toHaveBeenCalled(); + }); +}); diff --git a/src/matches/jobs/StopMatchBroadcast.ts b/src/matches/jobs/StopMatchBroadcast.ts new file mode 100644 index 00000000..c0571479 --- /dev/null +++ b/src/matches/jobs/StopMatchBroadcast.ts @@ -0,0 +1,44 @@ +import { Job } from "bullmq"; +import { Logger } from "@nestjs/common"; +import { WorkerHost } from "@nestjs/bullmq"; +import { UseQueue } from "../../utilities/QueueProcessors"; +import { MatchQueues } from "../enums/MatchQueues"; +import { MatchAssistantService } from "../match-assistant/match-assistant.service"; +import { GameStreamerService } from "../game-streamer/game-streamer.service"; +import { MatchRelayService } from "../match-relay/match-relay.service"; +import { ClipsService } from "../clips/clips.service"; + +@UseQueue("Matches", MatchQueues.ScheduledMatches) +export class StopMatchBroadcast extends WorkerHost { + constructor( + private readonly logger: Logger, + private readonly matchAssistant: MatchAssistantService, + private readonly gameStreamer: GameStreamerService, + private readonly matchRelay: MatchRelayService, + private readonly clips: ClipsService, + ) { + super(); + } + + async process(job: Job<{ matchId: string }>): Promise { + const { matchId } = job.data; + + if (!(await this.matchAssistant.hasMatchEnded(matchId))) { + this.logger.log( + `[${matchId}] match was started again, leaving its broadcast running`, + ); + return; + } + + this.matchRelay.removeBroadcast(matchId); + + if (!(await this.gameStreamer.stopLiveIfRunning(matchId))) { + return; + } + + const { promoted } = await this.gameStreamer.promotePendingLiveStreams(); + if (promoted.length === 0) { + await this.clips.resumeAllPausedBatches(); + } + } +} diff --git a/src/matches/match-assistant/match-assistant.service.ts b/src/matches/match-assistant/match-assistant.service.ts index 62efbef1..f0407cda 100644 --- a/src/matches/match-assistant/match-assistant.service.ts +++ b/src/matches/match-assistant/match-assistant.service.ts @@ -1399,7 +1399,7 @@ export class MatchAssistantService { }); } - private async hasMatchEnded(matchId: string): Promise { + public async hasMatchEnded(matchId: string): Promise { const { matches_by_pk } = await this.hasura.query({ matches_by_pk: { __args: { diff --git a/src/matches/matches.controller.match-events.spec.ts b/src/matches/matches.controller.match-events.spec.ts index 3632937e..65c37c9e 100644 --- a/src/matches/matches.controller.match-events.spec.ts +++ b/src/matches/matches.controller.match-events.spec.ts @@ -14,6 +14,8 @@ describe("MatchesController — match_events on-demand servers", () => { let scheduledMatchesQueue: { add: jest.Mock }; let discordBotMessaging: { removeMatchChannel: jest.Mock }; let utilityPractice: Record; + let gameStreamer: Record; + let matchRelay: { removeBroadcast: jest.Mock }; let servers: Record< string, { @@ -29,6 +31,11 @@ describe("MatchesController — match_events on-demand servers", () => { ([name]) => name === "StopOnDemandServer", ); + const broadcastStopJobs = () => + scheduledMatchesQueue.add.mock.calls.filter( + ([name]) => name === "StopMatchBroadcast", + ); + const row = (overrides: Record = {}) => ({ id: "match-1", source: "5stack", @@ -104,6 +111,15 @@ describe("MatchesController — match_events on-demand servers", () => { evictForMatch: jest.fn(async (): Promise => undefined), markEndedForMatch: jest.fn(), }; + gameStreamer = { + stopLive: jest.fn(), + stopLiveIfRunning: jest.fn(), + getLiveStreamMode: jest.fn(async (): Promise => "tv"), + promotePendingLiveStreams: jest.fn(async () => ({ + promoted: [] as string[], + })), + }; + matchRelay = { removeBroadcast: jest.fn() }; controller = new MatchesController( { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as any, @@ -130,18 +146,12 @@ describe("MatchesController — match_events on-demand servers", () => { { add: jest.fn(async (): Promise => undefined) } as any, scheduledMatchesQueue as any, {} as any, - { removeBroadcast: jest.fn() } as any, + matchRelay as any, { createMatchVoiceChannels: jest.fn(), movePlayersToMatchChannels: jest.fn(), } as any, - { - stopLive: jest.fn(), - stopLiveIfRunning: jest.fn(), - promotePendingLiveStreams: jest.fn(async () => ({ - promoted: [] as string[], - })), - } as any, + gameStreamer as any, {} as any, { resumeAllPausedBatches: jest.fn() } as any, {} as any, @@ -399,4 +409,88 @@ describe("MatchesController — match_events on-demand servers", () => { expect(stopJobs()).toHaveLength(1); }); }); + + describe("the broadcast of a match that ends", () => { + const finish = () => + controller + .match_events({ + op: "UPDATE", + old: row({ status: "Live" }), + new: row({ status: "Finished" }), + } as any) + .catch((): void => undefined); + + it("keeps a TV stream and the relay up until the delayed feed catches up", async () => { + await finish(); + + expect(gameStreamer.stopLive).not.toHaveBeenCalled(); + expect(gameStreamer.stopLiveIfRunning).not.toHaveBeenCalled(); + expect(matchRelay.removeBroadcast).not.toHaveBeenCalled(); + + expect(broadcastStopJobs()).toHaveLength(1); + const [[, data, options]] = broadcastStopJobs(); + expect(data).toEqual({ matchId: "match-1" }); + // Finished only lands once the plugin has waited out tv_delay, so what + // is left is the viewers' playback lag, not another tv_delay. + expect(options.delay).toBe(30 * 1000); + }); + + it("waits out tv_delay as well when the match is forfeited mid-game", async () => { + await controller.match_events({ + op: "UPDATE", + old: row({ status: "Live" }), + new: row({ status: "Forfeit" }), + } as any); + + expect(gameStreamer.stopLiveIfRunning).not.toHaveBeenCalled(); + expect(broadcastStopJobs()[0][2].delay).toBe((30 + 30) * 1000); + }); + + it("stops a live-mode stream straight away but still waits on the relay", async () => { + gameStreamer.getLiveStreamMode.mockResolvedValue("live"); + + await finish(); + + expect(gameStreamer.stopLiveIfRunning).toHaveBeenCalledWith("match-1"); + expect(matchRelay.removeBroadcast).not.toHaveBeenCalled(); + expect(broadcastStopJobs()).toHaveLength(1); + }); + + it("tears down a canceled match straight away", async () => { + await controller.match_events({ + op: "UPDATE", + old: row({ status: "Live" }), + new: row({ status: "Canceled" }), + } as any); + + expect(gameStreamer.stopLiveIfRunning).toHaveBeenCalledWith("match-1"); + expect(matchRelay.removeBroadcast).toHaveBeenCalledWith("match-1"); + expect(broadcastStopJobs()).toHaveLength(0); + }); + + it("tears down a deleted match straight away", async () => { + await controller.match_events({ + op: "DELETE", + old: row(), + new: {}, + } as any); + + expect(gameStreamer.stopLive).toHaveBeenCalledWith("match-1"); + expect(matchRelay.removeBroadcast).toHaveBeenCalledWith("match-1"); + expect(broadcastStopJobs()).toHaveLength(0); + }); + + it("tears down straight away when the delayed stop cannot be scheduled", async () => { + scheduledMatchesQueue.add.mockImplementation(async (name: string) => { + if (name === "StopMatchBroadcast") { + throw new Error("redis unavailable"); + } + }); + + await finish(); + + expect(gameStreamer.stopLiveIfRunning).toHaveBeenCalledWith("match-1"); + expect(matchRelay.removeBroadcast).toHaveBeenCalledWith("match-1"); + }); + }); }); diff --git a/src/matches/matches.controller.ts b/src/matches/matches.controller.ts index 598f4457..265227a2 100644 --- a/src/matches/matches.controller.ts +++ b/src/matches/matches.controller.ts @@ -50,6 +50,7 @@ import { PlayerEloRecomputeService } from "./player-elo-recompute.service"; import { BackfillSeasonElo } from "./jobs/BackfillSeasonElo"; import { SeasonEloBackfillService } from "./season-elo-backfill.service"; import { StopOnDemandServer } from "./jobs/StopOnDemandServer"; +import { StopMatchBroadcast } from "./jobs/StopMatchBroadcast"; import { S3Service } from "src/s3/s3.service"; import { ChatService } from "src/chat/chat.service"; import { ChatLobbyType } from "src/chat/enums/ChatLobbyTypes"; @@ -86,6 +87,16 @@ export class MatchesController { private static readonly BLOCKING_RESET_STATUSES: string[] = ["Live", "Veto"]; + // Viewers of an ended match are still watching it play out: the relay syncs + // clients 7 fragments behind the newest one, and the stream adds its own + // encode/playout latency on top. + private static readonly BROADCAST_END_GRACE_SECONDS = 30; + + // The plugin reports Finished and Surrendered only once it has waited out + // tv_delay, so the feed has already caught up. These land in real time, a + // whole tv_delay ahead of what the feed is showing. + private static readonly REAL_TIME_END_STATUSES: string[] = ["Forfeit", "Tie"]; + // A DELETE carries the row in `old` and an UPDATE in `new`, and a voice // channel is per lineup rather than per match. private static lineupIds(data: HasuraEventData) { @@ -797,21 +808,7 @@ export class MatchesController { data.op === "DELETE" || MatchesController.TERMINAL_STATUSES.includes(status) ) { - try { - if (data.op === "DELETE") { - await this.gameStreamer.stopLive(matchId); - } else { - await this.gameStreamer.stopLiveIfRunning(matchId); - } - } catch (error) { - this.logger.error( - `[${matchId}] failed to stop live stream on match end: ${ - (error as Error)?.message - }`, - ); - } - - this.matchRelayService.removeBroadcast(matchId); + await this.endMatchBroadcast(data, matchId); await this.removeDiscordIntegration(matchId); await this.matchmaking.cancelMatchMakingByMatchId(matchId); await this.releaseScrimScheduledNotifications(matchId); @@ -938,6 +935,88 @@ export class MatchesController { return null; } + private async endMatchBroadcast( + data: HasuraEventData, + matchId: string, + ) { + const delay = await this.broadcastEndDelaySeconds(data); + + let scheduled = false; + if (delay) { + try { + await this.scheduledMatchesQueue.add( + StopMatchBroadcast.name, + { matchId }, + MatchesController.stopOnDemandServerJobOptions(delay), + ); + scheduled = true; + } catch (error) { + this.logger.error( + `[${matchId}] failed to schedule the broadcast stop: ${ + (error as Error)?.message + }`, + ); + } + } + + try { + if (data.op === "DELETE") { + await this.gameStreamer.stopLive(matchId); + } else if ( + !scheduled || + // A live-mode stream watches the game port, not the delayed TV feed. + (await this.gameStreamer.getLiveStreamMode(matchId)) === "live" + ) { + await this.gameStreamer.stopLiveIfRunning(matchId); + } + } catch (error) { + this.logger.error( + `[${matchId}] failed to stop live stream on match end: ${ + (error as Error)?.message + }`, + ); + } + + if (!scheduled) { + this.matchRelayService.removeBroadcast(matchId); + } + } + + private async broadcastEndDelaySeconds( + data: HasuraEventData, + ): Promise { + if (data.op === "DELETE" || data.new.status === "Canceled") { + return 0; + } + + if (!MatchesController.REAL_TIME_END_STATUSES.includes(data.new.status)) { + return MatchesController.BROADCAST_END_GRACE_SECONDS; + } + + try { + const { match_options_by_pk: matchOptions } = await this.hasura.query({ + match_options_by_pk: { + __args: { + id: data.new.match_options_id, + }, + tv_delay: true, + }, + }); + + return ( + (matchOptions?.tv_delay ?? 0) + + MatchesController.BROADCAST_END_GRACE_SECONDS + ); + } catch (error) { + this.logger.error( + `[${data.new.id}] failed to read tv_delay for the broadcast stop: ${ + (error as Error)?.message + }`, + ); + return MatchesController.BROADCAST_END_GRACE_SECONDS; + } + } + private static stopOnDemandServerJobOptions(delaySeconds = 0) { return { ...(delaySeconds ? { delay: delaySeconds * 1000 } : {}), diff --git a/src/matches/matches.module.ts b/src/matches/matches.module.ts index d4174b53..a4fd8158 100644 --- a/src/matches/matches.module.ts +++ b/src/matches/matches.module.ts @@ -64,6 +64,7 @@ import { BackfillSeasonElo } from "./jobs/BackfillSeasonElo"; import { SeasonEloBackfillService } from "./season-elo-backfill.service"; import { PostgresService } from "src/postgres/postgres.service"; import { StopOnDemandServer } from "./jobs/StopOnDemandServer"; +import { StopMatchBroadcast } from "./jobs/StopMatchBroadcast"; import { ReconcileOnDemandServerJobs } from "./jobs/ReconcileOnDemandServerJobs"; import { MatchRelayController } from "./match-relay/match-relay.controller"; import { MatchRelayService } from "./match-relay/match-relay.service"; @@ -188,6 +189,7 @@ import { CameraMonitorService } from "./camera/camera-monitor.service"; CheckForScheduledMatches, RemoveCancelledMatches, StopOnDemandServer, + StopMatchBroadcast, ReconcileOnDemandServerJobs, CancelInvalidTournaments, CleanAbandonedMatches, diff --git a/test/elo.spec.ts b/test/elo.spec.ts index 49273dfd..74c12437 100644 --- a/test/elo.spec.ts +++ b/test/elo.spec.ts @@ -90,9 +90,13 @@ describe("ELO engine (SQL-driven)", () => { const wingman = async ( teamA: Array, teamB: Array, - { winner = "a", endedDaysAgo = 1 }: { winner?: "a" | "b"; endedDaysAgo?: number } = {}, + { + winner = "a", + endedDaysAgo = 1, + substitutes = 0, + }: { winner?: "a" | "b"; endedDaysAgo?: number; substitutes?: number } = {}, ) => { - const match = await fx.match({ type: "Wingman" }); + const match = await fx.match({ type: "Wingman", substitutes }); for (const steamId of teamA) { await fx.lineupPlayer(match.lineup_1_id, steamId); } @@ -399,6 +403,107 @@ describe("ELO engine (SQL-driven)", () => { ); }); + describe("substitutes", () => { + const mapFor = async (matchId: string) => { + const [existing] = await postgres.query>( + `SELECT id FROM match_maps WHERE match_id = $1 ORDER BY "order" LIMIT 1`, + [matchId], + ); + if (existing) { + return { matchId, mapId: existing.id }; + } + const [map] = await postgres.query>( + `INSERT INTO match_maps (match_id, map_id, "order") + SELECT $1, id, 1 FROM maps ORDER BY name LIMIT 1 RETURNING id`, + [matchId], + ); + return { matchId, mapId: map.id }; + }; + + it("does not rate a substitute who never played", async () => { + const [a, b, sub, c, d] = await fx.players(5); + await seedRatings({ + [a]: 5000, + [b]: 5000, + [sub]: 9000, + [c]: 5000, + [d]: 5000, + }); + const match = await wingman([a, b, sub], [c, d], { substitutes: 1 }); + const ctx = await mapFor(match.id); + await fx.kill(ctx, a, c); + await fx.kill(ctx, b, d); + + expect(await generate(match.id)).toBe(4); + + const rows = await eloRows(match.id); + expect(rows.map((r) => r.steam_id).sort()).toEqual([a, b, c, d].sort()); + + // The benched 9000 must not drag the team average up either. + const rated = rows.find((r) => r.steam_id === a)!; + expect(Number(rated.rating_for_expected)).toBeCloseTo(5000); + }); + + it("rates a substitute who came in and played", async () => { + const [a, b, sub, c, d] = await fx.players(5); + const match = await wingman([a, b, sub], [c, d], { substitutes: 1 }); + const ctx = await mapFor(match.id); + await fx.kill(ctx, a, c); + await fx.kill(ctx, c, sub, { round: 2 }); + + await generate(match.id); + + const steamIds = (await eloRows(match.id)).map((r) => r.steam_id); + expect(steamIds).toContain(sub); + expect(steamIds).toContain(a); + expect(steamIds).toContain(c); + }); + + it("still penalises a player who abandoned without ever playing", async () => { + await postgres.query( + "DELETE FROM settings WHERE name = 'leaver_elo_penalty'", + ); + const [a, b, c, d] = await fx.players(4); + const match = await wingman([a, b], [c, d]); + const ctx = await mapFor(match.id); + await fx.kill(ctx, a, c); + await fx.kill(ctx, b, c, { round: 2 }); + await postgres.query( + "INSERT INTO abandoned_matches (steam_id, match_id) VALUES ($1, $2)", + [d, match.id], + ); + await generate(match.id); + + const leaver = (await eloRows(match.id)).find((r) => r.steam_id === d)!; + expect(Number(leaver.change)).toBe(-150); + }); + + it("still rates against a lineup of unlinked placeholders", async () => { + const [a, b] = await fx.players(2); + const match = await wingman([a, b], []); + for (const name of ["ghost-1", "ghost-2"]) { + await postgres.query( + `INSERT INTO match_lineup_players (match_lineup_id, discord_id, placeholder_name) + VALUES ($1, $2, $2)`, + [match.lineup_2_id, name], + ); + } + + expect(await generate(match.id)).toBe(2); + + const rows = await eloRows(match.id); + expect(rows.every((r) => r.expected_score !== null)).toBe(true); + expect(rows.every((r) => Number(r.change) > 0)).toBe(true); + }); + + it("keeps a whole lineup rated when nothing was recorded against it", async () => { + const [a, b, sub, c, d] = await fx.players(5); + const match = await wingman([a, b, sub], [c, d], { substitutes: 1 }); + + expect(await generate(match.id)).toBe(5); + }); + }); + describe("abandon penalty", () => { beforeEach(async () => { await postgres.query( From 01a965c9c41ab82236b5b2b0a6c577e21f8daf03 Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Sat, 26 Sep 2026 10:13:57 -0400 Subject: [PATCH 2/2] bug: time the tv stream stop off when the feed actually ends (#633) - a map still in play means the match was ended mid-map, so wait out tv_delay; otherwise the plugin already waited and stops the tv 5s later - relay viewers get the fragments they buffered: 8 x the server's keyframe_interval - only schedule the stop on the transition into a terminal status --- .../match-relay/match-relay.service.spec.ts | 11 +++ .../match-relay/match-relay.service.ts | 24 ++++++- .../matches.controller.match-events.spec.ts | 71 +++++++++++++++---- src/matches/matches.controller.ts | 64 +++++++++++------ 4 files changed, 135 insertions(+), 35 deletions(-) diff --git a/src/matches/match-relay/match-relay.service.spec.ts b/src/matches/match-relay/match-relay.service.spec.ts index b65d28fc..2cfa5c26 100644 --- a/src/matches/match-relay/match-relay.service.spec.ts +++ b/src/matches/match-relay/match-relay.service.spec.ts @@ -127,6 +127,17 @@ describe("MatchRelayService", () => { // CS2 sends tps as a decimal (64.0), not an integer, so the coercion has to // accept a fractional part or clients get tps back as a string. + it("reports how long clients keep playing once the server stops posting", async () => { + await startBroadcastAt(42); + + // Clients sit 7 fragments behind the newest and still have that one to play. + expect(service.playoutSeconds(matchId)).toBe(8 * 3); + }); + + it("has nothing to play out for a broadcast it does not hold", () => { + expect(service.playoutSeconds(matchId)).toBe(0); + }); + it("reports a fractional tps as a number", async () => { await post("start", 42, { tick: "100", diff --git a/src/matches/match-relay/match-relay.service.ts b/src/matches/match-relay/match-relay.service.ts index 7e4f9131..5b488f3b 100644 --- a/src/matches/match-relay/match-relay.service.ts +++ b/src/matches/match-relay/match-relay.service.ts @@ -19,6 +19,10 @@ export class MatchRelayService { "protocol", ]; + // Like Valve's reference relay, /sync starts a new client this many fragments + // behind the newest one, and the client then plays in real time. + private static readonly SYNC_LAG_FRAGMENTS = 7; + private readonly gzip = promisify(zlib.gzip); private readonly broadcasts: { @@ -35,6 +39,21 @@ export class MatchRelayService { delete this.broadcasts[matchId]; } + // How long a client keeps playing after the game server stops posting: it is + // SYNC_LAG_FRAGMENTS behind the last fragment, plus that fragment itself. + // keyframe_interval is the fragment length the game server announced in start. + public playoutSeconds(matchId: string): number { + const keyframeInterval = Number( + this.broadcasts[matchId]?.fragments.get(0)?.start?.keyframe_interval, + ); + + if (!(keyframeInterval > 0)) { + return 0; + } + + return (MatchRelayService.SYNC_LAG_FRAGMENTS + 1) * keyframeInterval; + } + public getStart(response: Response, matchId: string, fragmentIndex: number) { const broadcast = this.broadcasts[matchId]; const startFragment = broadcast?.fragments.get(0); @@ -107,7 +126,10 @@ export class MatchRelayService { : 0; if (fragmentParam == null) { - fragmentIndex = Math.max(0, maxIndex - 7); + fragmentIndex = Math.max( + 0, + maxIndex - MatchRelayService.SYNC_LAG_FRAGMENTS, + ); if ( fragmentIndex >= 0 && diff --git a/src/matches/matches.controller.match-events.spec.ts b/src/matches/matches.controller.match-events.spec.ts index 65c37c9e..071cd7ea 100644 --- a/src/matches/matches.controller.match-events.spec.ts +++ b/src/matches/matches.controller.match-events.spec.ts @@ -15,7 +15,8 @@ describe("MatchesController — match_events on-demand servers", () => { let discordBotMessaging: { removeMatchChannel: jest.Mock }; let utilityPractice: Record; let gameStreamer: Record; - let matchRelay: { removeBroadcast: jest.Mock }; + let matchRelay: { removeBroadcast: jest.Mock; playoutSeconds: jest.Mock }; + let inPlayMaps: Array<{ id: string }>; let servers: Record< string, { @@ -85,6 +86,12 @@ describe("MatchesController — match_events on-demand servers", () => { servers_by_pk: servers[request.servers_by_pk.__args.id] ?? null, }; } + if (request.match_maps) { + return { + match_maps: inPlayMaps, + match_options_by_pk: { tv_delay: 30 }, + }; + } if (request.match_options_by_pk) { return { match_options_by_pk: { tv_delay: 30 } }; } @@ -119,7 +126,11 @@ describe("MatchesController — match_events on-demand servers", () => { promoted: [] as string[], })), }; - matchRelay = { removeBroadcast: jest.fn() }; + matchRelay = { + removeBroadcast: jest.fn(), + playoutSeconds: jest.fn(() => 0), + }; + inPlayMaps = []; controller = new MatchesController( { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as any, @@ -420,7 +431,9 @@ describe("MatchesController — match_events on-demand servers", () => { } as any) .catch((): void => undefined); - it("keeps a TV stream and the relay up until the delayed feed catches up", async () => { + const broadcastStopDelay = () => broadcastStopJobs()[0][2].delay; + + it("keeps a TV stream up until the plugin stops the TV", async () => { await finish(); expect(gameStreamer.stopLive).not.toHaveBeenCalled(); @@ -428,22 +441,52 @@ describe("MatchesController — match_events on-demand servers", () => { expect(matchRelay.removeBroadcast).not.toHaveBeenCalled(); expect(broadcastStopJobs()).toHaveLength(1); - const [[, data, options]] = broadcastStopJobs(); + const [[, data]] = broadcastStopJobs(); expect(data).toEqual({ matchId: "match-1" }); - // Finished only lands once the plugin has waited out tv_delay, so what - // is left is the viewers' playback lag, not another tv_delay. - expect(options.delay).toBe(30 * 1000); + // The plugin already waited out tv_delay before finishing the map, and + // runs tv_stop 5s later. + expect(broadcastStopDelay()).toBe(5 * 1000); }); - it("waits out tv_delay as well when the match is forfeited mid-game", async () => { - await controller.match_events({ - op: "UPDATE", - old: row({ status: "Live" }), - new: row({ status: "Forfeit" }), - } as any); + it("lets relay viewers play out what they had buffered", async () => { + matchRelay.playoutSeconds.mockReturnValue(24); + + await finish(); + + expect(matchRelay.playoutSeconds).toHaveBeenCalledWith("match-1"); + expect(broadcastStopDelay()).toBe((5 + 24) * 1000); + }); + + it("waits out tv_delay when an organizer ends the match mid-map", async () => { + // A forfeit with a winner is stored as Finished, so only the map still + // being played gives away that the TV feed is behind. + inPlayMaps = [{ id: "map-1" }]; + + await finish(); + + expect(broadcastStopDelay()).toBe(30 * 1000); + }); + + it("adds the relay play-out on top of tv_delay mid-map", async () => { + inPlayMaps = [{ id: "map-1" }]; + matchRelay.playoutSeconds.mockReturnValue(24); + + await finish(); + + expect(broadcastStopDelay()).toBe((30 + 24) * 1000); + }); + it("schedules the stop once, not on every later update", async () => { + await controller + .match_events({ + op: "UPDATE", + old: row({ status: "Finished" }), + new: row({ status: "Finished", server_id: null }), + } as any) + .catch((): void => undefined); + + expect(broadcastStopJobs()).toHaveLength(0); expect(gameStreamer.stopLiveIfRunning).not.toHaveBeenCalled(); - expect(broadcastStopJobs()[0][2].delay).toBe((30 + 30) * 1000); }); it("stops a live-mode stream straight away but still waits on the relay", async () => { diff --git a/src/matches/matches.controller.ts b/src/matches/matches.controller.ts index 265227a2..9ea548dd 100644 --- a/src/matches/matches.controller.ts +++ b/src/matches/matches.controller.ts @@ -31,6 +31,7 @@ import { match_lineup_players_set_input, e_notification_types_enum, e_player_roles_enum, + e_match_map_status_enum, } from "../../generated"; import { ConfigService } from "@nestjs/config"; import { AppConfig } from "src/configs/types/AppConfig"; @@ -87,15 +88,20 @@ export class MatchesController { private static readonly BLOCKING_RESET_STATUSES: string[] = ["Live", "Veto"]; - // Viewers of an ended match are still watching it play out: the relay syncs - // clients 7 fragments behind the newest one, and the stream adds its own - // encode/playout latency on top. - private static readonly BROADCAST_END_GRACE_SECONDS = 30; - - // The plugin reports Finished and Surrendered only once it has waited out - // tv_delay, so the feed has already caught up. These land in real time, a - // whole tv_delay ahead of what the feed is showing. - private static readonly REAL_TIME_END_STATUSES: string[] = ["Forfeit", "Tie"]; + // When the plugin ends a map itself it has already waited out tv_delay, so the + // TV feed has caught up; it then runs tv_stop 5s after marking the map + // Finished (delayChangeMap(5) in the game-server plugin). + private static readonly PLUGIN_TV_STOP_SECONDS = 5; + + // A map still in one of these when the match ends was cut short from outside + // the plugin (an organizer forfeit, tie or winner), so the TV feed is still a + // whole tv_delay behind. + private static readonly IN_PLAY_MAP_STATUSES: e_match_map_status_enum[] = [ + "Knife", + "Live", + "Overtime", + "Paused", + ]; // A DELETE carries the row in `old` and an UPDATE in `new`, and a voice // channel is per lineup rather than per match. @@ -939,7 +945,14 @@ export class MatchesController { data: HasuraEventData, matchId: string, ) { - const delay = await this.broadcastEndDelaySeconds(data); + if ( + data.op !== "DELETE" && + MatchesController.TERMINAL_STATUSES.includes(data.old?.status) + ) { + return; + } + + const delay = await this.broadcastEndDelaySeconds(data, matchId); let scheduled = false; if (delay) { @@ -982,19 +995,30 @@ export class MatchesController { } } + // Seconds until TV viewers have seen the end of the match: until the TV feed + // runs out, plus however long relay clients keep playing what they buffered. private async broadcastEndDelaySeconds( data: HasuraEventData, + matchId: string, ): Promise { if (data.op === "DELETE" || data.new.status === "Canceled") { return 0; } - if (!MatchesController.REAL_TIME_END_STATUSES.includes(data.new.status)) { - return MatchesController.BROADCAST_END_GRACE_SECONDS; - } + let feedEndsIn = MatchesController.PLUGIN_TV_STOP_SECONDS; try { - const { match_options_by_pk: matchOptions } = await this.hasura.query({ + const { match_maps, match_options_by_pk } = await this.hasura.query({ + match_maps: { + __args: { + where: { + match_id: { _eq: matchId }, + status: { _in: MatchesController.IN_PLAY_MAP_STATUSES }, + }, + limit: 1, + }, + id: true, + }, match_options_by_pk: { __args: { id: data.new.match_options_id, @@ -1003,18 +1027,18 @@ export class MatchesController { }, }); - return ( - (matchOptions?.tv_delay ?? 0) + - MatchesController.BROADCAST_END_GRACE_SECONDS - ); + if (match_maps.length > 0) { + feedEndsIn = match_options_by_pk?.tv_delay ?? 0; + } } catch (error) { this.logger.error( - `[${data.new.id}] failed to read tv_delay for the broadcast stop: ${ + `[${matchId}] failed to check how far behind the TV feed is: ${ (error as Error)?.message }`, ); - return MatchesController.BROADCAST_END_GRACE_SECONDS; } + + return feedEndsIn + this.matchRelayService.playoutSeconds(matchId); } private static stopOnDemandServerJobOptions(delaySeconds = 0) {