bug: keep match servers from hibernating, and keep SwiftlyS2's crash dump

apps/counterstrikesharp/scripts/setup.sh

@@ -45,7 +45,8 @@ echo "---Create Symbolic Links---"
 
 if [ "$SERVER_TYPE" = "Ranked" ]; then
   cp "/opt/server-cfg/ranked.server.cfg" "$INSTANCE_SERVER_DIR/game/csgo/cfg/server.cfg"
-  if [ "$HIBERNATE_WHEN_EMPTY" = "true" ]; then
+  # Never a match server: it is started for its match and stopped after it.
+  if [ "$HIBERNATE_WHEN_EMPTY" = "true" ] && [ -z "$MATCH_ID" ]; then
     sed -i 's/^sv_hibernate_when_empty.*/sv_hibernate_when_empty 1/' "$INSTANCE_SERVER_DIR/game/csgo/cfg/server.cfg"
   fi
   cp "/opt/server-cfg/5stack.competitive.cfg" "$INSTANCE_SERVER_DIR/game/csgo/cfg"

apps/counterstrikesharp/src/FiveStack.Services/EnvironmentService.cs

@@ -54,6 +54,12 @@ public bool isOnGameServerNode()
         return Environment.GetEnvironmentVariable("GAME_NODE_SERVER") == "true";
     }
 
+    // Started by the panel for one match and stopped once it is over
+    public bool IsMatchServer()
+    {
+        return !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("MATCH_ID"));
+    }
+
     public bool IsOfflineMode()
     {
         return Environment.GetEnvironmentVariable("OFFLINE_MODE") == "true";

apps/counterstrikesharp/src/FiveStack.Services/GameServer.cs

@@ -99,7 +99,7 @@ public void Ping(string pluginVersion)
             string? workshopID = _matchService.GetWorkshopID();
 
             string endpoint =
-                $"{_environmentService.GetApiUrl()}/game-server-node/ping/{serverId}?map={(string.IsNullOrEmpty(workshopID) ? Server.MapName : workshopID)}&pluginVersion={pluginVersion}&pluginRuntime=counterstrikesharp";
+                $"{_environmentService.GetApiUrl()}/game-server-node/ping/{serverId}?map={(string.IsNullOrEmpty(workshopID) ? Server.MapName : workshopID)}&pluginVersion={pluginVersion}&pluginRuntime=counterstrikesharp&hibernating={(HibernationUtility.IsHibernating ? "true" : "false")}";
 
             if (_steamRelay)
             {

apps/counterstrikesharp/src/FiveStack.Services/MatchService.cs

@@ -203,7 +203,11 @@ public void StayAwake(bool awake)
             if (hibernate.GetPrimitiveValue<bool>())
             {
                 hibernate.SetValue(false);
-                _heldAwake = true;
+
+                // A match server never hibernates. Its match can arrive
+                // before server.cfg has turned the engine's default off,
+                // and that default is not something to hand it back to.
+                _heldAwake = !_environmentService.IsMatchServer();
                 _logger.LogInformation("holding the server awake for its match");
             }
 

apps/counterstrikesharp/src/FiveStack.Utilities/HibernationUtility.cs

@@ -11,6 +11,8 @@ public static class HibernationUtility
 {
     private static volatile bool _hibernating;
 
+    public static bool IsHibernating => _hibernating;
+
     public static void SetHibernating(bool hibernating)
     {
         _hibernating = hibernating;

apps/swiftly/scripts/setup.sh

@@ -50,7 +50,8 @@ echo "---Create Symbolic Links---"
 
 if [ "$SERVER_TYPE" = "Ranked" ]; then
   cp "/opt/server-cfg/ranked.server.cfg" "$INSTANCE_SERVER_DIR/game/csgo/cfg/server.cfg"
-  if [ "$HIBERNATE_WHEN_EMPTY" = "true" ]; then
+  # Never a match server: it is started for its match and stopped after it.
+  if [ "$HIBERNATE_WHEN_EMPTY" = "true" ] && [ -z "$MATCH_ID" ]; then
     sed -i 's/^sv_hibernate_when_empty.*/sv_hibernate_when_empty 1/' "$INSTANCE_SERVER_DIR/game/csgo/cfg/server.cfg"
   fi
   cp "/opt/server-cfg/5stack.base.cfg" "$INSTANCE_SERVER_DIR/game/csgo/cfg"

apps/swiftly/src/FiveStack.Services/EnvironmentService.cs

@@ -125,6 +125,12 @@ public bool isOnGameServerNode()
         return Environment.GetEnvironmentVariable("GAME_NODE_SERVER") == "true";
     }
 
+    // Started by the panel for one match and stopped once it is over
+    public bool IsMatchServer()
+    {
+        return !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("MATCH_ID"));
+    }
+
     public bool IsOfflineMode()
     {
         return Environment.GetEnvironmentVariable("OFFLINE_MODE") == "true";

apps/swiftly/src/FiveStack.Services/GameServer.cs

@@ -123,7 +123,7 @@ public void Ping(string pluginVersion)
                 : workshopID;
 
             string endpoint =
-                $"{_environmentService.GetApiUrl()}/game-server-node/ping/{serverId}?map={mapName}&pluginVersion={pluginVersion}&pluginRuntime=swiftlys2";
+                $"{_environmentService.GetApiUrl()}/game-server-node/ping/{serverId}?map={mapName}&pluginVersion={pluginVersion}&pluginRuntime=swiftlys2&hibernating={(HibernationUtility.IsHibernating ? "true" : "false")}";
 
             if (_steamRelay)
             {

apps/swiftly/src/FiveStack.Services/MatchService.cs

@@ -205,7 +205,11 @@ public void StayAwake(bool awake)
                 {
                     // SetInternal applies now; a queued set waits for a tick
                     hibernate.SetInternal(false);
-                    _heldAwake = true;
+
+                    // A match server never hibernates. Its match can arrive
+                    // before server.cfg has turned the engine's default off,
+                    // and that default is not something to hand it back to.
+                    _heldAwake = !_environmentService.IsMatchServer();
                     _logger.LogInformation("holding the server awake for its match");
                 }
 

apps/swiftly/src/FiveStack.Utilities/HibernationUtility.cs

@@ -11,16 +11,22 @@ namespace FiveStack.Utilities;
 public static class HibernationUtility
 {
     private const long TickingWindowMs = 250;
+    private const long HibernatingAfterMs = 5000;
 
     private static ISwiftlyCore _core = null!;
     private static EventDelegates.OnTick? _tickHandler;
+    private static EventDelegates.OnWorldUpdate? _worldUpdateHandler;
     private static long _lastTickMs;
+    private static long _updatingWithoutTicksSinceMs;
 
     public static void Initialize(ISwiftlyCore core)
     {
         _core = core;
         _tickHandler = () => Volatile.Write(ref _lastTickMs, Environment.TickCount64);
         _core.Event.OnTick += _tickHandler;
+
+        _worldUpdateHandler = OnWorldUpdate;
+        _core.Event.OnWorldUpdate += _worldUpdateHandler;
     }
 
     public static void Shutdown()
@@ -30,11 +36,45 @@ public static void Shutdown()
             _core.Event.OnTick -= _tickHandler;
             _tickHandler = null;
         }
+
+        if (_worldUpdateHandler != null)
+        {
+            _core.Event.OnWorldUpdate -= _worldUpdateHandler;
+            _worldUpdateHandler = null;
+        }
+    }
+
+    private static bool IsTicking =>
+        Environment.TickCount64 - Volatile.Read(ref _lastTickMs) < TickingWindowMs;
+
+    // A map load stops ticks and world updates together and plugin load starts
+    // with neither seen, so only world updates that keep arriving with no tick
+    // between them say the server is hibernating.
+    private static void OnWorldUpdate()
+    {
+        if (IsTicking)
+        {
+            Volatile.Write(ref _updatingWithoutTicksSinceMs, 0);
+        }
+        else if (Volatile.Read(ref _updatingWithoutTicksSinceMs) == 0)
+        {
+            Volatile.Write(ref _updatingWithoutTicksSinceMs, Environment.TickCount64);
+        }
+    }
+
+    public static bool IsHibernating
+    {
+        get
+        {
+            long since = Volatile.Read(ref _updatingWithoutTicksSinceMs);
+
+            return since != 0 && Environment.TickCount64 - since > HibernatingAfterMs;
+        }
     }
 
     public static void NextTick(Action callback)
     {
-        if (Environment.TickCount64 - Volatile.Read(ref _lastTickMs) < TickingWindowMs)
+        if (IsTicking)
         {
             _core.Scheduler.NextTick(callback);
             return;

apps/utility-sw/src/UtilityPracticePlugin.cs

@@ -267,8 +267,8 @@ public override void Load(bool hotReload)
         // One repeating job for the whole plugin, not one per player. These
         // deliberately do not get StopOnMapChange: the plugin is not reloaded
         // on a map change, so a timer that stopped there would never come back.
-        _secondTimer = Core.Scheduler.RepeatBySeconds(1, Surviving(OnSecond));
-        _refillTimer = Core.Scheduler.RepeatBySeconds(0.1f, Surviving(OnFastTick));
+        _secondTimer = Core.Scheduler.RepeatBySeconds(1, Surviving(1, OnSecond));
+        _refillTimer = Core.Scheduler.RepeatBySeconds(0.1f, Surviving(0.1f, OnFastTick));
 
         // Only on a hot reload. A cold boot has no engine globals yet -- asking
         // for the map here is what stopped the plugin loading at all -- and the
@@ -302,17 +302,31 @@ public override void Load(bool hotReload)
 
     private bool _tornDown;
 
-    // SwiftlyS2 never reschedules a repeating timer whose callback throws, and
-    // still runs a timer's first pass if it was cancelled before then.
-    private Action Surviving(Action callback)
+    // SwiftlyS2 never reschedules a repeating timer whose callback throws,
+    // still runs a timer's first pass if it was cancelled before then, and
+    // after a spell without ticks replays every run it missed, one a tick. The
+    // gap check turns that replay into at most twice the usual rate.
+    private Action Surviving(float interval, Action callback)
     {
+        long lastRunMs = 0;
+        long minimumGapMs = (long)(interval * 500);
+
         return () =>
         {
             if (_tornDown)
             {
                 return;
             }
 
+            long nowMs = Environment.TickCount64;
+
+            if (nowMs - lastRunMs < minimumGapMs)
+            {
+                return;
+            }
+
+            lastRunMs = nowMs;
+
             try
             {
                 callback();

shared/scripts/crash-report.sh

@@ -13,6 +13,12 @@ CONSOLE_TAIL_LINES=2000
 # Where the engine's crash handler writes minidumps, inside the container.
 MINIDUMP_DIR="/tmp/dumps"
 
+# SwiftlyS2 takes the crash signals over from the engine's handler and writes
+# its own dump here instead, under the instance, in a folder per start and
+# beside what it could read out of it. This is the container's own disk and
+# goes when the container does.
+SWIFTLY_CRASH_DIR="game/csgo/addons/swiftlys2/dumps/crashreport"
+
 CONSOLE_PIPE_DIR="/tmp"
 
 # Runs the server and, when it crashes, saves a report before the container
@@ -170,6 +176,17 @@ write_crash_report() {
     fi
   done
 
+  local swiftly_file
+  for swiftly_file in "${INSTANCE_SERVER_DIR}/${SWIFTLY_CRASH_DIR}"/*/*; do
+    if [ -f "$swiftly_file" ]; then
+      mv "$swiftly_file" "${report_dir}/"
+
+      case "$swiftly_file" in
+        *.dmp) minidumps=$((minidumps + 1)) ;;
+      esac
+    fi
+  done
+
   {
     echo "server_id=${SERVER_ID:-}"
     echo "match_id=${MATCH_ID:-}"

shared/scripts/test/crash-report.test.sh

@@ -55,7 +55,7 @@ setup() {
   export RELEASE_VERSION="0.0.87"
   export ENABLED_PLUGINS="deathmatch@1.1.2,map-chooser@1.3.2"
   export EXTRA_GAME_PARAMS="-maxplayers 16 +map de_cache +game_type 0 +sv_password hunter2 +game_mode 1"
-  export MINIDUMP_DIR INSTANCE_SERVER_DIR
+  export MINIDUMP_DIR INSTANCE_SERVER_DIR SWIFTLY_CRASH_DIR
 }
 
 teardown() {
@@ -78,6 +78,12 @@ echo "console after changelevel"
 if [ -n "${WRITE_DUMP:-}" ]; then
   echo "minidump" > "$MINIDUMP_DIR/d5e1c3a2-0000-0000-0000-000000000000.dmp"
 fi
+if [ -n "${WRITE_SWIFTLY_DUMP:-}" ]; then
+  swiftly_dumps="$INSTANCE_SERVER_DIR/$SWIFTLY_CRASH_DIR/2de1586d-0000-0000-0000-000000000000"
+  mkdir -p "$swiftly_dumps"
+  echo "swiftly minidump" > "$swiftly_dumps/3e37c382-0000-0000-0000-000000000000.dmp"
+  echo '{"crash_reason":"SIGSEGV"}' > "$swiftly_dumps/crashinfo.json"
+fi
 if [ -n "${UPDATE_GAME:-}" ]; then
   printf 'PatchVersion=1.41.9.0\r\n' > "$INSTANCE_SERVER_DIR/game/csgo/steam.inf"
 fi
@@ -156,6 +162,19 @@ assert_contains "$(cat "$(only_report)/crash.txt" 2> /dev/null)" "minidumps=1" \
   "a minidump without an exit status was not reported"
 teardown
 
+echo "run_server saves the dump SwiftlyS2 writes in place of the engine's"
+setup
+server="$(fake_server)"
+LINES_BEFORE_CHANGE=1 EXIT_STATUS=1 WRITE_SWIFTLY_DUMP=1 run_server "$server" > /dev/null
+report="$(only_report)"
+assert_equals "$(cat "$report/3e37c382-0000-0000-0000-000000000000.dmp" 2> /dev/null)" "swiftly minidump" \
+  "SwiftlyS2's minidump was not moved into the report"
+assert_equals "$(cat "$report/crashinfo.json" 2> /dev/null)" '{"crash_reason":"SIGSEGV"}' \
+  "what SwiftlyS2 read out of its minidump was not moved into the report"
+assert_contains "$(cat "$report/crash.txt" 2> /dev/null)" "minidumps=1" \
+  "SwiftlyS2's minidump was not counted, or what sits beside it was"
+teardown
+
 echo "run_server files a report without a server id under unknown"
 setup
 SERVER_ID=""