Watch graceful shutdown

README.md

@@ -171,7 +171,7 @@ You can pass the following options via CLI arguments. You can also use `--config
 | Set the prefix                                                                                                                          | `-x`          | `--prefix`         | `FASTIFY_PREFIX`         |
 | Set the plugin timeout                                                                                                                  | `-T`          | `--plugin-timeout` | `FASTIFY_PLUGIN_TIMEOUT` |
 | Defines the maximum payload, in bytes,<br>that the server is allowed to accept                                                               |               | `--body-limit`     | `FASTIFY_BODY_LIMIT`     |
-| Set the maximum ms delay before forcefully closing pending requests after receiving SIGTERM or SIGINT signals; and uncaughtException or unhandledRejection errors (default: 500)                                                           | `-g`         | `--close-grace-delay`     | `FASTIFY_CLOSE_GRACE_DELAY`     |
+| Set the maximum ms delay for graceful shutdown after SIGTERM, SIGINT, uncaughtException, or unhandledRejection, and for watch-mode restarts (default: 500)                                                           | `-g`         | `--close-grace-delay`     | `FASTIFY_CLOSE_GRACE_DELAY`     |
 | Set the boolean value for `trustProxy` (1st precedence)                                                                                 |               | `--trust-proxy-enabled` | `FASTIFY_TRUST_PROXY_ENABLED`         |
 | Set the IP/CIDR value for `trustProxy` (2nd precedence)                                                                                 |               | `--trust-proxy-ips` | `FASTIFY_TRUST_PROXY_IPS`             |
 | Set the nth hop value for `trustProxy` (3rd precedence)                                                                                 |               | `--trust-proxy-hop` | `FASTIFY_TRUST_PROXY_HOP`             |

lib/watch/constants.js

@@ -2,6 +2,5 @@
 
 module.exports = {
   GRACEFUL_SHUT: 'GRACEFUL SHUTDOWN',
-  READY: 'ready',
-  TIMEOUT: 5000
+  READY: 'ready'
 }

lib/watch/fork.js

@@ -2,11 +2,13 @@
 
 const chalk = require('chalk').default
 const { runFastify } = require('../../start')
+const parseArgs = require('../../args')
+const args = process.argv.slice(2)
+const { closeGraceDelay } = parseArgs(args)
 
 const {
   GRACEFUL_SHUT,
-  READY,
-  TIMEOUT
+  READY
 } = require('./constants.js')
 
 let fastify
@@ -19,7 +21,7 @@ function exit () {
 process.on('message', function (event) {
   if (event === GRACEFUL_SHUT) {
     const message = chalk.red('[fastify-cli] process forced end')
-    setTimeout(exit.bind({ message }), TIMEOUT).unref()
+    setTimeout(exit.bind({ message }), closeGraceDelay).unref()
     if (fastify) {
       fastify.close(() => {
         process.exit(0)
@@ -37,7 +39,7 @@ process.on('uncaughtException', (err) => {
 })
 
 const main = async () => {
-  fastify = await runFastify(process.argv.splice(2))
+  fastify = await runFastify(args)
   const type = process.env.childEvent
 
   process.send({ type, err: null })

lib/watch/index.js

@@ -5,18 +5,77 @@ const cp = require('node:child_process')
 const chalk = require('chalk').default
 const { arrayToRegExp, logWatchVerbose } = require('./utils')
 const { GRACEFUL_SHUT } = require('./constants.js')
+const parseArgs = require('../../args')
 
 const EventEmitter = require('node:events')
 const chokidar = require('chokidar')
 const { loadEnvQuitely } = require('../../env-loader.js')
 const forkPath = path.join(__dirname, './fork.js')
 
 const watch = function (args, ignoreWatch, verboseWatch, followWatch) {
+  const { closeGraceDelay } = parseArgs(args)
   const emitter = new EventEmitter()
   let allStop = false
   let childs = []
+  // Restart requests remove children from childs before their cleanup completes.
+  const liveChildren = new Set()
+  let stopping
+
+  function removeListeners () {
+    process.removeListener('SIGINT', onSigint)
+    process.removeListener('SIGTERM', onSigterm)
+    process.removeListener('uncaughtException', onUncaughtException)
+  }
+
+  function shutdown (signal) {
+    if (stopping) return stopping
+    allStop = true
+    const childrenClosed = Array.from(liveChildren, closeChild)
+    stopping = finishShutdown(signal, childrenClosed)
+    return stopping
+  }
+
+  async function finishShutdown (signal, childrenClosed) {
+    const results = await Promise.allSettled([
+      closeWatcher(),
+      ...childrenClosed
+    ])
+    removeListeners()
+    const failure = results.find(result => result.status === 'rejected')
+    if (failure) {
+      console.error(failure.reason)
+      process.exitCode = 1
+    } else {
+      process.exitCode = signal === 'SIGINT' ? 130 : 143
+    }
+  }
+
+  async function closeWatcher () {
+    await watcher.close()
+  }
+
+  function closeChild (child) {
+    return new Promise(resolve => {
+      // Give the child's own deadline time to fire, but also bound failed IPC
+      // and children whose event loop cannot process the shutdown request.
+      const timer = setTimeout(() => child.kill('SIGKILL'), Number(closeGraceDelay) + 1000)
+      child.once('close', () => {
+        clearTimeout(timer)
+        resolve()
+      })
+      // IPC avoids a second signal when terminal Ctrl-C also reaches the child.
+      // If delivery fails, the deadline above still terminates the child.
+      try {
+        if (child.connected) child.send(GRACEFUL_SHUT, () => {})
+      } catch {
+        // The IPC channel can close between checking connected and sending.
+      }
+    })
+  }
+  const onSigint = () => { shutdown('SIGINT') }
+  const onSigterm = () => { shutdown('SIGTERM') }
   const stop = (watcher = null, err = null) => {
-    childs.forEach(function (child) {
+    liveChildren.forEach(function (child) {
       child.kill()
     })
 
@@ -26,14 +85,19 @@ const watch = function (args, ignoreWatch, verboseWatch, followWatch) {
     }
     if (watcher) {
       allStop = true
+      removeListeners()
       return watcher.close()
     }
   }
 
-  process.on('uncaughtException', () => {
+  const onUncaughtException = () => {
+    if (allStop) return
     stop()
     childs.push(run('restart'))
-  })
+  }
+  process.on('uncaughtException', onUncaughtException)
+  process.on('SIGINT', onSigint)
+  process.on('SIGTERM', onSigterm)
 
   let readyEmitted = false
 
@@ -47,6 +111,9 @@ const watch = function (args, ignoreWatch, verboseWatch, followWatch) {
       encoding: 'utf8'
     })
 
+    liveChildren.add(_child)
+    _child.once('close', () => liveChildren.delete(_child))
+
     _child.on('exit', function (code, signal) {
       if (childs.length === 0 && !allStop) {
         childs.push(run('restart'))
@@ -83,6 +150,7 @@ const watch = function (args, ignoreWatch, verboseWatch, followWatch) {
   const watcher = chokidar.watch(watchDir, { ignored: ignoredPattern })
   watcher.on('ready', function () {
     watcher.on('all', function (event, filepath) {
+      if (allStop) return
       if (verboseWatch) {
         logWatchVerbose(event, filepath)
       }

test/data/delayed-close-plugin.js

@@ -0,0 +1,13 @@
+'use strict'
+
+const { setTimeout } = require('node:timers/promises')
+
+module.exports = async function (fastify, opts) {
+  fastify.addHook('onListen', async () => {
+    setImmediate(() => process.stdout.write('application-ready\n'))
+  })
+  fastify.addHook('onClose', async () => {
+    await setTimeout(200)
+    process.stdout.write('application-closed\n')
+  })
+}

test/watch-fork.test.js

@@ -77,6 +77,19 @@ test('should force the exit when the server does not close in time', testOptions
   t.assert.match(output(), /process forced end/)
 })
 
+test('should use the configured IPC shutdown deadline', testOptions, async t => {
+  const { child, message, output } = forkChild(t, [
+    '-p', '0', '--close-grace-delay', '50',
+    './test/data/hanging-close-plugin.js'
+  ])
+  await message('ready')
+  child.send(GRACEFUL_SHUT)
+  const [code] = await once(child, 'close')
+
+  t.assert.strictEqual(code, 1)
+  t.assert.match(output(), /process forced end/)
+})
+
 test('should exit immediately on graceful shutdown when the server is not up yet', testOptions, async (t) => {
   const { child } = forkChild(t, ['-p', '0', './test/data/slow-plugin.js'])
   child.send(GRACEFUL_SHUT)

test/watch-shutdown.test.js

@@ -0,0 +1,86 @@
+'use strict'
+
+const { test } = require('node:test')
+const { spawn } = require('node:child_process')
+const { once } = require('node:events')
+const path = require('node:path')
+const net = require('node:net')
+
+const cliPath = path.join(__dirname, '..', 'cli.js')
+const pluginPath = path.join(__dirname, 'data', 'delayed-close-plugin.js')
+// Windows kill() terminates processes rather than delivering catchable POSIX signals.
+const testOptions = { timeout: 15000, skip: process.platform === 'win32' }
+const signals = [
+  { signal: 'SIGINT', expectedCode: 130 },
+  { signal: 'SIGTERM', expectedCode: 143 }
+]
+const targets = ['parent only']
+// Negative PIDs address process groups on POSIX, but are not supported on Windows.
+if (process.platform !== 'win32') targets.push('process group')
+
+async function getAvailablePort (t) {
+  const server = net.createServer().listen(0, '127.0.0.1')
+  t.after(() => server.close())
+  await once(server, 'listening')
+  const { port } = server.address()
+  await new Promise(resolve => server.close(resolve))
+  return port
+}
+
+async function startWatcher (t) {
+  const port = await getAvailablePort(t)
+  const child = spawn(process.execPath, [
+    cliPath, 'start', '--watch',
+    '--port', String(port), '--address', '127.0.0.1',
+    '--follow-watch', pluginPath,
+    pluginPath
+  ], { stdio: ['ignore', 'pipe', 'pipe'], detached: true })
+
+  t.after(() => {
+    // Clean up the whole tree even when the test signals only the supervisor.
+    try {
+      process.kill(-child.pid, 'SIGKILL')
+    } catch (err) {
+      if (err.code !== 'ESRCH') throw err
+    }
+  })
+
+  let stdout = ''
+  let stderr = ''
+  let outputAtExit = ''
+  child.stderr.on('data', chunk => { stderr += chunk })
+  child.once('exit', () => { outputAtExit = stdout })
+  const closed = once(child, 'close')
+  const ready = new Promise((resolve, reject) => {
+    child.once('error', reject)
+    child.once('exit', () => reject(new Error(`Watcher exited before ready: ${stderr}`)))
+    child.stdout.on('data', chunk => {
+      stdout += chunk
+      if (stdout.includes('application-ready')) resolve()
+    })
+  })
+
+  return { child, ready, closed, outputAtExit: () => outputAtExit, errors: () => stderr }
+}
+
+for (const { signal, expectedCode } of signals) {
+  for (const target of targets) {
+    test(`should await application cleanup on ${signal} (${target})`, testOptions, async t => {
+      const processGroup = target === 'process group'
+      const watcher = await startWatcher(t)
+      await watcher.ready
+
+      if (processGroup) {
+        process.kill(-watcher.child.pid, signal)
+      } else {
+        watcher.child.kill(signal)
+      }
+      const [code, exitSignal] = await watcher.closed
+
+      // Checking final stdout alone would also accept cleanup after the parent exited.
+      t.assert.match(watcher.outputAtExit(), /application-closed/)
+      t.assert.strictEqual(code, expectedCode, watcher.errors())
+      t.assert.strictEqual(exitSignal, null)
+    })
+  }
+}