From 77f11bf00f2d135ee38639249d4d23a65c9908f8 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Thu, 19 Nov 2026 21:45:25 +0100 Subject: [PATCH] Freeze the objects allocated before forking the gunicorn workers With --preload, the workers share the master's memory until they write to it, and a full garbage collection writes to every object it traverses: when the preloaded heap is large, this unshares it in every process. With the v0.0.655 production image and traffic, each of the four gunicorn processes got its own copy of ~170 MB of preloaded memory within a minute or two (~420 MB -> ~890 MB for the container), which gc.freeze() in the pre_fork hook prevents (~405 MB). As recommended by the gc.freeze() documentation, collections are also disabled while the app is preloaded, so that they don't leave holes in the shared pages that the workers would fill. They are enabled again after freezing, in the master too, as it keeps running. Now that the web processes don't import the ML and AI libraries anymore, there is little left to unshare and the freeze makes no measurable difference: this is insurance against heavy imports creeping back into them. --- http_service/Dockerfile | 2 +- http_service/bugbug_http/gunicorn_conf.py | 23 +++++++++++++++++++++++ scripts/integration_test.sh | 2 +- 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 http_service/bugbug_http/gunicorn_conf.py diff --git a/http_service/Dockerfile b/http_service/Dockerfile index 192d9660c62..ac05994f01b 100644 --- a/http_service/Dockerfile +++ b/http_service/Dockerfile @@ -19,4 +19,4 @@ ENV MPLCONFIGDIR=/opt/matplotlib RUN python -c "import matplotlib.font_manager" && chmod -R a+rwX $MPLCONFIGDIR # Run the Pulse listener in the background -CMD (bugbug-http-pulse-listener &) && gunicorn -b 0.0.0.0:$PORT bugbug_http.app --preload --timeout 30 -w 3 +CMD (bugbug-http-pulse-listener &) && gunicorn -c python:bugbug_http.gunicorn_conf -b 0.0.0.0:$PORT bugbug_http.app --preload --timeout 30 -w 3 diff --git a/http_service/bugbug_http/gunicorn_conf.py b/http_service/bugbug_http/gunicorn_conf.py new file mode 100644 index 00000000000..6ff0f12c1ad --- /dev/null +++ b/http_service/bugbug_http/gunicorn_conf.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Gunicorn configuration for the web processes (loaded with ``-c python:bugbug_http.gunicorn_conf``).""" + +import gc + +# With --preload, the workers share the master's memory (the imported modules) until they write to +# it, and a garbage collection writes to every object it traverses. Following +# https://docs.python.org/3/library/gc.html#gc.freeze, collections are disabled while the app is +# preloaded (this module is loaded before it), so that they don't free objects and leave holes +# that the workers' allocations would fill, writing to the shared pages. +gc.disable() + + +def pre_fork(server, worker): + # Move the objects allocated so far to the permanent generation, which collections don't + # traverse, then enable the collections again: in the master, which keeps running (e.g. Sentry's + # threads), and in the worker, which inherits them. + gc.freeze() + gc.enable() diff --git a/scripts/integration_test.sh b/scripts/integration_test.sh index a3ebee110c8..73f24e711f8 100755 --- a/scripts/integration_test.sh +++ b/scripts/integration_test.sh @@ -61,7 +61,7 @@ sleep 1 redis-cli -n 4 FLUSHDB # Start the http server -gunicorn -b 127.0.0.1:8000 bugbug_http.app --preload --timeout 30 -w 3 & +gunicorn -c python:bugbug_http.gunicorn_conf -b 127.0.0.1:8000 bugbug_http.app --preload --timeout 30 -w 3 & gunicorn_pid=$! # Start the background worker