perf: cut nightly load and expose pool observability

containers/nginx/config/conf.d/status.conf

@@ -12,4 +12,20 @@ server {
     location /status {
         stub_status on;
     }
+
+    # php-fpm pool status. Read it with:
+    #   podman exec parceler-nginx curl -s localhost/fpm-status
+    # "max children reached" is the counter that proves pool saturation.
+    #
+    # "listen localhost" above resolves to both 127.0.0.1 and ::1 at nginx
+    # startup, so this server - not default.conf's catch-all - is the one that
+    # actually answers loopback requests on port 80, path or Host header
+    # notwithstanding. That's also why no per-location allow/deny is needed:
+    # the server-level allow/deny above already covers it. fastcgi_pass is
+    # hardcoded because this file is static conf.d, not an envsubst template.
+    location = /fpm-status {
+        fastcgi_pass php:9000;
+        fastcgi_param SCRIPT_FILENAME $fastcgi_script_name;
+        include fastcgi_params;
+    }
 }

containers/nginx/config/templates/default.conf.template

@@ -29,13 +29,40 @@ server {
         application/xml+rss
         image/svg+xml;
 
+    # opkg/wget send no credentials on the first request and wait for a
+    # WWW-Authenticate challenge before retrying. Answer that challenge here
+    # instead of booting PHP for it. Requests that DO carry credentials fall
+    # through to Laravel, where ForceBasicAuth + the licence middleware run
+    # as normal.
+    location ~ ^/repository/(community|enterprise)/ {
+        if ($http_authorization = "") {
+            add_header WWW-Authenticate "Basic" always;
+            return 401;
+        }
+        try_files $uri $uri/ /index.php?$query_string;
+    }
+
     location / {
         try_files $uri $uri/ /index.php?$query_string;
     }
 
     location = /favicon.ico { access_log off; log_not_found off; }
     location = /robots.txt  { access_log off; log_not_found off; }
 
+    # php-fpm pool status, in-pod only. conf.d/status.conf's "listen localhost"
+    # server actually answers loopback requests to this path (see there for the
+    # reachable copy) - this copy exists so requests arriving via the published
+    # port, which land here instead, still get denied rather than 404ing.
+    location = /fpm-status {
+        allow 127.0.0.1;
+        allow ::1;
+        deny all;
+        access_log off;
+        fastcgi_pass ${FPM_HOST}:${FPM_PORT};
+        fastcgi_param SCRIPT_FILENAME $fastcgi_script_name;
+        include fastcgi_params;
+    }
+
     location ~ \.php$ {
         fastcgi_pass ${FPM_HOST}:${FPM_PORT};
         fastcgi_param SCRIPT_FILENAME $realpath_root$fastcgi_script_name;

containers/php/Dockerfile

@@ -10,6 +10,7 @@ ADD --chmod=777 \
     --checksum=sha256:206a8f9b2177703fc5aa924d85ad6c72e82413e2d09635b4c9c82a1b65b5b3d5 \
     https://github.com/eficode/wait-for/releases/download/v2.2.4/wait-for /usr/local/bin/wait-for
 COPY containers/php/config/conf.d/expose_php.ini /usr/local/etc/php/conf.d/expose_php.ini
+COPY containers/php/config/php-fpm.d/zzz-observability.conf /usr/local/etc/php-fpm.d/zzz-observability.conf
 # configure entrypoints and image
 ARG PHP_EXTENSIONS
 ARG ADDITIONAL_PACKAGES
@@ -48,6 +49,7 @@ FROM base AS production
 RUN install-php-extensions opcache \
     && mv "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini"
 COPY containers/php/config/conf.d/opcache.ini /usr/local/etc/php/conf.d/opcache.ini
+COPY containers/php/config/php-fpm.d/zzz-pool.conf /usr/local/etc/php-fpm.d/zzz-pool.conf
 ARG COMPOSER_ALLOW_SUPERUSER=1
 COPY --from=vendor /var/www/html/composer.json .
 COPY --from=vendor /var/www/html/composer.lock .

containers/php/config/php-fpm.d/zzz-observability.conf

@@ -0,0 +1,20 @@
+; Request-level observability for the fpm pool.
+;
+; The default access.format carries no timing field, which meant production
+; latency could not be measured from the logs at all - the nginx access log
+; has no $request_time either. %{milli}d closes that gap: every request's
+; duration lands in journald, so peak-vs-baseline latency is a log query
+; rather than a guess.
+
+[www]
+; %{milli}d = wall time in ms, %{kilo}M = peak memory, %C = CPU percent.
+access.format = "%R - %u %t \"%m %r%Q%q\" %s %{milli}d %{kilo}M %C%%"
+
+; Backtrace anything pathologically slow. Goes to stderr, which the container
+; runtime forwards to journald alongside the access log.
+slowlog = /proc/self/fd/2
+request_slowlog_timeout = 5s
+
+; Pool status page, exposed by nginx on localhost only. "max children reached"
+; in its output is the definitive saturation counter.
+pm.status_path = /fpm-status

containers/php/config/php-fpm.d/zzz-pool.conf

@@ -0,0 +1,27 @@
+; Pool sizing for the production image.
+;
+; The upstream php:8.5-fpm-alpine default of pm.max_children = 5 caps throughput
+; at roughly 16 rps given the ~0.3s per-request cost of this workload, while the
+; nightly NethSecurity update window peaks at ~37 rps (133k requests in the
+; 04:00 hour, reproducible night to night). The pool was oversubscribed ~2.3x
+; for four hours a night, which showed up as ~13k client-abandoned connections
+; (HTTP 499) per hour at peak.
+;
+; This workload is I/O bound - each request waits on S3 and, on cache miss, on
+; the My Nethesis licence endpoint - so the worker count is set well above the
+; 4 available cores. At ~50MB RSS per worker, 32 workers is ~1.6GB, which is
+; comfortable on a 7.7GB host that also runs nginx, the queue worker, the
+; scheduler, the nightwatch agent and redis.
+;
+; Do not set `listen` here: it is defined in the upstream zz-docker.conf and
+; overriding it would break nginx's fastcgi_pass.
+
+[www]
+pm = dynamic
+pm.max_children = 32
+pm.start_servers = 8
+pm.min_spare_servers = 4
+pm.max_spare_servers = 12
+
+; Recycle workers periodically so memory growth in long-lived processes is bounded.
+pm.max_requests = 500