From df6d0ccae67b1fefb35ddf5805e275f62a3f3675 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Fri, 25 Sep 2026 17:12:05 +0000 Subject: [PATCH] root-ubuntu: size-capped log rotation, fail2ban, quiet networkd-dispatcher configure_logrotate: nginx logs rotate daily or past LOGROTATE_MAXSIZE (200M), 14 kept, and logrotate.timer goes hourly so the cap is checked. dev2 had a 1.2 GB and a 516 MB access log after two days. configure_fail2ban: sshd jail layered on Ubuntu's defaults, bans double on each repeat (1h up to 1w), ignoreip = loopback, the tailnet, the operator's FAIL2BAN_IGNOREIP and whoever is connected over ssh while it runs. Reads auth.log, or the journal on a box without rsyslog. quiet_networkd_dispatcher: with Docker, every container stop makes the dispatcher run networkctl on a veth that is already gone and log an error: 3,895 in six hours on dev2. Stopped only when it has no hooks. Co-Authored-By: Claude Opus 5.5 (1M context) --- root-ubuntu.sh | 213 +++++++++++++++++++++++++++++++++++++++ test/root-ubuntu.test.ts | 179 ++++++++++++++++++++++++++++++++ 2 files changed, 392 insertions(+) diff --git a/root-ubuntu.sh b/root-ubuntu.sh index 24836d2..e613f7a 100755 --- a/root-ubuntu.sh +++ b/root-ubuntu.sh @@ -132,6 +132,13 @@ # EARLYOOM_ENABLE=1 kill the biggest hog early instead of at the wall # EARLYOOM_MEM=10,5 / EARLYOOM_SWAP=10,5 SIGTERM% , SIGKILL% # EARLYOOM_AVOID=... / EARLYOOM_PREFER= unquoted regexes (see the note) +# LOGROTATE_ENABLE=1 nginx logs rotate early past LOGROTATE_MAXSIZE, hourly timer +# LOGROTATE_MAXSIZE=200M / LOGROTATE_KEEP=14 +# FAIL2BAN_ENABLE=1 ban repeated SSH login failures +# FAIL2BAN_BANTIME=1h / FAIL2BAN_MAXTIME=1w first ban, doubling to this ceiling +# FAIL2BAN_FINDTIME=10m / FAIL2BAN_MAXRETRY=5 +# FAIL2BAN_IGNOREIP= extra addresses/CIDRs never banned (your admin boxes) +# NETWORKD_DISPATCHER_QUIET=1 stop networkd-dispatcher when it has no hooks # ASSUME_YES=1 don't prompt (defaults: $DEFAULT_GROUPS; no privkey copy) # DEFAULT_GROUPS=... groups an account lands in when --groups is not passed. # An unattended run never prompts, so this is what every @@ -4363,6 +4370,212 @@ log "configuring OOM protection" try "earlyoom" configure_earlyoom try "zram" configure_zram +# ------------------------------------------------------------- logrotate --- + +# Stock Ubuntu rotates nginx daily and runs logrotate once a day. That is fine +# until a box serves real traffic: on dev2 (2026-09-25) one access log reached +# 1.2 GB in under two days and another 516 MB. maxsize rotates a log early once +# it passes the cap, but logrotate only looks when it runs, so the timer goes +# hourly too; `daily` still means daily for every log that stays small. +LOGROTATE_ENABLE="${LOGROTATE_ENABLE:-1}" # 0 leaves rotation as the distro ships it +LOGROTATE_MAXSIZE="${LOGROTATE_MAXSIZE:-200M}" +LOGROTATE_KEEP="${LOGROTATE_KEEP:-14}" + +configure_logrotate() { + local timer_changed=0 + + if [[ -z "$LOGROTATE_ENABLE" || "$LOGROTATE_ENABLE" == 0 ]]; then + info "logrotate left alone (LOGROTATE_ENABLE=$LOGROTATE_ENABLE)" + return 0 + fi + + if ! command -v logrotate >/dev/null 2>&1; then + try "install logrotate" apt-get install -y -qq logrotate || return 0 + fi + + # The whole stanza, not a sed on the distro's: logrotate refuses a second + # stanza for the same files ("duplicate log entry"), so a drop-in next to + # it cannot work, and unattended-upgrades keeps a changed conffile. + if [[ -d /etc/nginx ]]; then + write_if_changed /etc/logrotate.d/nginx <<-EOF && note "nginx logs rotate daily or past $LOGROTATE_MAXSIZE, $LOGROTATE_KEEP kept" + # Managed by root-ubuntu.sh (LOGROTATE_MAXSIZE, LOGROTATE_KEEP). + /var/log/nginx/*.log { + daily + maxsize $LOGROTATE_MAXSIZE + missingok + rotate $LOGROTATE_KEEP + compress + delaycompress + notifempty + create 0640 www-data adm + sharedscripts + prerotate + if [ -d /etc/logrotate.d/httpd-prerotate ]; then \\ + run-parts /etc/logrotate.d/httpd-prerotate; \\ + fi \\ + endscript + postrotate + invoke-rc.d nginx rotate >/dev/null 2>&1 + endscript + } + EOF + fi + + mkdir -p /etc/systemd/system/logrotate.timer.d + write_if_changed /etc/systemd/system/logrotate.timer.d/hourly.conf <<'EOF' && timer_changed=1 +# Managed by root-ubuntu.sh: hourly, so maxsize is checked more than once a day. +[Timer] +OnCalendar= +OnCalendar=hourly +EOF + if (( timer_changed )); then + systemctl daemon-reload + try "logrotate hourly" systemctl restart logrotate.timer && note "logrotate runs hourly" + else + info "logrotate already hourly" + fi + systemctl enable --quiet logrotate.timer 2>/dev/null || true + + # -d parses every config without rotating anything. A broken stanza + # anywhere in /etc/logrotate.d stops ALL rotation, so say so loudly. + if ! logrotate -d /etc/logrotate.conf >/dev/null 2>&1; then + warn "logrotate -d /etc/logrotate.conf reports errors; rotation may be stalled" + fi +} + +# -------------------------------------------------------------- fail2ban --- + +# Bans an address that keeps failing SSH logins. Ubuntu's package already +# enables the sshd jail; this layers our tuning on top in a file that sorts +# after defaults-debian.conf: repeat offenders are banned longer each time +# (bantime.increment doubles it, capped at FAIL2BAN_MAXTIME), and the +# addresses that must never be locked out are written into ignoreip. +FAIL2BAN_ENABLE="${FAIL2BAN_ENABLE:-1}" # 0 leaves fail2ban off the box +FAIL2BAN_BANTIME="${FAIL2BAN_BANTIME:-1h}" # first ban; doubles on each repeat +FAIL2BAN_MAXTIME="${FAIL2BAN_MAXTIME:-1w}" # ceiling for the doubling +FAIL2BAN_FINDTIME="${FAIL2BAN_FINDTIME:-10m}" +FAIL2BAN_MAXRETRY="${FAIL2BAN_MAXRETRY:-5}" +# Space-separated addresses or CIDRs that are never banned, on top of loopback, +# the tailnet range and whoever is connected over SSH while this runs. Put the +# boxes you administer from here. +FAIL2BAN_IGNOREIP="${FAIL2BAN_IGNOREIP:-}" + +# Addresses connected to sshd right now: the operator running this script. +# Banning them is a lockout, so they go in ignoreip. +_fail2ban_ssh_peers() { + local peer + [[ -n "${SSH_CLIENT:-}" ]] && printf '%s\n' "${SSH_CLIENT%% *}" + command -v ss >/dev/null 2>&1 || return 0 + ss -tnH state established "( sport = :${SSH_PORT:-22} )" 2>/dev/null \ + | awk '{print $4}' | while read -r peer; do + peer="${peer%:*}"; peer="${peer#[}"; peer="${peer%]}" + [[ -n "$peer" ]] && printf '%s\n' "$peer" + done + return 0 +} + +# ignoreip, deduplicated: loopback, the tailnet, the operator's list, and the +# SSH peers connected right now. +_fail2ban_ignoreip() { + local list + # shellcheck disable=SC2086,SC2046 # both are space-separated lists, split on purpose + list="$(printf '%s\n' 127.0.0.1/8 ::1 100.64.0.0/10 $FAIL2BAN_IGNOREIP $(_fail2ban_ssh_peers) \ + | awk 'NF && !seen[$0]++' | tr '\n' ' ')" + printf '%s' "${list% }" +} + +configure_fail2ban() { + local changed=0 ignore backend + + if [[ -z "$FAIL2BAN_ENABLE" || "$FAIL2BAN_ENABLE" == 0 ]]; then + info "fail2ban disabled (FAIL2BAN_ENABLE=$FAIL2BAN_ENABLE)" + return 0 + fi + + if ! dpkg -s fail2ban >/dev/null 2>&1; then + try "install fail2ban" apt-get install -y -qq fail2ban || return 0 + fi + + ignore="$(_fail2ban_ignoreip)" + + # auth.log is what the stock sshd filter reads. A box without rsyslog has + # only the journal, and there fail2ban needs python3-systemd to read it. + backend=auto + if [[ ! -f /var/log/auth.log ]]; then + backend=systemd + if ! dpkg -s python3-systemd >/dev/null 2>&1; then + try "install python3-systemd" apt-get install -y -qq python3-systemd || return 0 + fi + fi + + mkdir -p /etc/fail2ban/jail.d + write_if_changed /etc/fail2ban/jail.d/zz-root-ubuntu.local </dev/null || true + + if (( changed )) || ! systemctl is-active --quiet fail2ban; then + try "fail2ban" systemctl restart fail2ban || return 0 + note "fail2ban guarding sshd (ban $FAIL2BAN_BANTIME, doubling to $FAIL2BAN_MAXTIME; never: $ignore)" + else + info "fail2ban already guarding sshd" + fi + + # A jail that failed to load leaves the service running and nothing banned. + if ! fail2ban-client status sshd >/dev/null 2>&1; then + warn "fail2ban is running but the sshd jail is not loaded: fail2ban-client status sshd" + fi +} + +# ---------------------------------------------------- networkd-dispatcher --- + +# networkd-dispatcher runs hook scripts on link changes. With Docker on the box, +# every container stop deletes a veth, the dispatcher calls `networkctl status` +# on it a moment after it is gone, and logs 'ERROR:Failed to get interface ... +# not found': 3,895 of them in six hours on dev2 (2026-09-25), each one an +# err-priority journal line that monitoring then reports. With no hook scripts +# installed it does nothing else, so stop it; any box that has hooks keeps it. +NETWORKD_DISPATCHER_QUIET="${NETWORKD_DISPATCHER_QUIET:-1}" # 0 never touches it + +quiet_networkd_dispatcher() { + local hook + [[ -z "$NETWORKD_DISPATCHER_QUIET" || "$NETWORKD_DISPATCHER_QUIET" == 0 ]] && return 0 + systemctl cat networkd-dispatcher.service >/dev/null 2>&1 || return 0 + if ! systemctl is-enabled --quiet networkd-dispatcher 2>/dev/null \ + && ! systemctl is-active --quiet networkd-dispatcher 2>/dev/null; then + info "networkd-dispatcher already off" + return 0 + fi + hook="$(find /etc/networkd-dispatcher /usr/lib/networkd-dispatcher -type f 2>/dev/null | head -1)" + if [[ -n "$hook" ]]; then + info "networkd-dispatcher kept: it has hook scripts ($hook)" + return 0 + fi + try "stop networkd-dispatcher" systemctl disable --now networkd-dispatcher \ + && note "networkd-dispatcher off (no hooks; it only logged veth errors)" +} + +log "configuring log rotation" +try "logrotate" configure_logrotate + +log "configuring fail2ban" +try "fail2ban" configure_fail2ban + +log "quieting networkd-dispatcher" +try "networkd-dispatcher" quiet_networkd_dispatcher + # ------------------------------------------------------------------ motd --- # Fetched into a cache file; login just cats it and kicks off a background diff --git a/test/root-ubuntu.test.ts b/test/root-ubuntu.test.ts index 9ea2b61..9d0d7c4 100644 --- a/test/root-ubuntu.test.ts +++ b/test/root-ubuntu.test.ts @@ -1337,3 +1337,182 @@ describe('the user confine', () => { expect(SOURCE).toContain('confine is OFF (CONFINE=0)'); }); }); + +/** + * A temp dir standing in for /, and the functions under test rewritten to + * write inside it. systemctl/apt/dpkg/logrotate/fail2ban-client are stubbed: + * what is asserted is what lands on disk and what the run says it changed. + */ +function rooted(fns: string[], dir: string, extra = ''): string { + return ` + CHANGED=(); FAILED=(); log() { :; }; info() { echo "$*"; }; warn() { echo "warn: $*"; } + note() { echo "changed: $*"; } + try() { local d="$1"; shift; "$@" || { echo "failed: $d"; return 1; }; } + systemctl() { echo "systemctl $*" >> ${dir}/calls; case "$1" in is-active|is-enabled) return \${FAKE_ACTIVE:-1} ;; cat) return \${FAKE_UNIT:-0} ;; esac; return 0; } + apt-get() { echo "apt-get $*" >> ${dir}/calls; } + dpkg() { return 0; } + logrotate() { return 0; } + fail2ban-client() { return 0; } + ss() { printf '%s' "\${FAKE_SS:-}"; } + eval "$(declare -f ${fns.join(' ')} \\ + | sed 's#/etc/#${dir}/etc/#g; s#/var/log/#${dir}/var/log/#g; s#/usr/lib/#${dir}/usr/lib/#g')" + ${extra} + `; +} + +function fakeRoot(): string { + const dir = mkdtempSync(join(tmpdir(), 'root-ubuntu-rotate-')); + for (const d of ['etc/nginx', 'etc/logrotate.d', 'var/log', 'etc/networkd-dispatcher/routable.d']) { + mkdirSync(join(dir, d), { recursive: true }); + } + return dir; +} + +const decls = (keys: string[]) => + keys.map((k) => new RegExp(`^${k}=.*$`, 'm').exec(SOURCE)?.[0] ?? '').join('\n'); + +describe('configure_logrotate', () => { + const FNS = ['configure_logrotate', 'write_if_changed']; + const DECLS = decls(['LOGROTATE_ENABLE', 'LOGROTATE_MAXSIZE', 'LOGROTATE_KEEP']); + const run = (dir: string, env = '') => + shell(FNS, `${rooted(FNS, dir, DECLS)}\n${env} configure_logrotate`); + + it('caps nginx logs by size and makes the timer hourly', () => { + const dir = fakeRoot(); + const out = run(dir); + const conf = readFileSync(join(dir, 'etc/logrotate.d/nginx'), 'utf8'); + expect(conf).toContain('maxsize 200M'); + expect(conf).toContain('rotate 14'); + expect(conf).toContain('/var/log/nginx/*.log {'); + const timer = readFileSync(join(dir, 'etc/systemd/system/logrotate.timer.d/hourly.conf'), 'utf8'); + expect(timer).toContain('OnCalendar=\nOnCalendar=hourly'); + expect(out).toContain('changed: nginx logs rotate daily or past 200M, 14 kept'); + expect(out).toContain('changed: logrotate runs hourly'); + }); + + it('writes a stanza logrotate itself accepts', () => { + const dir = fakeRoot(); + run(dir); + const conf = join(dir, 'etc/logrotate.d/nginx'); + // The real logrotate, in debug mode: parses, rotates nothing. + const out = execFileSync('logrotate', ['-d', '-s', join(dir, 'state'), conf], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + expect(out).not.toMatch(/error/i); + }); + + it('changes nothing on a second run', () => { + const dir = fakeRoot(); + run(dir); + const out = run(dir); + expect(out).not.toContain('changed:'); + expect(out).toContain('logrotate already hourly'); + }); + + it('honours the knobs', () => { + const dir = fakeRoot(); + run(dir, 'LOGROTATE_MAXSIZE=50M LOGROTATE_KEEP=7'); + const conf = readFileSync(join(dir, 'etc/logrotate.d/nginx'), 'utf8'); + expect(conf).toContain('maxsize 50M'); + expect(conf).toContain('rotate 7'); + }); + + it('leaves everything alone when LOGROTATE_ENABLE=0', () => { + const dir = fakeRoot(); + expect(run(dir, 'LOGROTATE_ENABLE=0')).toContain('logrotate left alone'); + expect(() => readFileSync(join(dir, 'etc/logrotate.d/nginx'))).toThrow(); + }); +}); + +describe('configure_fail2ban', () => { + const FNS = ['configure_fail2ban', '_fail2ban_ignoreip', '_fail2ban_ssh_peers', 'write_if_changed']; + const DECLS = decls([ + 'FAIL2BAN_ENABLE', 'FAIL2BAN_BANTIME', 'FAIL2BAN_MAXTIME', 'FAIL2BAN_FINDTIME', + 'FAIL2BAN_MAXRETRY', 'FAIL2BAN_IGNOREIP', + ]); + const run = (dir: string, env = '') => + shell(FNS, `${rooted(FNS, dir, DECLS)}\n${env} configure_fail2ban`); + const jail = (dir: string) => readFileSync(join(dir, 'etc/fail2ban/jail.d/zz-root-ubuntu.local'), 'utf8'); + + it('guards sshd with bans that grow on each repeat', () => { + const dir = fakeRoot(); + writeFileSync(join(dir, 'var/log/auth.log'), ''); + run(dir); + const conf = jail(dir); + expect(conf).toContain('bantime.increment = true'); + expect(conf).toContain('bantime = 1h'); + expect(conf).toContain('bantime.maxtime = 1w'); + expect(conf).toMatch(/\[sshd\]\nenabled = true\nport = 22\nbackend = auto/); + }); + + it('never bans loopback, the tailnet, the admin list or the operator connected now', () => { + const dir = fakeRoot(); + writeFileSync(join(dir, 'var/log/auth.log'), ''); + run( + dir, + "FAIL2BAN_IGNOREIP='67.205.189.229 10.0.0.0/8' SSH_CLIENT='203.0.113.5 51234 22' " + + "FAKE_SS=$'0 0 23.95.228.174:22 198.51.100.7:40000\\n0 0 23.95.228.174:22 203.0.113.5:51234\\n'", + ); + const line = /^ignoreip = (.*)$/m.exec(jail(dir))?.[1].split(' ') ?? []; + expect(line).toEqual([ + '127.0.0.1/8', '::1', '100.64.0.0/10', '67.205.189.229', '10.0.0.0/8', '203.0.113.5', '198.51.100.7', + ]); + }); + + it('reads the journal on a box with no auth.log', () => { + const dir = fakeRoot(); + run(dir); + expect(jail(dir)).toContain('backend = systemd'); + }); + + it('follows SSH_PORT', () => { + const dir = fakeRoot(); + writeFileSync(join(dir, 'var/log/auth.log'), ''); + run(dir, 'SSH_PORT=2048'); + expect(jail(dir)).toContain('port = 2048'); + }); + + it('does not restart a correctly configured, running fail2ban', () => { + const dir = fakeRoot(); + writeFileSync(join(dir, 'var/log/auth.log'), ''); + run(dir); + const out = run(dir, 'FAKE_ACTIVE=0'); + expect(out).toContain('fail2ban already guarding sshd'); + expect(out).not.toContain('changed:'); + }); + + it('stays off when FAIL2BAN_ENABLE=0', () => { + const dir = fakeRoot(); + expect(run(dir, 'FAIL2BAN_ENABLE=0')).toContain('fail2ban disabled'); + }); +}); + +describe('quiet_networkd_dispatcher', () => { + const FNS = ['quiet_networkd_dispatcher']; + const DECLS = decls(['NETWORKD_DISPATCHER_QUIET']); + const run = (dir: string, env = '') => shell(FNS, `${rooted(FNS, dir, DECLS)}\n${env} quiet_networkd_dispatcher`); + const calls = (dir: string) => { + try { return readFileSync(join(dir, 'calls'), 'utf8'); } catch { return ''; } + }; + + it('stops it when it has no hook scripts', () => { + const dir = fakeRoot(); + expect(run(dir, 'FAKE_ACTIVE=0')).toContain('changed: networkd-dispatcher off'); + expect(calls(dir)).toContain('systemctl disable --now networkd-dispatcher'); + }); + + it('keeps it when a hook script exists', () => { + const dir = fakeRoot(); + writeFileSync(join(dir, 'etc/networkd-dispatcher/routable.d/50-vpn'), '#!/bin/sh\n'); + expect(run(dir, 'FAKE_ACTIVE=0')).toContain('networkd-dispatcher kept'); + expect(calls(dir)).not.toContain('disable'); + }); + + it('does nothing where it is not installed or already off', () => { + const dir = fakeRoot(); + run(dir, 'FAKE_UNIT=1'); + expect(calls(dir)).not.toContain('disable'); + expect(run(dir, 'FAKE_ACTIVE=1')).toContain('already off'); + }); +});