From e5947fdc66774dfb41bedbfaeedff4e4248dc67f Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Mon, 16 Nov 2026 18:28:23 +0100 Subject: [PATCH] feat(diagnostics, ticket-136): detect host anomalies beyond cache cleanup Add 'fixos anomalies': read-only detection of systemd restart loops, container logs without rotation, large stale directories in tool state stores, regenerable IDE indexes, and zombie or leaked identical processes. Removal is limited to stale-state and idle IDE-index findings confirmed by a fresh scan, inside the user's home, with a receipt. Co-Authored-By: Claude Fable 5.2 --- fixos/cli/anomalies_cmd.py | 103 ++++++ fixos/cli/main.py | 1 + fixos/diagnostics/host_anomalies.py | 502 ++++++++++++++++++++++++++++ project/ticket-136/README.md | 47 +++ project/ticket-136/intent.json | 99 ++++++ tests/unit/test_host_anomalies.py | 254 ++++++++++++++ 6 files changed, 1006 insertions(+) create mode 100644 fixos/cli/anomalies_cmd.py create mode 100644 fixos/diagnostics/host_anomalies.py create mode 100644 project/ticket-136/README.md create mode 100644 project/ticket-136/intent.json create mode 100644 tests/unit/test_host_anomalies.py diff --git a/fixos/cli/anomalies_cmd.py b/fixos/cli/anomalies_cmd.py new file mode 100644 index 0000000..0a532c1 --- /dev/null +++ b/fixos/cli/anomalies_cmd.py @@ -0,0 +1,103 @@ +"""`fixos anomalies`: report host anomalies, remove only selected stale data.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import click + +from fixos.cli._cleanup_utils import _format_bytes +from fixos.diagnostics.host_anomalies import GIB, remove_paths, scan_all + +_ORDER = {"high": 0, "medium": 1, "review": 2, "safe": 3, "low": 4, "protected": 5} + + +@click.command("anomalies") +@click.option("--json", "json_output", is_flag=True, help="Wynik maszynowy JSON.") +@click.option( + "--min-size-gb", + type=click.FloatRange(min=0.01), + default=1.0, + show_default=True, + help="Minimalny rozmiar katalogu stanu lub indeksu IDE.", +) +@click.option( + "--min-age-days", + type=click.IntRange(min=1), + default=14, + show_default=True, + help="Katalog stanu jest kandydatem, gdy nic w nim nie zmieniono tyle dni.", +) +@click.option( + "--remove", + "remove", + multiple=True, + type=click.Path(path_type=Path), + help="Usuń wskazany katalog; tylko jeśli bieżący skan oznacza go jako usuwalny.", +) +@click.option( + "--remove-safe", + is_flag=True, + help="Usuń wszystkie pozycje 'safe' (indeksy IDE, gdy IDE nie działa).", +) +@click.option("--dry-run", is_flag=True, help="Pokaż, co zostałoby usunięte.") +def anomalies(json_output, min_size_gb, min_age_days, remove, remove_safe, dry_run): + """Wykrywa anomalie hosta, których nie widzi zwykłe czyszczenie cache. + + \b + Sprawdza: usługi systemd w pętli restartów, logi kontenerów bez rotacji, + duże nieużywane katalogi w ~/.local/state i ~/.local/share, indeksy IDE + oraz procesy zombie i grupy osieroconych procesów. + + \b + Przykłady: + fixos anomalies # raport + fixos anomalies --json # raport maszynowy + fixos anomalies --remove-safe # usuń indeksy niedziałających IDE + fixos anomalies --remove ~/.local/state/x/backups --dry-run + """ + report = scan_all(min_bytes=int(min_size_gb * GIB), min_age_days=min_age_days) + findings = report["findings"] + targets = [path.expanduser() for path in remove] + if remove_safe: + targets += [ + Path(f["subject"]) + for f in findings + if f["removable"] and f["severity"] == "safe" + ] + if targets: + if dry_run: + report["would_remove"] = [ + str(path) + for path in targets + if any(f["removable"] and f["subject"] == str(path) for f in findings) + ] + else: + report["removal"] = remove_paths( + targets, + findings, + receipt_dir=Path.home() / ".local/state/fixos/receipts", + ) + if json_output: + click.echo(json.dumps(report, ensure_ascii=False, indent=2)) + return + if not findings: + click.echo("Nie wykryto anomalii.") + for item in sorted(findings, key=lambda f: _ORDER.get(f["severity"], 9)): + click.echo(f"[{item['severity']}] {item['category']}: {item['subject']}") + click.echo(f" {item['detail']}") + for command in item["suggested_commands"]: + click.echo(f" $ {command}") + click.echo( + f"\nDo odzyskania po przeglądzie: {_format_bytes(report['reclaimable_bytes'])}" + ) + for path in report.get("would_remove", []): + click.echo(f"[DRY-RUN] rm -rf {path}") + removal = report.get("removal") + if removal: + for item in removal["removed"]: + click.echo(f"usunięto: {item['path']}") + for item in removal["refused"]: + click.echo(f"pominięto: {item['path']} ({item['reason']})") + click.echo(f"Zwolniono: {_format_bytes(removal['freed_bytes'])}") diff --git a/fixos/cli/main.py b/fixos/cli/main.py index 57bfd49..e66dd63 100644 --- a/fixos/cli/main.py +++ b/fixos/cli/main.py @@ -9,6 +9,7 @@ from fixos.config import FixOsConfig LAZY_COMMANDS: dict[str, tuple[str, str]] = { + "anomalies": ("fixos.cli.anomalies_cmd", "anomalies"), "ask": ("fixos.cli.ask_cmd", "ask"), "auto": ("fixos.cli.fix_cmd", "auto_cmd"), "cleanup": ("fixos.cli.cleanup_cmd", "cleanup_services"), diff --git a/fixos/diagnostics/host_anomalies.py b/fixos/diagnostics/host_anomalies.py new file mode 100644 index 0000000..e13bdec --- /dev/null +++ b/fixos/diagnostics/host_anomalies.py @@ -0,0 +1,502 @@ +"""Host anomalies that ordinary cache cleanup does not see. + +Five read-only detectors, each grounded in a failure observed on a shared +build host: a service restarting in a loop, container logs growing without +rotation, large stale directories in tool state stores, regenerable IDE +indexes, and leaked or unreaped processes. + +Detectors never mutate. Only :func:`remove_paths` deletes, and only paths that a +fresh scan still reports as removable inside the user's own state roots. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import time +from collections import Counter +from collections.abc import Callable, Sequence +from pathlib import Path + +from fixos.diagnostics.artifact_store import is_protected_path + +SCHEMA = "fixos.host-anomalies/v1" +RECEIPT_SCHEMA = "fixos.host-anomaly-removal/v1" +STATE_ROOTS = (".local/state", ".local/share") +REMOVABLE = frozenset({"stale-state", "ide-index"}) +GIB = 1024**3 +MIB = 1024**2 + +Runner = Callable[[Sequence[str]], str] +DU_TIMEOUT_SECONDS = 1800 +ROTATION_HINT = ( + 'dodaj do daemon.json: "log-driver": "json-file", ' + '"log-opts": {"max-size": "50m", "max-file": "3"}' +) + + +def _run(command: Sequence[str]) -> str: + """Bounded command; an unavailable tool is an empty observation. + + ``du`` over a multi-hundred-gigabyte state store takes minutes on a busy + host, so it gets a longer bound than the quick status queries. + """ + timeout = DU_TIMEOUT_SECONDS if command and command[0] == "du" else 60 + try: + done = subprocess.run( + list(command), capture_output=True, text=True, check=False, timeout=timeout + ) + except (OSError, subprocess.TimeoutExpired): + return "" + return done.stdout + + +def _finding(category, severity, subject, detail, commands=(), **evidence) -> dict: + return { + "category": category, + "severity": severity, + "subject": str(subject), + "detail": detail, + "suggested_commands": list(commands), + "removable": category in REMOVABLE and severity != "protected", + "evidence": evidence, + } + + +# --- 1. services restarting in a loop --------------------------------------- + + +def scan_restart_loops(*, run: Runner = _run, min_restarts: int = 50) -> list[dict]: + """Units that systemd keeps restarting; each start costs CPU and log volume.""" + findings = [] + for scope in ("system", "user"): + flag = ["--user"] if scope == "user" else [] + output = run( + ["systemctl", *flag, "show", "*.service", "--no-pager"] + + ["-p", "Id", "-p", "NRestarts", "-p", "ActiveState", "-p", "SubState"] + ) + for block in output.split("\n\n"): + unit = dict( + line.split("=", 1) for line in block.splitlines() if "=" in line + ) + try: + restarts = int(unit.get("NRestarts", "0")) + except ValueError: + continue + if restarts < min_restarts or not unit.get("Id"): + continue + looping = unit.get("SubState") == "auto-restart" or ( + unit.get("ActiveState") == "activating" + ) + if not looping and restarts < min_restarts * 10: + continue + ctl = " ".join(["systemctl", *flag]) + findings.append( + _finding( + "restart-loop", + "high" if looping else "medium", + unit["Id"], + f"{restarts} restartów, stan {unit.get('ActiveState')}/" + f"{unit.get('SubState')} ({scope}).", + [ + " ".join(["journalctl", *flag, "-u", unit["Id"]]) + + " -n 50 --no-pager", + f"{ctl} disable --now {unit['Id']}", + ], + scope=scope, + restarts=restarts, + looping=looping, + ) + ) + return sorted(findings, key=lambda f: -f["evidence"]["restarts"]) + + +# --- 2. container logs without rotation ------------------------------------- + + +def scan_container_logs( + *, + containers_root: Path = Path("/var/lib/docker/containers"), + daemon_json: Path = Path("/etc/docker/daemon.json"), + min_bytes: int = 100 * MIB, +) -> list[dict]: + """json-file logs grow forever unless the daemon sets ``log-opts.max-size``.""" + try: + config = json.loads(daemon_json.read_text(encoding="utf-8")) + except (OSError, ValueError): + config = {} + driver = config.get("log-driver", "json-file") + rotated = driver != "json-file" or bool( + (config.get("log-opts") or {}).get("max-size") + ) + logs, unreadable = [], False + try: + for entry in containers_root.iterdir(): + for log in entry.glob("*-json.log"): + size = log.stat().st_size + if size >= min_bytes: + logs.append((size, log)) + except PermissionError: + unreadable = True + except OSError: + return [] + findings = [] + if not rotated and (logs or unreadable): + findings.append( + _finding( + "container-logs", + "medium", + daemon_json, + "Docker nie rotuje logów kontenerów (brak log-opts.max-size).", + [ + ROTATION_HINT, + f"dockerd --validate --config-file {daemon_json}", + ], + rotation_configured=False, + unreadable=unreadable, + ) + ) + for size, log in sorted(logs, reverse=True): + findings.append( + _finding( + "container-logs", + "low", + log, + f"Log kontenera {log.parent.name[:12]}: {size / MIB:.0f} MiB.", + [f"truncate -s 0 {log}"], + size_bytes=size, + ) + ) + return findings + + +# --- 3. large stale directories in tool state stores ------------------------ + + +def _is_stale(path: Path, cutoff: float) -> bool: + """True when nothing below *path* changed after *cutoff*; stops at first hit.""" + try: + for parent, _dirs, files in os.walk(path, followlinks=False): + for name in files: + if os.lstat(os.path.join(parent, name)).st_mtime > cutoff: + return False + except OSError: + return False + return True + + +def _du(root: Path, depth: int, run: Runner) -> dict[Path, int]: + sizes = {} + output = run(["du", "-x", "-B1", f"--max-depth={depth}", str(root)]) + for line in output.splitlines(): + size, _, name = line.partition("\t") + if size.isdigit() and name: + sizes[Path(name)] = int(size) + return sizes + + +def _visit_state( + directory: Path, + depth: int, + *, + sizes: dict[Path, int], + home: Path, + cutoff: float, + min_bytes: int, + min_age_days: int, + max_depth: int, + findings: list[dict], +) -> None: + """Report the top-most wholly stale directory; descend only into live ones.""" + try: + children = [c for c in directory.iterdir() if c.is_dir() and not c.is_symlink()] + except OSError: + return + for child in sorted(children): + size = sizes.get(child, 0) + if size < min_bytes or child.is_mount(): + continue + if is_protected_path(child, home=home): + continue + if _is_stale(child, cutoff): + findings.append( + _finding( + "stale-state", + "review", + child, + f"{size / GIB:.1f} GiB bez zmian od ponad {min_age_days} dni.", + [f"du -sh {child}", f"rm -rf {child}"], + size_bytes=size, + ) + ) + elif depth < max_depth: + _visit_state( + child, + depth + 1, + sizes=sizes, + home=home, + cutoff=cutoff, + min_bytes=min_bytes, + min_age_days=min_age_days, + max_depth=max_depth, + findings=findings, + ) + + +def scan_stale_state( + home: Path, + *, + run: Runner = _run, + min_bytes: int = GIB, + min_age_days: int = 14, + max_depth: int = 4, + now: float | None = None, +) -> list[dict]: + """Largest untouched directories below ``~/.local/state`` and ``~/.local/share``. + + Reports the top-most directory that is wholly older than the threshold, so + one abandoned run or backup is one candidate. Always a review decision: + state stores hold evidence as well as waste. + """ + cutoff = (now or time.time()) - min_age_days * 86400 + findings = [] + for relative in STATE_ROOTS: + root = home / relative + if root.is_symlink() or not root.is_dir(): + continue + _visit_state( + root, + 1, + sizes=_du(root, max_depth, run), + home=home, + cutoff=cutoff, + min_bytes=min_bytes, + min_age_days=min_age_days, + max_depth=max_depth, + findings=findings, + ) + return sorted(findings, key=lambda f: -f["evidence"]["size_bytes"]) + + +# --- 4. regenerable IDE indexes --------------------------------------------- + + +def _running_commands(proc: Path) -> list[str]: + commands = [] + try: + entries = [e for e in proc.iterdir() if e.name.isdigit()] + except OSError: + return commands + for entry in entries: + try: + raw = (entry / "cmdline").read_bytes() + except OSError: + continue + commands.append(raw.replace(b"\0", b" ").decode("utf-8", "replace")) + return commands + + +def scan_ide_indexes( + home: Path, + *, + run: Runner = _run, + proc: Path = Path("/proc"), + min_bytes: int = GIB, +) -> list[dict]: + """JetBrains ``index`` directories are rebuilt on start; protect a running IDE.""" + root = home / ".cache" / "JetBrains" + if root.is_symlink() or not root.is_dir(): + return [] + running = _running_commands(proc) + findings = [] + for product in sorted(p for p in root.iterdir() if p.is_dir()): + index = product / "index" + if index.is_symlink() or not index.is_dir(): + continue + size = _du(index, 0, run).get(index, 0) + if size < min_bytes: + continue + in_use = any(product.name in command for command in running) + findings.append( + _finding( + "ide-index", + "protected" if in_use else "safe", + index, + f"Indeks {product.name}: {size / GIB:.1f} GiB" + + (" (IDE działa, pominięto)." if in_use else ", odtwarzalny."), + [] if in_use else [f"rm -rf {index}"], + size_bytes=size, + ide_running=in_use, + ) + ) + return findings + + +# --- 5. leaked and unreaped processes --------------------------------------- + + +def scan_processes( + *, + proc: Path = Path("/proc"), + min_group: int = 50, + min_zombies: int = 10, + min_age_seconds: int = 86400, + uptime: float | None = None, + ticks: int | None = None, +) -> list[dict]: + """Zombies per parent and big groups of identical long-idle processes. + + A group is keyed by the complete command line: leaked helpers repeat one + exact invocation, while daemons that merely share a name (one proxy per + port, one shim per container) differ and are not reported. + """ + try: + ticks = ticks or os.sysconf("SC_CLK_TCK") + if uptime is None: + uptime = float((proc / "uptime").read_text().split()[0]) + except (OSError, ValueError, IndexError): + return [] + zombies: Counter[int] = Counter() + idle: Counter[tuple[str, str, int]] = Counter() + for entry in proc.iterdir(): + if not entry.name.isdigit(): + continue + try: + stat = (entry / "stat").read_text() + except OSError: + continue + comm = stat[stat.find("(") + 1 : stat.rfind(")")] + fields = stat[stat.rfind(")") + 2 :].split() + if len(fields) < 20: + continue + state, parent = fields[0], int(fields[1]) + age = uptime - int(fields[19]) / ticks + if state == "Z": + zombies[parent] += 1 + elif state == "S" and age >= min_age_seconds: + try: + cmdline = (entry / "cmdline").read_bytes() + except OSError: + continue + if cmdline: + text = cmdline.replace(b"\0", b" ").decode("utf-8", "replace") + idle[(comm, text.strip(), parent)] += 1 + findings = [] + for parent, count in zombies.most_common(): + if count < min_zombies: + break + findings.append( + _finding( + "zombie-processes", + "medium", + f"pid {parent}", + f"{count} procesów zombie: rodzic nie odbiera potomków " + "(w kontenerze uruchom z init: true).", + [f"ps -o pid,etimes,args -p {parent}"], + parent_pid=parent, + count=count, + ) + ) + for (comm, cmdline, parent), count in idle.most_common(): + if count < min_group: + break + findings.append( + _finding( + "idle-process-group", + "medium", + comm, + f"{count} uśpionych procesów o identycznym poleceniu, starszych " + f"niż {min_age_seconds // 3600} h, pod rodzicem {parent}: " + f"{cmdline[:120]}", + [f"pgrep -a -x {comm} --older {min_age_seconds}"], + parent_pid=parent, + count=count, + ) + ) + return findings + + +# --- aggregate and removal -------------------------------------------------- + + +def scan_all( + home: Path | None = None, + *, + run: Runner = _run, + min_bytes: int = GIB, + min_age_days: int = 14, +) -> dict: + home = home or Path.home() + findings = ( + scan_restart_loops(run=run) + + scan_container_logs() + + scan_stale_state( + home, run=run, min_bytes=min_bytes, min_age_days=min_age_days + ) + + scan_ide_indexes(home, run=run, min_bytes=min_bytes) + + scan_processes() + ) + return { + "schema": SCHEMA, + "generated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "findings": findings, + "reclaimable_bytes": sum( + f["evidence"].get("size_bytes", 0) for f in findings if f["removable"] + ), + } + + +def remove_paths( + paths: Sequence[Path], + findings: Sequence[dict], + *, + home: Path | None = None, + receipt_dir: Path | None = None, +) -> dict: + """Delete only paths a current scan marks removable; write one receipt.""" + home = home or Path.home() + allowed = {f["subject"]: f for f in findings if f["removable"]} + removed, refused = [], [] + for path in paths: + finding = allowed.get(str(path)) + reason = None + if finding is None: + reason = "not_a_current_removable_finding" + elif path.is_symlink() or not path.is_dir(): + reason = "not_a_plain_directory" + elif home.resolve() not in path.resolve().parents: + reason = "outside_home" + elif is_protected_path(path, home=home): + reason = "protected_path" + if reason: + refused.append({"path": str(path), "reason": reason}) + continue + try: + shutil.rmtree(path) + except OSError as error: + refused.append({"path": str(path), "reason": f"os_error:{error.errno}"}) + continue + removed.append( + { + "path": str(path), + "category": finding["category"], + "size_bytes": finding["evidence"].get("size_bytes", 0), + } + ) + receipt = { + "schema": RECEIPT_SCHEMA, + "created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "removed": removed, + "refused": refused, + "freed_bytes": sum(item["size_bytes"] for item in removed), + } + if receipt_dir is not None and removed: + receipt_dir.mkdir(parents=True, exist_ok=True) + name = f"host-anomaly-removal-{int(time.time())}.json" + (receipt_dir / name).write_text( + json.dumps(receipt, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + receipt["receipt_path"] = str(receipt_dir / name) + return receipt diff --git a/project/ticket-136/README.md b/project/ticket-136/README.md new file mode 100644 index 0000000..c01f1d8 --- /dev/null +++ b/project/ticket-136/README.md @@ -0,0 +1,47 @@ +# Ticket 136: Host anomaly detection (restart loops, unrotated container logs, stale state stores, IDE indexes, leaked processes) + +- **ID**: ticket-136 +- **Owner**: unresolved:human +- **Status**: IN_PROGRESS +- **Workflow state**: EDIT +- **Created**: 2026-11-11 + +## Goal and scope + +Reused from the unassigned allocation "Release 2.4.0 …" (2026-11-11, never +committed), which blocked new allocation. The version bump is out of scope: +`VERSION` and `pyproject.toml` are not owned by the `application` workstream. + +On 2026-11-16 a manual host clean-up found causes of load and disk pressure +that `fixos auto` and `fixos cleanup` did not report: + +- `k3s.service` restarting in a loop (16 881 restarts) and flooding syslog; +- Docker container logs without rotation (6 GiB, no `log-opts.max-size`); +- large untouched directories in `~/.local/state` (17 GiB of backups from + October, a 9 GiB retained runtime backup); +- a 22 GiB JetBrains index while the IDE was not running; +- about 420 zombie processes under four container PID 1 processes and 470 + leaked identical helper processes. + +Scope: a read-only detector module `fixos/diagnostics/host_anomalies.py` and a +new command `fixos anomalies`. System-level remedies (service changes, log +truncation, signals) stay suggested commands. Removal is limited to +`stale-state` and idle `ide-index` findings that a fresh scan still reports, +inside the user's home, never through a symlink or a protected directory, and +writes a `fixos.host-anomaly-removal/v1` receipt. + +## Acceptance criteria + +- [x] AC-01: 10 unit tests pass (`tests/unit/test_host_anomalies.py`), covering each + detector, the false-positive guard for daemons sharing a name, and the + removal guards (not a current finding, symlink, outside home). +- [x] AC-02: `python -m fixos.cli anomalies` on the live host reported restart + loops not found manually (`safetytwin-agent`, `minis-ollama-tunnel`, + `twinerd-minis-novnc`) and the zombie parents. +- [x] AC-03: `ruff check` and `ruff format --check` pass for the three new files. +- [ ] AC-04: Governance check and exact-head review pass. + +## Tracking boundary + +This directory contains the minimal reviewed intent. Optional participant prose +and raw command logs are not required delivery output. diff --git a/project/ticket-136/intent.json b/project/ticket-136/intent.json new file mode 100644 index 0000000..00b0cc0 --- /dev/null +++ b/project/ticket-136/intent.json @@ -0,0 +1,99 @@ +{ + "schema": "new-project.intent/v3", + "ticket": "ticket-136", + "summary": "Host anomaly detection: restart loops, unrotated container logs, stale state stores, IDE indexes, leaked processes", + "workstream": "application", + "classification": { + "kind": "FEATURE", + "priority": "P1", + "origin": "requested" + }, + "allowedPaths": [ + "project/ticket-136/**", + "TODO.md", + "project/TICKETS.md", + "fixos/diagnostics/host_anomalies.py", + "fixos/cli/anomalies_cmd.py", + "fixos/cli/main.py", + "tests/unit/test_host_anomalies.py" + ], + "forbiddenPaths": [ + "project/ticket-*/user-*.md" + ], + "stacks": [], + "dependsOn": [], + "conflictsWith": [], + "integrationTicket": null, + "delivery": { + "acceptedBaseSha": "a33cac68d28d37edc78250ad00cb024da3788e47", + "targetBranch": "main", + "outcome": "fixos anomalies reports restart-looping services, unrotated container logs, large stale state directories, regenerable IDE indexes and leaked processes.", + "nonGoals": [ + "No automatic service changes, log truncation or process signals; those stay suggested commands.", + "No removal outside the user's state roots or of paths a current scan does not mark removable." + ], + "complexity": "M", + "estimatedMinutes": 45, + "budgets": { + "maxImplementationFiles": 4, + "maxAffectedComponents": 2, + "maxPublicInterfaceChanges": 1, + "maxRuntimeDependencies": 0 + }, + "architecture": { + "status": "accepted", + "decision": "Add a stdlib-only read-only detector module with injectable command runner and /proc root, exposed as a new lazy CLI command. Removal is limited to stale-state and idle IDE-index findings revalidated against a fresh scan, with a receipt.", + "components": [ + { + "name": "diagnostics", + "paths": [ + "fixos/diagnostics/host_anomalies.py", + "tests/unit/test_host_anomalies.py" + ] + }, + { + "name": "cli", + "paths": [ + "fixos/cli/anomalies_cmd.py", + "fixos/cli/main.py" + ] + } + ], + "responsibilityChanges": false, + "interfaceChanges": [ + "fixos anomalies [--json] [--min-size-gb] [--min-age-days] [--remove PATH] [--remove-safe] [--dry-run]" + ], + "dataChanges": [], + "ui": { + "impact": "none", + "states": [], + "evidence": [] + }, + "rollback": "Revert the commit; the command is additive and no existing behaviour changes." + }, + "runtimeDependencies": [], + "validation": [ + { + "criterion": "AC-01", + "commands": [ + "python -m pytest tests/unit/test_host_anomalies.py -q" + ], + "evidence": "10 unit tests" + }, + { + "criterion": "AC-02", + "commands": [ + "python -m fixos.cli anomalies --json" + ], + "evidence": "report on the live host" + }, + { + "criterion": "AC-03", + "commands": [ + "./project/governance-check.sh" + ], + "evidence": "governance-check output" + } + ] + } +} diff --git a/tests/unit/test_host_anomalies.py b/tests/unit/test_host_anomalies.py new file mode 100644 index 0000000..3c69327 --- /dev/null +++ b/tests/unit/test_host_anomalies.py @@ -0,0 +1,254 @@ +"""Tests for host anomaly detection.""" + +from __future__ import annotations + +import json +import os +import time +from pathlib import Path + +from click.testing import CliRunner + +from fixos.diagnostics import host_anomalies as ha + +OLD = time.time() - 30 * 86400 +MIB = 1024 * 1024 + + +def _write(path: Path, size: int = 10, when: float | None = None) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"x" * size) + if when is not None: + os.utime(path, (when, when)) + return path + + +def _du_runner(sizes: dict[Path, int]): + def run(command): + if command[0] != "du": + return "" + root = Path(command[-1]) + return "".join( + f"{size}\t{path}\n" + for path, size in sizes.items() + if path == root or root in path.parents + ) + + return run + + +def test_restart_loop_reports_looping_unit_and_ignores_stable_ones(): + system = ( + "Id=k3s.service\nNRestarts=16881\nActiveState=activating\n" + "SubState=auto-restart\n\n" + "Id=cron.service\nNRestarts=0\nActiveState=active\nSubState=running\n\n" + "Id=journald.service\nNRestarts=60\nActiveState=active\nSubState=running\n" + ) + user = "Id=tunnel.service\nNRestarts=704\nActiveState=active\nSubState=running\n" + + def run(command): + return user if "--user" in command else system + + found = ha.scan_restart_loops(run=run) + + assert [f["subject"] for f in found] == ["k3s.service", "tunnel.service"] + assert found[0]["severity"] == "high" + assert "systemctl disable --now k3s.service" in found[0]["suggested_commands"] + assert found[1]["severity"] == "medium" + assert ( + "systemctl --user disable --now tunnel.service" + in (found[1]["suggested_commands"]) + ) + + +def test_container_logs_flag_missing_rotation_and_large_logs(tmp_path): + daemon = tmp_path / "daemon.json" + daemon.write_text(json.dumps({"default-address-pools": []})) + root = tmp_path / "containers" + big = _write(root / ("a" * 64) / ("a" * 64 + "-json.log"), 3 * MIB) + _write(root / ("b" * 64) / ("b" * 64 + "-json.log"), 10) + + found = ha.scan_container_logs( + containers_root=root, daemon_json=daemon, min_bytes=MIB + ) + + assert found[0]["subject"] == str(daemon) + assert found[0]["evidence"]["rotation_configured"] is False + assert [f["subject"] for f in found[1:]] == [str(big)] + + +def test_container_logs_accept_configured_rotation(tmp_path): + daemon = tmp_path / "daemon.json" + daemon.write_text(json.dumps({"log-opts": {"max-size": "50m"}})) + root = tmp_path / "containers" + _write(root / ("a" * 64) / ("a" * 64 + "-json.log"), 3 * MIB) + + found = ha.scan_container_logs( + containers_root=root, daemon_json=daemon, min_bytes=MIB + ) + + assert [f["severity"] for f in found] == ["low"] + + +def test_stale_state_reports_topmost_stale_directory_only(tmp_path): + state = tmp_path / ".local/state" + backups = state / "codex/run-1/backups" + _write(backups / "a.tar", when=OLD) + _write(backups / "deep/b.tar", when=OLD) + _write(state / "codex/run-1/notes.txt") # recent sibling keeps run-1 live + _write(state / "subactor/diagnostics/checkpoint.sqlite3") # live store + sizes = { + state: 40, + state / "codex": 20, + state / "codex/run-1": 20, + backups: 18, + backups / "deep": 9, + state / "subactor": 20, + state / "subactor/diagnostics": 20, + } + + found = ha.scan_stale_state( + tmp_path, run=_du_runner(sizes), min_bytes=10, min_age_days=14 + ) + + assert [f["subject"] for f in found] == [str(backups)] + assert found[0]["severity"] == "review" and found[0]["removable"] is True + + +def test_stale_state_never_offers_protected_directories(tmp_path): + secret = tmp_path / ".local/share/keys/.ssh" + _write(secret / "id", when=OLD) + sizes = {tmp_path / ".local/share/keys": 50, secret: 50} + + found = ha.scan_stale_state( + tmp_path, run=_du_runner(sizes), min_bytes=10, min_age_days=14 + ) + + assert str(secret) not in [f["subject"] for f in found] + + +def test_ide_index_is_safe_only_when_ide_is_not_running(tmp_path): + index = tmp_path / ".cache/JetBrains/PyCharm2026.3/index" + _write(index / "stubs.bin") + proc = tmp_path / "proc" + _write(proc / "100/cmdline", 1).write_bytes(b"/usr/bin/bash\0-c\0true") + run = _du_runner({index: 50}) + + idle = ha.scan_ide_indexes(tmp_path, run=run, proc=proc, min_bytes=10) + _write(proc / "200/cmdline", 1).write_bytes( + b"java\0-Didea.paths.selector=PyCharm2026.3" + ) + busy = ha.scan_ide_indexes(tmp_path, run=run, proc=proc, min_bytes=10) + + assert idle[0]["severity"] == "safe" and idle[0]["removable"] is True + assert busy[0]["severity"] == "protected" and busy[0]["removable"] is False + + +def _stat(pid: int, comm: str, state: str, parent: int, start_ticks: int) -> str: + rest = [state, str(parent)] + ["0"] * 17 + [str(start_ticks)] + ["0"] * 10 + return f"{pid} ({comm}) " + " ".join(rest) + + +def test_processes_report_zombies_per_parent_and_idle_groups(tmp_path): + proc = tmp_path / "proc" + for pid in range(1000, 1012): + _write(proc / str(pid) / "stat").write_text(_stat(pid, "python", "Z", 15, 0)) + for pid in range(2000, 2060): + _write(proc / str(pid) / "stat").write_text( + _stat(pid, "mime helper", "S", 16, 0) + ) + (proc / str(pid) / "cmdline").write_bytes(b"mime-helper\0--launch\0URL") + for pid in range(4000, 4090): # daemons sharing a name, one per port + _write(proc / str(pid) / "stat").write_text(_stat(pid, "proxy", "S", 17, 0)) + (proc / str(pid) / "cmdline").write_bytes(b"proxy\0-port\0%d" % pid) + _write(proc / "3000" / "stat").write_text(_stat(3000, "fresh", "S", 1, 99_000_000)) + + found = ha.scan_processes(proc=proc, uptime=1_000_000.0, ticks=100) + + by_category = {f["category"]: f for f in found} + assert by_category["zombie-processes"]["evidence"] == { + "parent_pid": 15, + "count": 12, + } + assert by_category["idle-process-group"]["subject"] == "mime helper" + assert by_category["idle-process-group"]["evidence"]["count"] == 60 + assert ( + "pgrep -a -x mime helper" + in (by_category["idle-process-group"]["suggested_commands"][0]) + ) + assert len(found) == 2 + + +def test_remove_paths_deletes_only_current_removable_findings(tmp_path): + home = tmp_path + wanted = home / ".local/state/tool/backups" + other = home / ".local/state/tool/live" + _write(wanted / "a.bin") + _write(other / "b.bin") + findings = [ + ha._finding("stale-state", "review", wanted, "x", size_bytes=7), + ha._finding("ide-index", "protected", other, "x", size_bytes=3), + ] + + receipt = ha.remove_paths( + [wanted, other, Path("/etc")], + findings, + home=home, + receipt_dir=home / "receipts", + ) + + assert not wanted.exists() and other.exists() + assert [item["path"] for item in receipt["removed"]] == [str(wanted)] + assert {item["reason"] for item in receipt["refused"]} == { + "not_a_current_removable_finding" + } + assert receipt["freed_bytes"] == 7 + assert json.loads(Path(receipt["receipt_path"]).read_text())["schema"] == ( + ha.RECEIPT_SCHEMA + ) + + +def test_remove_paths_refuses_symlink_and_paths_outside_home(tmp_path): + home = tmp_path / "home" + outside = tmp_path / "outside" + _write(outside / "data.bin") + link = home / ".local/state/link" + link.parent.mkdir(parents=True) + link.symlink_to(outside) + findings = [ + ha._finding("stale-state", "review", link, "x", size_bytes=1), + ha._finding("stale-state", "review", outside, "x", size_bytes=1), + ] + + receipt = ha.remove_paths([link, outside], findings, home=home) + + assert outside.exists() and (outside / "data.bin").exists() + assert [item["reason"] for item in receipt["refused"]] == [ + "not_a_plain_directory", + "outside_home", + ] + + +def test_cli_reports_and_dry_run_removes_nothing(tmp_path, monkeypatch): + from fixos.cli import anomalies_cmd + + target = tmp_path / ".cache/JetBrains/PyCharm2026.3/index" + _write(target / "stubs.bin") + finding = ha._finding("ide-index", "safe", target, "Indeks", size_bytes=5) + monkeypatch.setattr( + anomalies_cmd, + "scan_all", + lambda **_: { + "schema": ha.SCHEMA, + "findings": [finding], + "reclaimable_bytes": 5, + }, + ) + + result = CliRunner().invoke( + anomalies_cmd.anomalies, ["--remove-safe", "--dry-run", "--json"] + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output)["would_remove"] == [str(target)] + assert target.exists()