diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 15dffac..a275b7d 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -1,5 +1,68 @@ # Open audit items +## 2026-11-05 — Dash auto-config Slice 6 CLI: cleanup / audit (P5) + +Plan: `../_internal/DASH_AUTOCONFIG_PLAN_2026-11-03.md` (section 9). Branch +`slice6-cleanup`. `teploy server audit` (read-only findings + remedy argv), +`route remove` (exact-block CAS), `releases prune --keep N`, +`server prune --build-cache --dangling`, keep_versions D3 (unset → 5 after a +recorded 7-day notice, `init` writes 3, 0 keeps everything), contracts rev 19. + +Deliberately report-only (design principle 4: hand-written config and other +people's containers are never edited by teploy): +- Hand-written Caddy blocks: dead upstream, non-local address. Evidence only. +- Large container writable layers (managed ≥ 5 GB) and large unmanaged + containers (≥ 1 GB): a layer's data belongs to whoever wrote it; the + evidence names the move-to-volume fix. +- Dangling volumes: may hold the only copy; there is never a remedy argv. +- Unmanaged containers (portainer et al.): not teploy's to remove. +- A managed block that still serves files (a `root` that exists), carries the + autodeploy webhook, or whose app is live: `route remove` refuses; the audit + reports the finding with the refusal as its evidence and a null remedy. +- An upstream that is not this app's (a `caddy_extra` proxy to another app's + dead container): this app's redeploy cannot fix it, so no redeploy remedy. + +Deferred surface: +- **Backup output sizes in the audit** (plan lesson: "backup jobs whose + outputs are implausibly small"): covered at write time by the engine's + minimum-size refusal (Slice 2, `internal/backup/engine.go`) and + `backup status`'s per-target last success/failure; the audit does not + re-walk buckets, which would need target credentials it must not use. +- **`server audit` on a server with no caddyfile at all** reports no route + findings by construction; a Caddyfile with no Caddy container is flagged + (`caddyfile_without_caddy`, compute-2's case). +- **Disk attribution beyond docker + /deployments**: `du` of other trees + (logs, /var) is not attempted; the pressure finding attributes what teploy + can name and links the reclaimable findings. +- **Audit does not pin the notice file's mtime against clock skew**: the + notice is a RFC 3339 server-side timestamp written by the deploy that + starts it, same trust domain as state.json. + +Decisions worth reviewing: +- `route remove` protects running-container versions of an app's name even + without state (`ownedContainers`, unlabelled `-*` names included): + an older deploy's container keeps the name alive, so the route is not + proven dead until it is gone. +- `releases prune --keep 0` (and the audit never emits it) prunes everything + the protections do not name — deliberately the opposite reading of + `keep_versions: 0`, documented in the command's help, because "retain none + beyond the protected set" is a coherent manual operation. +- Automatic pruning under the default keep_versions runs only inside a + deploy of that app (never from the audit, never fleet-wide), so the notice + is always written by an operation that already holds the app lock. +- `retentionProtected` includes any version with a running container, beyond + the plan's pins/current/previous: `docker rm -f` of a running stranger is + not something a cleanup command should do even inside the keep window. + +Verified: unit/hermetic only (mock SSH executors, the same discipline as +`domains check`); every audit finding kind, all `route remove` refusal +paths, the CAS refusal on a concurrent rewrite, prune selections (pins/ +current/previous/running survive, `--keep 0`, unreadable pins refuse), +the notice-period state machine, and the read-only pin (no mutating verb, +docker subcommand or upload in any command `server audit` issues). Not yet +exercised live: the four fleet servers' first real audit run and the Dash +cleanup page (Dash side, next slice). + ## 2026-11-05 — Dash auto-config Slice 4 CLI: fleet backup engine (G1, G3, G4, G5) Plan: `../_internal/DASH_AUTOCONFIG_PLAN_2026-11-03.md` (sections 2.1-2.3, 5). Branch diff --git a/CHANGELOG.md b/CHANGELOG.md index a7205f0..f335b8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,46 @@ All notable changes to teploy are documented here. Format follows [Keep a Change ## [Unreleased] +### Added + +- `teploy server audit --json`: what is left over on a server, each + finding with its evidence and — only where a CLI command fixes it safely — + the remedy argv. Stale and empty managed routes, managed routes with dead + upstreams, app state with no containers, versions beyond `keep_versions`, + reclaimable build cache and dangling images, large container writable + layers, large containers teploy does not manage, dangling volumes, a + Caddyfile with no Caddy container, scheduled jobs with no cron daemon, and + disk pressure with attribution. Read-only: every command the audit runs on + the server only reads (a test pins that). A source that could not be read + is an `unknown` check and yields no finding — never a clean bill. +- `teploy route remove --app X`: removes a managed Caddy block proven dead + (no app state, no containers carrying the name, no upstream that still + resolves, no files it still serves, nothing unreadable), under the app's + lock and an exact-block compare-and-swap, so a concurrent rewrite is + refused rather than undone. Recorded as a `route.remove` event. +- `teploy releases prune --app X --keep N`: removes old versions' containers + and images beyond `--keep`. Pinned, current, previous and running versions + are never removed whatever `--keep` says; an unreadable pin set, state or + container list refuses the prune. Recorded as a `releases.prune` event. +- `teploy server prune --host S --build-cache --dangling`: reclaims docker's + unused build cache and dangling images — nothing else (no tagged image, no + container, no volume). Recorded as a `server.prune` event under `_fleet`. +- Contracts: `server-audit-envelope` (schema + generated fixture, corpus rev + 19); version-handshake advertises `server-audit-v1`. + +### Changed + +- `keep_versions` is now optional (decision D3): unset means 5, pruned by + deploys only after a 7-day notice recorded on the server (the deploy + output and `server audit` warn first); `teploy init` writes 3; `0` keeps + everything. Applied manifests record unset and N as before, so existing + digests hold; an explicit 0 records `keep_all_versions`. +- `teploy remove` on an app whose directory is already gone (an earlier + removal finished) succeeds instead of failing its state deletion. +- `teploy doctor` disk remediations point at `teploy server audit`, which + attributes the space; one named a `releases prune` command that did not + exist. + ## [0.1.52] - 2026-11-05 ### Added diff --git a/README.md b/README.md index 905173c..4ce7a77 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,9 @@ drain_seconds: 10 # request drain: keep the old container serving in # the drain policy — size it to your longest normal request. memory: 2g # cgroup RAM cap (docker units: 512m, 2g). Unset = unlimited. cpu: "1.5" # CPU cap in cores. Unset = unlimited. -keep_versions: 3 # auto-prune older versions after deploy (0 = keep all, default) +keep_versions: 3 # versions retained after a deploy. Unset = 5, pruned only after a + # 7-day notice (teploy init writes 3); 0 keeps everything. + # The current, previous and pinned versions are never pruned. backup_before_hooks: true # back up running databases before a deploy whose pre_deploy hook will run (default off) # `teploy pin ` protects a version from this prune @@ -354,6 +356,7 @@ teploy plan # preview what a deploy would change ( teploy plan --out plan.json # write a bound plan record (config digest + target identity + generation) teploy apply plan.json # execute a reviewed plan; refuses naming what drifted since it was made teploy scale # multi-server deploy + LB update +teploy server audit # read-only: what is left over and what fixes it teploy version / update # version info and self-update ``` @@ -595,6 +598,40 @@ serves. Read-only, run from your machine over SSH, nothing new on the server: `ingress: external` and an unfronted `ingress: host` domain are labels: they carry a `skip` reason. A server that cannot be reached is reported as such. +### Fleet cleanup & audit + +```bash +teploy server audit # what is left over, with evidence and the fix +teploy server audit --json # contracts/schema/server-audit-envelope.schema.json +teploy route remove --app X --host S # remove a managed Caddy block proven dead (CAS) +teploy releases prune --app X --host S --keep N # remove versions beyond N (pins/current/previous survive) +teploy server prune --host S --build-cache --dangling # reclaim unused build cache and dangling images +``` + +`server audit` changes nothing: every command it runs on the server only +reads (a test pins that). Each finding says what it is, the evidence for it, +and — only where a CLI command can fix it safely — the remedy: + +| Finding | Remedy | +|---|---| +| stale/empty managed route | `teploy route remove --app X` | +| managed route, dead upstream | `teploy redeploy` (confirm) | +| app state, no containers | `teploy remove --app X -y` (keeps volumes; confirm) | +| versions beyond `keep_versions` | `teploy releases prune --keep N` (confirm) | +| build cache / dangling images | `teploy server prune --build-cache --dangling` | +| hand-written blocks, big writable layers, dangling volumes, unmanaged containers, disk pressure, a Caddyfile with no Caddy, scheduled jobs with no cron daemon | report only | + +A source that could not be read shows as `unknown` in `checks[]` and the +findings depending on it are missing — a server is never reported clean +because it could not be inspected. With `--json` the exit code is 0 whenever +the envelope was produced; read `state` (`ok` / `partial` / `unreachable`). + +`route remove` only takes a block proven dead — no app state, no containers +carrying the name, no upstream that still resolves (asked inside Caddy's +network), no files it still serves, nothing unreadable — and commits under +an exact-block compare-and-swap, so a deploy that rewrites the block in +between is never undone. + ### Accessories ``` teploy accessory list # list running accessories diff --git a/contracts/MANIFEST.md b/contracts/MANIFEST.md index 5f6720b..f520cbc 100644 --- a/contracts/MANIFEST.md +++ b/contracts/MANIFEST.md @@ -10,6 +10,7 @@ Neutron/Nucleus dependency and a public mirror. | Corpus rev | Emitting CLI | Machine Interface | Notes | |---|---|---|---| +| 19 | main (Dash auto-config Slice 6: cleanup / audit) | 2 | Additive. New artifact server-audit-envelope (`teploy server audit --json`, read-only: findings with stable `id` = kind:subject, severity, evidence, and remedy {argv (null when nothing the CLI can do safely), destructive, confirm}, per-source `checks` where unknown means findings depending on it are missing — never clean — and disk attribution), fixture GENERATED from the real `auditServer` over a mock executor (one of each finding incl. null-remedy, one unknown check). New remedy commands `route remove --app X` (exact-block CAS), `releases prune --keep N` (pins/current/previous/running never pruned) and `server prune --build-cache --dangling`; events `route.remove`, `releases.prune`, `server.prune` (the last under `_fleet`). keep_versions becomes optional: unset → 5 pruned only after a 7-day notice recorded server-side (`deploy/retention.go`), `init` writes 3, explicit 0 keeps everything and records `keep_all_versions` in the applied manifest (unset and N record what they always did, so existing digests hold). version-handshake advertises `server-audit-v1` (regenerated mi2.json). No MI bump. | | 18 | main (Dash auto-config Slice 4: fleet backup engine) | 2 | Additive. New artifacts backup-status-envelope (`backup status --json`: every DB-like container and app volume with coverage state, last success/failure, newest object, last verify, plus target and policy summaries, the aws CLI's `aws_cli_problem` and the target's `incoming_lifecycle`; no secrets), backup-run-envelope (`backup run --all --json`) and backup-policy-envelope (`server backup-policy show/set --json`), fixtures GENERATED from the real DTOs. backup-list rows gain optional `encrypted`, `previous_prefix`, and `database` only with `--databases` or `--adopted` (a caller restoring by id alone would replace the primary); backup-create gains `encrypted` and `objects`; backup-restore gains `pre_backup_id`; backup-target gains `previous_prefixes`. Event kinds `backup.all`, `backup_policy.set` and `backup_target.owners_clear` (a marker removal or refusal) under the pseudo-app `_fleet`; adopted containers log under `_adopted`. version-handshake advertises `backup-fleet-v1` (regenerated mi2.json). No MI bump. | | 17 | main (dash auto-config slice 5: domain health) | 2 | Additive. New artifact domains-check-envelope (`teploy domains check [--server] [--app] [--log-window] --json`: per (server, app, host) DNS via the fixed public resolvers, Caddy certificate failures from its logs — `failures` is null, never 0, on short or unavailable logs — and the route with upstream resolution; per-server state so an unreachable server is explicit), fixture GENERATED from the real `sweepDomains` over a mock executor. version-handshake advertises `domains-check-v1` (regenerated mi2.json). release-record's `caddy` gains optional `tls_http_only` (absent on older records; every caddy release since records true or false). No MI bump. | | 16 | main (server-authority spec P3 / plan D4, git sources) | 2 | Additive. New artifacts source-show-envelope (`config source show --json`: repository/branch/poll, public credential facts — type, deploy-key public half + fingerprint, token username, pinned host-key fingerprints — and the per-ref admission state), source-key-envelope (`config source key --json` for ssh/token and `host-key --json`) and trigger-poll-envelope (`trigger poll --json`: one result object per app — ref, head, admitted, reason vocabulary incl. force-push/retry-limit/not-due, deployed_release), all fixtures GENERATED from the real DTOs. version-handshake advertises `git-source-v1` (git: code source, per-app credentials outside /deployments, build_secrets) and `trigger-poll-v1` (the one admitter + reset/retry/install-timer) (regenerated mi2.json). No MI bump. | @@ -60,6 +61,7 @@ Neutron/Nucleus dependency and a public mirror. | backup-run-envelope | yes (rev 18) | valid x2 (partial, skipped; real backupRunEnvelope) | teploy-cli | | backup-policy-envelope | yes (rev 18) | valid x2 (configured, unconfigured; real backupPolicyDTO) | teploy-cli | | domains-check-envelope | yes (rev 17) | valid (one server with each finding, one unreachable; real `sweepDomains`) | teploy-cli | +| server-audit-envelope | yes (rev 19) | valid (one of each finding incl. null-remedy, one unknown check; real `auditServer` over a mock executor) | teploy-cli | | releases-envelope | yes (rev 14) | valid x2 (container app, static app; real releasesDTO) | teploy-cli | | rollback-result | yes (rev 14) | valid (real rollbackResult) | teploy-cli | | redeploy-result | yes (rev 14) | valid x2 (teploy-home, folder-home; real redeployDTO) | teploy-cli | diff --git a/contracts/fixtures/server-audit-envelope/valid/findings.json b/contracts/fixtures/server-audit-envelope/valid/findings.json new file mode 100644 index 0000000..de4683b --- /dev/null +++ b/contracts/fixtures/server-audit-envelope/valid/findings.json @@ -0,0 +1,352 @@ +{ + "machine_interface": 2, + "server": "edge", + "host": "192.0.2.10", + "audited_at": "2026-11-05T12:00:00Z", + "state": "partial", + "checks": [ + { + "name": "apps", + "status": "unknown", + "detail": "could not read the state of broken" + }, + { + "name": "caddyfile", + "status": "ok" + }, + { + "name": "containers", + "status": "ok" + }, + { + "name": "container_sizes", + "status": "ok" + }, + { + "name": "upstreams", + "status": "ok" + }, + { + "name": "static_roots", + "status": "ok" + }, + { + "name": "addresses", + "status": "ok" + }, + { + "name": "docker_usage", + "status": "ok" + }, + { + "name": "dangling_images", + "status": "ok" + }, + { + "name": "dangling_volumes", + "status": "ok" + }, + { + "name": "disk", + "status": "ok" + }, + { + "name": "scheduler", + "status": "ok" + }, + { + "name": "retention", + "status": "ok" + } + ], + "disk": { + "path": "/", + "available_bytes": 800000000, + "used_percent": 96, + "attribution": [ + { + "name": "images", + "size_bytes": 20000000000 + }, + { + "name": "container_layers", + "size_bytes": 5000000000 + }, + { + "name": "volumes", + "size_bytes": 4000000000 + }, + { + "name": "build_cache", + "size_bytes": 2000000000 + }, + { + "name": "deployments", + "size_bytes": 53687091200 + } + ] + }, + "findings": [ + { + "id": "disk_pressure:/", + "kind": "disk_pressure", + "severity": "critical", + "subject": "/", + "evidence": "800.0 MB available on / (96% used); deployments 53.7 GB, images 20.0 GB, container_layers 5.0 GB, volumes 4.0 GB, build_cache 2.0 GB", + "related": [ + "dangling_volumes:docker", + "large_writable_layer:reacher-web-1f2e3d", + "reclaimable_build_artifacts:docker", + "unmanaged_container:portainer", + "versions_beyond_policy:many", + "versions_beyond_policy:unset" + ], + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "managed_route_dead_upstream:blog", + "kind": "managed_route_dead_upstream", + "severity": "critical", + "subject": "blog", + "app": "blog", + "evidence": "blog.example.com routes to blog-web-v3:3000, which does not resolve in Caddy's network", + "remedy": { + "argv": [ + "teploy", + "redeploy", + "--app", + "blog", + "--host", + "edge" + ], + "destructive": false, + "confirm": true + } + }, + { + "id": "scheduler_missing:cron", + "kind": "scheduler_missing", + "severity": "critical", + "subject": "cron", + "evidence": "jobs are scheduled but no cron daemon is running, so none of them run: run `teploy setup` against the server (it installs and enables cron), or on the server: apt-get install cron \u0026\u0026 systemctl enable --now cron", + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "large_writable_layer:reacher-web-1f2e3d", + "kind": "large_writable_layer", + "severity": "warning", + "subject": "reacher-web-1f2e3d", + "app": "reacher", + "evidence": "reacher-web-1f2e3d (running) has written 35.7 GB to its container layer, which a redeploy discards: move that data to a volume", + "size_bytes": 35700000000, + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "manual_route_dead_upstream:hand.example.com", + "kind": "manual_route_dead_upstream", + "severity": "warning", + "subject": "hand.example.com", + "evidence": "hand-written block for hand.example.com routes to gone-svc:80, which does not resolve", + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "orphan_state:orphan", + "kind": "orphan_state", + "severity": "warning", + "subject": "orphan", + "app": "orphan", + "evidence": "orphan has deploy state (release o1) but no containers; removing keeps its volumes, accessory data and credentials", + "remedy": { + "argv": [ + "teploy", + "remove", + "--app", + "orphan", + "--host", + "edge", + "-y" + ], + "destructive": true, + "confirm": true + } + }, + { + "id": "stale_route:files", + "kind": "stale_route", + "severity": "warning", + "subject": "files", + "app": "files", + "evidence": "no app files on this server, but its managed block serves files.example.com; not removable: it serves files from /deployments/files/site, which exists", + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "stale_route:ghost", + "kind": "stale_route", + "severity": "warning", + "subject": "ghost", + "app": "ghost", + "evidence": "no app ghost on this server, but its managed block serves ghost.example.com and routes to ghost-web-v1:3000, which does not resolve", + "remedy": { + "argv": [ + "teploy", + "route", + "remove", + "--app", + "ghost", + "--host", + "edge" + ], + "destructive": false, + "confirm": false + } + }, + { + "id": "unmanaged_container:portainer", + "kind": "unmanaged_container", + "severity": "warning", + "subject": "portainer", + "evidence": "portainer (running) is not managed by teploy and holds 2.0 GB in its container layer", + "size_bytes": 2000000000, + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "versions_beyond_policy:unset", + "kind": "versions_beyond_policy", + "severity": "warning", + "subject": "unset", + "app": "unset", + "evidence": "2 version(s) beyond keep_versions 5: u2, u1; keep_versions is not set, so deploys after 2026-11-08 remove them automatically; set keep_versions (0 keeps everything) or pin what you need", + "size_bytes": 20000000, + "remedy": { + "argv": [ + "teploy", + "releases", + "prune", + "--app", + "unset", + "--host", + "edge", + "--keep", + "5" + ], + "destructive": true, + "confirm": true + } + }, + { + "id": "dangling_volumes:docker", + "kind": "dangling_volumes", + "severity": "info", + "subject": "docker", + "evidence": "21 volume(s) attached to no container, 3.1 GB reclaimable: 0000000000000000000000000000000xxxxxxxdeadbeef, 0000000000000000000000000000001xxxxxxxdeadbeef, 0000000000000000000000000000002xxxxxxxdeadbeef, 0000000000000000000000000000003xxxxxxxdeadbeef, 0000000000000000000000000000004xxxxxxxdeadbeef (+16 more)", + "size_bytes": 3150000000, + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "empty_managed_region:hollow", + "kind": "empty_managed_region", + "severity": "info", + "subject": "hollow", + "app": "hollow", + "evidence": "the managed region holds no site block", + "remedy": { + "argv": [ + "teploy", + "route", + "remove", + "--app", + "hollow", + "--host", + "edge" + ], + "destructive": false, + "confirm": false + } + }, + { + "id": "manual_route_nonlocal_address:203.0.113.9", + "kind": "manual_route_nonlocal_address", + "severity": "info", + "subject": "203.0.113.9", + "evidence": "hand-written block is addressed to 203.0.113.9, which is not an address of this server", + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "reclaimable_build_artifacts:docker", + "kind": "reclaimable_build_artifacts", + "severity": "info", + "subject": "docker", + "evidence": "2.0 GB of unused build cache and 3 dangling image(s)", + "size_bytes": 2000000000, + "remedy": { + "argv": [ + "teploy", + "server", + "prune", + "--host", + "edge", + "--build-cache", + "--dangling" + ], + "destructive": false, + "confirm": false + } + }, + { + "id": "versions_beyond_policy:many", + "kind": "versions_beyond_policy", + "severity": "info", + "subject": "many", + "app": "many", + "evidence": "1 version(s) beyond keep_versions 2: v2; the next deploy removes them", + "size_bytes": 10000000, + "remedy": { + "argv": [ + "teploy", + "releases", + "prune", + "--app", + "many", + "--host", + "edge", + "--keep", + "2" + ], + "destructive": true, + "confirm": true + } + } + ] +} diff --git a/contracts/fixtures/version-handshake/valid/mi2.json b/contracts/fixtures/version-handshake/valid/mi2.json index e08815f..fe7538b 100644 --- a/contracts/fixtures/version-handshake/valid/mi2.json +++ b/contracts/fixtures/version-handshake/valid/mi2.json @@ -32,6 +32,7 @@ "repair-debt", "rollback-json-v1", "secret-stdin-v1", + "server-audit-v1", "server-rename", "server-status-machine", "server-update", diff --git a/contracts/schema/server-audit-envelope.schema.json b/contracts/schema/server-audit-envelope.schema.json new file mode 100644 index 0000000..e2356a7 --- /dev/null +++ b/contracts/schema/server-audit-envelope.schema.json @@ -0,0 +1,83 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/server-audit-envelope.schema.json", + "title": "`teploy server audit --json` envelope: what is left over on a server, with evidence and the command that fixes it where one can (rev 19). The audit itself is read-only; a source that could not be read is an unknown check, never a clean bill.", + "type": "object", + "required": ["machine_interface", "server", "host", "audited_at", "state", "checks", "disk", "findings"], + "properties": { + "machine_interface": {"type": "integer", "minimum": 1}, + "server": {"type": "string", "minLength": 1}, + "host": {"type": "string"}, + "audited_at": {"type": "string", "format": "date-time"}, + "state": {"enum": ["ok", "partial", "unreachable"]}, + "error": {"type": "string"}, + "checks": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "status"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "status": {"enum": ["ok", "unknown"]}, + "detail": {"type": "string"} + } + } + }, + "disk": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", + "required": ["path", "available_bytes", "used_percent", "attribution"], + "properties": { + "path": {"type": "string"}, + "available_bytes": {"type": "integer", "minimum": 0}, + "used_percent": {"type": "integer", "minimum": 0, "maximum": 100}, + "attribution": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "size_bytes"], + "properties": { + "name": {"type": "string"}, + "size_bytes": {"type": "integer", "minimum": 0} + } + } + } + } + } + ] + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "kind", "severity", "subject", "evidence", "remedy"], + "properties": { + "id": {"type": "string", "minLength": 1, "description": "kind:subject — stable across runs so a finding can be dismissed"}, + "kind": {"enum": ["stale_route", "empty_managed_region", "managed_route_dead_upstream", "orphan_state", "manual_route_dead_upstream", "manual_route_nonlocal_address", "versions_beyond_policy", "reclaimable_build_artifacts", "large_writable_layer", "unmanaged_container", "dangling_volumes", "caddyfile_without_caddy", "scheduler_missing", "disk_pressure"]}, + "severity": {"enum": ["critical", "warning", "info"]}, + "subject": {"type": "string", "minLength": 1}, + "app": {"type": "string", "description": "set when the finding belongs to one app: the name a destructive confirmation asks for"}, + "evidence": {"type": "string", "minLength": 1}, + "size_bytes": {"type": "integer", "minimum": 0}, + "related": {"type": "array", "items": {"type": "string"}, "description": "finding ids whose remedies would free space on a disk_pressure"}, + "remedy": { + "type": "object", + "required": ["destructive", "confirm"], + "properties": { + "argv": { + "oneOf": [ + {"type": "null", "description": "nothing the CLI can do safely: report only"}, + {"type": "array", "items": {"type": "string"}, "minItems": 1, "description": "the command that fixes it; argv[0] is teploy"} + ] + }, + "destructive": {"type": "boolean"}, + "confirm": {"type": "boolean", "description": "ask a person before running (a destructive remedy always confirms)"} + } + } + } + } + } + } +} diff --git a/internal/caddy/inspect.go b/internal/caddy/inspect.go index 8e5cab0..f16189d 100644 --- a/internal/caddy/inspect.go +++ b/internal/caddy/inspect.go @@ -98,6 +98,62 @@ func (e *unterminatedRegionError) Error() string { return "the managed block for " + e.app + " has no end marker" } +// ManagedRegion is one app's marker-delimited region of a Caddyfile. +type ManagedRegion struct { + App string + // Text is the region including its markers, as ManagedRegionHash expects. + Text string + // Sites is empty for a region that holds no site block. + Sites []RouteSite +} + +// ManagedRegions lists a Caddyfile's managed regions in file order. An +// unparseable file is an error, as for InspectRoutes. +func ManagedRegions(content string) ([]ManagedRegion, error) { + sites, err := InspectRoutes(content) + if err != nil { + return nil, err + } + var regions []ManagedRegion + seen := map[string]bool{} + for _, line := range strings.Split(content, "\n") { + app, ok := beginMarkerApp(line) + if !ok || seen[app] { + continue + } + seen[app] = true + region := ManagedRegion{App: app, Text: extractManagedRegion(content, app)} + for _, site := range sites { + if site.Owner == app { + region.Sites = append(region.Sites, site) + } + } + regions = append(regions, region) + } + return regions, nil +} + +// RegionHasWebhook reports whether a region carries the autodeploy webhook route. +func RegionHasWebhook(region string) bool { + return strings.Contains(region, "# TEPLOY WEBHOOK BEGIN") +} + +// RegionRoots lists the paths a region's root directives serve files from. +func RegionRoots(region string) []string { + var roots []string + for _, line := range strings.Split(region, "\n") { + f := strings.Fields(line) + if len(f) < 2 || f[0] != "root" { + continue + } + path := f[len(f)-1] + if !slices.Contains(roots, path) { + roots = append(roots, path) + } + } + return roots +} + // RegionHosts lists the hosts a managed region serves. func RegionHosts(region string) []string { body := make([]string, 0) diff --git a/internal/cli/contracts_golden_test.go b/internal/cli/contracts_golden_test.go index a6f8cb2..65b7a65 100644 --- a/internal/cli/contracts_golden_test.go +++ b/internal/cli/contracts_golden_test.go @@ -614,6 +614,16 @@ func TestContractsGitSourceGolden(t *testing.T) { // TestContractsBackupFleetGolden pins the fleet engine's envelopes (rev 18, // backup-fleet-v1). Dash's backups page builds on backup-status-envelope. +// TestContractsServerAuditGolden pins `teploy server audit --json` +// (rev 19, server-audit-v1): one server carrying one of each finding class — +// including report-only findings whose remedy argv is null — plus one unknown +// check (an app whose state could not be read), so a consumer can never +// mistake unreadable for clean. Generated from the real `auditServer` over a +// mock executor. +func TestContractsServerAuditGolden(t *testing.T) { + writeFixture(t, "server-audit-envelope/valid/findings.json", auditKitchenSink().audit(t)) +} + func TestContractsBackupFleetGolden(t *testing.T) { at := time.Date(2026, 11, 4, 3, 17, 4, 0, time.UTC) failedAt := at.Add(-time.Hour) diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index 7cb6acb..c4bd413 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -451,12 +451,12 @@ func doctorDiskCheck(ctx context.Context, exec ssh.Executor) doctorCheck { case avail < doctorDiskFailAvailable: return doctorCheck{ Name: "disk", Result: doctorFail, Detail: detail, - Remediation: "free disk space on / (docker system prune, teploy releases/preview prune, old backups) — deploys fail mid-flight below 2 GiB", + Remediation: "free disk space on / — deploys fail mid-flight below 2 GiB. `teploy server audit ` names what takes the space and what removes it", } case avail < doctorDiskWarnAvailable || pct >= doctorDiskWarnUsedPct: return doctorCheck{ Name: "disk", Result: doctorWarn, Detail: detail, - Remediation: "free disk space on / before the next deploy (docker system prune, teploy releases prune)", + Remediation: "free disk space on / before the next deploy. `teploy server audit ` names what takes the space and what removes it", } } return doctorCheck{Name: "disk", Result: doctorOK, Detail: detail} diff --git a/internal/cli/init.go b/internal/cli/init.go index 89c643d..a1f27ea 100644 --- a/internal/cli/init.go +++ b/internal/cli/init.go @@ -43,6 +43,11 @@ func newInitCmd(flags *Flags) *cobra.Command { return cmd } +// initKeepVersions is the keep_versions a generated config starts with +// (decision D3): a deliberate, small rollback window rather than the +// unset default. +const initKeepVersions = 3 + // initFlow prompts for the minimal config and writes it to dir. Shared by // `teploy init` and the zero-config first run inside `teploy deploy`. // Returns the path of the file written. @@ -109,6 +114,12 @@ func initFlow(reader *bufio.Reader, dir string, useTOML, force bool) (string, er } cfg.Server = prompt(reader, "Server (name from servers.yml or IP)", defaultServer) + // New apps start with a small rollback window rather than the default. + if cfg.KeepVersions == nil && !cfg.IsStatic() { + keep := initKeepVersions + cfg.KeepVersions = &keep + } + // Write config. var out []byte var err error diff --git a/internal/cli/init_test.go b/internal/cli/init_test.go index 2609968..5397acd 100644 --- a/internal/cli/init_test.go +++ b/internal/cli/init_test.go @@ -40,6 +40,22 @@ func TestInitFlowWritesValidConfigWithDomain(t *testing.T) { } } +// D3: a generated container config starts with keep_versions 3 — a small +// rollback window rather than the unset default. +func TestInitFlowWritesKeepVersionsThree(t *testing.T) { + dir := t.TempDir() + if _, err := initFlow(scripted("myapp", "myapp.com", "1.2.3.4"), dir, false, false); err != nil { + t.Fatalf("initFlow: %v", err) + } + cfg, err := config.LoadApp(dir) + if err != nil { + t.Fatalf("generated config does not load: %v", err) + } + if cfg.KeepVersions == nil || *cfg.KeepVersions != initKeepVersions { + t.Errorf("keep_versions = %v, want %d", cfg.KeepVersions, initKeepVersions) + } +} + func TestInitFlowEmptyDomainBecomesHostIngress(t *testing.T) { dir := t.TempDir() diff --git a/internal/cli/machineinterface.go b/internal/cli/machineinterface.go index f2f0ff2..3aa1576 100644 --- a/internal/cli/machineinterface.go +++ b/internal/cli/machineinterface.go @@ -193,6 +193,12 @@ const ( // failures and the route per (server, app, host); read-only, runs from the // caller over SSH with nothing new on the server. CapDomainsCheckV1 = "domains-check-v1" + // CapServerAuditV1: `teploy server audit --json` (read-only + // findings with evidence and a remedy argv or null) and its remedies + // `route remove --app X`, `releases prune --keep N` and + // `server prune --build-cache --dangling`. keep_versions unset now means 5 + // after a notice period. server-audit-envelope pins the JSON. + CapServerAuditV1 = "server-audit-v1" ) // MachineCapabilities returns every capability token this build @@ -237,7 +243,8 @@ func MachineCapabilities() []string { CapRollbackJsonV1, CapRedeployV1, CapEnvConfigGuardV1, - CapDomainsCheckV1} + CapDomainsCheckV1, + CapServerAuditV1} sort.Strings(tokens) return tokens } diff --git a/internal/cli/machineinterface_test.go b/internal/cli/machineinterface_test.go index 935c8cb..3bba1f8 100644 --- a/internal/cli/machineinterface_test.go +++ b/internal/cli/machineinterface_test.go @@ -106,7 +106,7 @@ func TestCapabilityTokenRegistry(t *testing.T) { "trigger-convergence-v1", } want = append(want, "actor-flag-v1", "events-v1", "config-home-v1", "undeployed-list-v1", "lock-wait-v1", "config-merge-v1", "config-unprivileged-v1", "deploy-home-v1", "backup-target-v1", "backup-json-v1", "backup-fleet-v1") - want = append(want, "secret-stdin-v1", "releases-json-v1", "rollback-json-v1", "redeploy-v1", "env-config-guard-v1", "git-source-v1", "trigger-poll-v1") + want = append(want, "secret-stdin-v1", "releases-json-v1", "rollback-json-v1", "redeploy-v1", "env-config-guard-v1", "git-source-v1", "trigger-poll-v1", "server-audit-v1") sort.Strings(want) got := MachineCapabilities() if len(got) != len(want) { diff --git a/internal/cli/releases.go b/internal/cli/releases.go index 30563e9..d5de7f1 100644 --- a/internal/cli/releases.go +++ b/internal/cli/releases.go @@ -50,6 +50,7 @@ for teploy.yml-in-a-folder invocations without --app).`, }, } cmd.Flags().StringVar(&appName, "app", "", "app name — read state from the server instead of teploy.yml (requires --host)") + cmd.AddCommand(newReleasesPruneCmd(flags)) return cmd } diff --git a/internal/cli/remove.go b/internal/cli/remove.go index 1b508d3..8ecc99f 100644 --- a/internal/cli/remove.go +++ b/internal/cli/remove.go @@ -378,10 +378,13 @@ func executeRemove(ctx context.Context, exec ssh.Executor, app string, domains [ for _, keep := range keeps { keepArgs += " ! -name " + ssh.ShellQuote(keep) } + // No directory (lk nil) means an earlier removal already finished. delCmd := fmt.Sprintf("find %s -mindepth 1 -maxdepth 1%s -exec rm -rf {} +", ssh.ShellQuote(appDir), keepArgs) - if _, err := exec.Run(ctx, delCmd); err != nil { - return nil, fmt.Errorf("removing deploy state under %s: %w", appDir, err) + if lk != nil { + if _, err := exec.Run(ctx, delCmd); err != nil { + return nil, fmt.Errorf("removing deploy state under %s: %w", appDir, err) + } } if left, err := exec.Run(ctx, "ls "+ssh.ShellQuote(appDir)+" 2>/dev/null"); err == nil { for _, d := range strings.Fields(left) { diff --git a/internal/cli/remove_test.go b/internal/cli/remove_test.go index c729fbe..644dfc7 100644 --- a/internal/cli/remove_test.go +++ b/internal/cli/remove_test.go @@ -110,6 +110,8 @@ func TestExecuteRemovePurgeFallsBackToContainerForForeignOwnedData(t *testing.T) func TestExecuteRemoveDefaultPreservesData(t *testing.T) { cmds := append([]ssh.MockCommand{ removeTestContainers(), + {Match: "[ -d '/deployments/scratch' ]", Output: ""}, + {Match: "mkdir /deployments/scratch/.lock", Output: ""}, {Match: "docker stop -t 10 'scratch-web-aaa'", Output: ""}, {Match: "docker rm 'scratch-web-aaa'", Output: ""}, {Match: "find '/deployments/scratch'", Output: ""}, diff --git a/internal/cli/retention.go b/internal/cli/retention.go new file mode 100644 index 0000000..2f421e7 --- /dev/null +++ b/internal/cli/retention.go @@ -0,0 +1,222 @@ +package cli + +// The two disk remedies `server audit` offers: `releases prune --keep N` and +// `server prune --build-cache --dangling`. Neither touches a volume. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/signal" + "strings" + + "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/events" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +type releasesPruneResult struct { + MachineInterface int `json:"machine_interface"` + App string `json:"app"` + Server string `json:"server"` + Keep int `json:"keep"` + Pruned []string `json:"pruned"` + Protected []string `json:"protected"` +} + +func newReleasesPruneCmd(flags *Flags) *cobra.Command { + var app string + var keep int + cmd := &cobra.Command{ + Use: "prune", + Short: "Remove old versions' containers and images beyond --keep", + Long: `Keeps the --keep most recent versions of a container app and removes the +containers and images of the rest. Pinned, current and previous versions and any +version with a running container are never removed, whatever --keep says. Volumes +and accessories are not touched. Takes the app's lock; recorded in its event +history. Nothing beyond --keep is success. + +--keep 0 removes every version the protections above do not name — unlike +keep_versions: 0 in teploy.yml, which keeps everything.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if !cmd.Flags().Changed("keep") { + return fmt.Errorf("--keep is required: how many recent versions to retain") + } + if keep < 0 { + return fmt.Errorf("--keep must be >= 0") + } + return runReleasesPrune(flags, app, keep, cmd.OutOrStdout()) + }, + } + cmd.Flags().StringVar(&app, "app", "", "app name — act on server state instead of teploy.yml (requires --host)") + cmd.Flags().IntVar(&keep, "keep", 0, "how many recent versions to retain") + return cmd +} + +func runReleasesPrune(flags *Flags, app string, keep int, out io.Writer) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + appCfg, exec, err := resolveApp(ctx, flags, app) + if err != nil { + return err + } + defer exec.Close() + res, err := executeReleasesPrune(ctx, exec, appCfg.App, keep) + if err != nil { + return err + } + if flags.JSON { + return json.NewEncoder(out).Encode(res) + } + if len(res.Pruned) == 0 { + fmt.Fprintf(out, "Nothing to prune for %s\n", res.App) + } else { + fmt.Fprintf(out, "Pruned %d version(s) of %s: %s\n", len(res.Pruned), res.App, strings.Join(res.Pruned, ", ")) + } + return nil +} + +func executeReleasesPrune(ctx context.Context, exec ssh.Executor, app string, keep int) (*releasesPruneResult, error) { + res := &releasesPruneResult{MachineInterface: MachineInterface, App: app, Server: exec.Host(), Keep: keep, Pruned: []string{}, Protected: []string{}} + exists, err := appDirExists(ctx, exec, app) + if err != nil { + return nil, err + } + if !exists { + return res, nil // already gone + } + lk, err := state.AcquireLockFenced(ctx, exec, app) + if err != nil { + return nil, fmt.Errorf("acquiring deploy lock: %w", err) + } + defer state.ReleaseLockFenced(exec, lk, app) + + // Every read is under the lock, and each failure stops the prune: acting + // without knowing current, previous or the pins could delete them. + st, err := state.Read(ctx, exec, app) + if err != nil { + return nil, fmt.Errorf("reading state for %s: %w", app, err) + } + if st == nil { + return res, nil + } + if st.DeploymentType == "static" { + return nil, fmt.Errorf("%s is a static app: its releases are pruned on deploy by keep_releases", app) + } + pins, err := state.ReadPins(ctx, exec, app) + if err != nil { + return nil, fmt.Errorf("refusing to prune: %w", err) + } + dk := docker.NewClient(exec) + containers, err := dk.ListContainers(ctx, app) + if err != nil { + return nil, fmt.Errorf("refusing to prune: %w", err) + } + for _, v := range retentionProtected(st, pins, containers) { + if v != "" { + res.Protected = append(res.Protected, v) + } + } + if len(docker.PrunableVersions(containers, keep, res.Protected...)) == 0 { + return res, nil + } + fenced := docker.NewClient(&state.FencedExecutor{Executor: exec, Lock: lk}) + pruned, pruneErr := fenced.PruneVersions(ctx, app, keep, res.Protected...) + res.Pruned = append(res.Pruned, pruned...) + events.Record(ctx, exec, events.Event{App: app, Kind: events.KindReleasesPrune, Outcome: events.Outcome(pruneErr), Release: st.CurrentHash, + Detail: events.ErrDetail(map[string]any{"keep": keep, "pruned": res.Pruned, "protected": res.Protected, "trigger": "command"}, pruneErr)}) + if pruneErr != nil { + return nil, fmt.Errorf("prune incomplete: %w", pruneErr) + } + return res, nil +} + +type serverPruneResult struct { + MachineInterface int `json:"machine_interface"` + Server string `json:"server"` + BuildCache bool `json:"build_cache"` + Dangling bool `json:"dangling"` + Output string `json:"output"` +} + +func newServerPruneCmd(flags *Flags) *cobra.Command { + var buildCache, dangling bool + cmd := &cobra.Command{ + Use: "prune", + Short: "Remove unused build cache and dangling images", + Long: `--build-cache removes docker's unused build cache: the next build rebuilds it. +--dangling removes untagged images no container uses. Nothing else: no tagged +image, no container, no volume. Recorded in the server's event history.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if !buildCache && !dangling { + return fmt.Errorf("say what to prune: --build-cache and/or --dangling") + } + if flags.Host == "" { + return fmt.Errorf("--host is required") + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + host, user, key, err := config.ResolveServer(flags.Host, flags.Host, flags.User, flags.Key) + if err != nil { + return err + } + exec, err := ssh.Connect(ctx, ssh.ConnectConfig{Host: host, User: user, KeyPath: key}) + if err != nil { + return err + } + defer exec.Close() + res, err := executeServerPrune(ctx, exec, flags.Host, buildCache, dangling) + if err != nil { + return err + } + if flags.JSON { + return json.NewEncoder(cmd.OutOrStdout()).Encode(res) + } + fmt.Fprintln(cmd.OutOrStdout(), res.Output) + return nil + }, + } + cmd.Flags().BoolVar(&buildCache, "build-cache", false, "remove unused build cache") + cmd.Flags().BoolVar(&dangling, "dangling", false, "remove dangling images") + return cmd +} + +func executeServerPrune(ctx context.Context, exec ssh.Executor, server string, buildCache, dangling bool) (*serverPruneResult, error) { + res := &serverPruneResult{MachineInterface: MachineInterface, Server: server, BuildCache: buildCache, Dangling: dangling} + var failures []error + var lines []string + run := func(cmd string) { + out, err := exec.Run(ctx, cmd) + if err != nil { + failures = append(failures, fmt.Errorf("%s: %w", cmd, err)) + return + } + for _, line := range doctorLines(out) { + if strings.HasPrefix(line, "Total") { + lines = append(lines, line) + } + } + } + if buildCache { + run("docker builder prune -a -f") + } + if dangling { + run("docker image prune -f") + } + res.Output = strings.Join(lines, "; ") + err := errors.Join(failures...) + events.Record(ctx, exec, events.Event{App: "_fleet", Kind: events.KindServerPrune, Outcome: events.Outcome(err), + Detail: events.ErrDetail(map[string]any{"build_cache": buildCache, "dangling": dangling, "output": res.Output}, err)}) + if err != nil { + return nil, err + } + return res, nil +} diff --git a/internal/cli/retention_test.go b/internal/cli/retention_test.go new file mode 100644 index 0000000..52fe0a5 --- /dev/null +++ b/internal/cli/retention_test.go @@ -0,0 +1,284 @@ +package cli + +// `releases prune --keep N` and `server prune --build-cache --dangling` +// tests: the selection keeps exactly what the plan protects, and acting +// without knowing current, previous or the pins refuses. + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/ssh" +) + +// pruneFixture models a container app with versions v1..v4 on a server. +func pruneFixture(t *testing.T, manifest string, pins string, extra ...ssh.MockCommand) *ssh.MockExecutor { + t.Helper() + cmds := append([]ssh.MockCommand{}, extra...) + cmds = append(cmds, + ssh.MockCommand{Match: "if [ -d '/deployments/blog' ]", Output: "yes"}, + ssh.MockCommand{Match: "mkdir /deployments/blog/.lock", Output: ""}, + ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='blog'", Output: strings.Join([]string{ + auditContainer("blog-web-v4", "blog:v4", "running", "2026-11-04 09:00:00 +0000 UTC", + map[string]string{"teploy.app": "blog", "teploy.process": "web", "teploy.version": "v4"}), + auditContainer("blog-web-v3", "blog:v3", "exited", "2026-11-03 09:00:00 +0000 UTC", + map[string]string{"teploy.app": "blog", "teploy.process": "web", "teploy.version": "v3"}), + auditContainer("blog-web-v2", "blog:v2", "exited", "2026-11-02 09:00:00 +0000 UTC", + map[string]string{"teploy.app": "blog", "teploy.process": "web", "teploy.version": "v2"}), + auditContainer("blog-web-v1", "blog:v1", "exited", "2026-11-01 09:00:00 +0000 UTC", + map[string]string{"teploy.app": "blog", "teploy.process": "web", "teploy.version": "v1"}), + }, "\n")}, + ssh.MockCommand{Match: "docker rm -f", Output: ""}, + ssh.MockCommand{Match: "docker rmi", Output: ""}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq": 1, "ts": "2026-11-05T12:00:00Z"}`}, + ) + m := ssh.NewMockExecutor("192.0.2.10", cmds...) + m.Files["/deployments/blog/state.json"] = []byte(auditStateJSON("container", "caddy", "v4", "v3", manifestMap(manifest))) + if pins != "" { + m.Files["/deployments/blog/pinned"] = []byte(pins) + } + return m +} + +func manifestMap(s string) map[string]any { + if s == "" { + return nil + } + var m map[string]any + if err := json.Unmarshal([]byte(s), &m); err != nil { + panic(err) + } + return m +} + +func TestReleasesPrune_KeepsExactlyTheExpectedSet(t *testing.T) { + m := pruneFixture(t, `{"container":{"keep_versions":2}}`, "v1\n") + res, err := executeReleasesPrune(context.Background(), m, "blog", 2) + if err != nil { + t.Fatalf("executeReleasesPrune: %v", err) + } + if fmt.Sprint(res.Pruned) != fmt.Sprint([]string{"v2"}) { + t.Errorf("pruned = %v, want [v2] (v4 current, v3 previous, v1 pinned)", res.Pruned) + } + if fmt.Sprint(res.Protected) != fmt.Sprint([]string{"v4", "v3", "v1"}) { + t.Errorf("protected = %v, want [v4 v3 v1]", res.Protected) + } + for _, call := range m.Calls { + if strings.HasPrefix(call, "docker rm -f") || strings.HasPrefix(call, "docker rmi") { + for _, keep := range []string{"v1", "v3", "v4"} { + if strings.Contains(call, keep) { + t.Errorf("protected version %s was removed: %s", keep, call) + } + } + } + } + if !sawCall(m, "docker rm -f 'blog-web-v2'") || !sawCall(m, "docker rmi 'blog:v2'") { + t.Errorf("v2's container and image were not both removed; calls: %v", m.Calls) + } +} + +// keep 0 on the command means "everything the protections do not name" — +// deliberately different from keep_versions: 0 in teploy.yml. +func TestReleasesPrune_KeepZeroPrunesAllUnprotected(t *testing.T) { + m := pruneFixture(t, `{"container":{"keep_versions":2}}`, "v1\n") + res, err := executeReleasesPrune(context.Background(), m, "blog", 0) + if err != nil { + t.Fatalf("executeReleasesPrune: %v", err) + } + if fmt.Sprint(res.Pruned) != fmt.Sprint([]string{"v2"}) { + t.Errorf("pruned = %v, want [v2]: current, previous and pins survive even --keep 0", res.Pruned) + } +} + +func TestReleasesPrune_NothingBeyondKeepIsSuccess(t *testing.T) { + m := pruneFixture(t, `{"container":{"keep_versions":5}}`, "") + res, err := executeReleasesPrune(context.Background(), m, "blog", 5) + if err != nil { + t.Fatalf("executeReleasesPrune: %v", err) + } + if len(res.Pruned) != 0 { + t.Errorf("pruned = %v, want nothing", res.Pruned) + } + for _, call := range m.Calls { + if strings.HasPrefix(call, "docker rm") || strings.HasPrefix(call, "docker rmi") { + t.Errorf("nothing was beyond keep, yet: %s", call) + } + } +} + +func TestReleasesPrune_RunsUnderTheAppLock(t *testing.T) { + m := pruneFixture(t, `{"container":{"keep_versions":2}}`, "v1\n") + if _, err := executeReleasesPrune(context.Background(), m, "blog", 2); err != nil { + t.Fatal(err) + } + if !sawCall(m, "mkdir /deployments/blog/.lock 2>/dev/null") { + t.Errorf("the prune never took the app lock; calls: %v", m.Calls) + } + // Container removals run under the lock's fence guard. + var fenced bool + for _, call := range m.Calls { + if strings.HasPrefix(call, "grep -q ") && strings.Contains(call, "docker rm -f") { + fenced = true + } + } + if !fenced { + t.Errorf("container removals were not fenced by the lock; calls: %v", m.Calls) + } +} + +func TestReleasesPrune_RefusesWhenItCannotKnowTheProtections(t *testing.T) { + tests := []struct { + name string + extra []ssh.MockCommand + want string + }{ + { + name: "unreadable pins", + extra: []ssh.MockCommand{{Match: "if [ ! -e '/deployments/blog/pinned'", Err: errors.New("permission denied")}}, + want: "refusing to prune", + }, + { + name: "unreadable state", + extra: []ssh.MockCommand{{Match: "if [ ! -e '/deployments/blog/state.json'", Err: errors.New("permission denied")}}, + want: "reading state", + }, + { + name: "unreadable containers", + extra: []ssh.MockCommand{{Match: "docker ps --all --filter label=teploy.app='blog'", Err: errors.New("docker: no daemon")}}, + want: "refusing to prune", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := pruneFixture(t, `{"container":{"keep_versions":1}}`, "", tt.extra...) + _, err := executeReleasesPrune(context.Background(), m, "blog", 1) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("error = %v, want it to contain %q", err, tt.want) + } + for _, call := range m.Calls { + if strings.HasPrefix(call, "docker rm") || strings.HasPrefix(call, "docker rmi") { + t.Errorf("a refused prune removed things: %s", call) + } + } + }) + } +} + +func TestReleasesPrune_StaticAppExplainsItself(t *testing.T) { + m := pruneFixture(t, "", "") + m.Files["/deployments/blog/state.json"] = []byte(auditStateJSON("static", "caddy", "s1", "", nil)) + _, err := executeReleasesPrune(context.Background(), m, "blog", 2) + if err == nil || !strings.Contains(err.Error(), "keep_releases") { + t.Fatalf("error = %v, want the static keep_releases explanation", err) + } +} + +func TestReleasesPrune_AlreadyRemovedAppIsSuccess(t *testing.T) { + m := ssh.NewMockExecutor("192.0.2.10", + ssh.MockCommand{Match: "if [ -d '/deployments/blog' ]", Output: "no"}, + ) + res, err := executeReleasesPrune(context.Background(), m, "blog", 2) + if err != nil { + t.Fatalf("an app with no directory must be a no-op success: %v", err) + } + if len(res.Pruned) != 0 { + t.Errorf("pruned = %v", res.Pruned) + } +} + +func sawCall(m *ssh.MockExecutor, prefix string) bool { + for _, call := range m.Calls { + if strings.HasPrefix(call, prefix) { + return true + } + if strings.Contains(call, prefix) { // a fenced/guarded composition + return true + } + } + return false +} + +func TestServerPrune_RunsExactlyTheSelectedPrunes(t *testing.T) { + m := ssh.NewMockExecutor("192.0.2.10", + ssh.MockCommand{Match: "docker builder prune -a -f", Output: "Total reclaimed space: 1.5GB"}, + ssh.MockCommand{Match: "docker image prune -f", Output: "Total reclaimed space: 300MB"}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq": 1, "ts": "2026-11-05T12:00:00Z"}`}, + ) + res, err := executeServerPrune(context.Background(), m, "edge", true, true) + if err != nil { + t.Fatalf("executeServerPrune: %v", err) + } + if !res.BuildCache || !res.Dangling { + t.Errorf("result flags = %+v", res) + } + if res.Output != "Total reclaimed space: 1.5GB; Total reclaimed space: 300MB" { + t.Errorf("output = %q", res.Output) + } + // Only the two reclaim prunes ran: no tagged image, container or volume. + for _, call := range m.Calls { + if strings.Contains(call, "docker volume prune") || strings.Contains(call, "docker system prune") || + strings.Contains(call, "docker container prune") || strings.Contains(call, "-a -f --volumes") { + t.Errorf("server prune went beyond build cache and dangling images: %s", call) + } + } + // Recorded under the fleet pseudo-app. + var fleetEvent bool + for _, input := range m.Inputs { + if strings.Contains(input, `"app":"_fleet"`) && strings.Contains(input, `"server.prune"`) { + fleetEvent = true + } + } + if !fleetEvent { + t.Errorf("no server.prune event under _fleet; inputs: %v", m.Inputs) + } +} + +func TestServerPrune_OnlyTheSelectedFlagRuns(t *testing.T) { + m := ssh.NewMockExecutor("192.0.2.10", + ssh.MockCommand{Match: "docker image prune -f", Output: "Total reclaimed space: 300MB"}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq": 1, "ts": "2026-11-05T12:00:00Z"}`}, + ) + if _, err := executeServerPrune(context.Background(), m, "edge", false, true); err != nil { + t.Fatal(err) + } + for _, call := range m.Calls { + if strings.Contains(call, "builder prune") { + t.Errorf("--build-cache was not asked for: %s", call) + } + } +} + +func TestServerPrune_FailureIsReported(t *testing.T) { + m := ssh.NewMockExecutor("192.0.2.10", + ssh.MockCommand{Match: "docker builder prune -a -f", Err: errors.New("docker: no space left on device")}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq": 1, "ts": "2026-11-05T12:00:00Z"}`}, + ) + _, err := executeServerPrune(context.Background(), m, "edge", true, false) + if err == nil { + t.Fatal("a failed prune must surface its error") + } +} + +func TestServerPruneFlagValidation(t *testing.T) { + var out bytes.Buffer + cmd := newServerPruneCmd(&Flags{}) + cmd.SetOut(&out) + cmd.SetErr(&out) + cmd.SetArgs([]string{}) + if err := cmd.Execute(); err == nil || !strings.Contains(err.Error(), "say what to prune") { + t.Errorf("no flags: %v", err) + } + + cmd = newServerPruneCmd(&Flags{}) + cmd.SetOut(&out) + cmd.SetErr(&out) + cmd.SetArgs([]string{"--build-cache"}) + if err := cmd.Execute(); err == nil || !strings.Contains(err.Error(), "--host is required") { + t.Errorf("missing --host: %v", err) + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index e06be71..c80c595 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -136,6 +136,7 @@ func NewRootCmd(version string) *cobra.Command { root.AddCommand(newValidateCmd(flags)) root.AddCommand(newDoctorCmd(flags, version)) root.AddCommand(newDomainsCmd(flags)) + root.AddCommand(newRouteCmd(flags)) root.AddCommand(newMetaCmd(flags)) root.AddCommand(newPlanCmd(flags)) root.AddCommand(newApplyCmd(flags)) diff --git a/internal/cli/route.go b/internal/cli/route.go new file mode 100644 index 0000000..6bfd4fe --- /dev/null +++ b/internal/cli/route.go @@ -0,0 +1,215 @@ +package cli + +// `teploy route remove`: retire a managed Caddy block that nothing serves. Only +// a route proven dead goes (removalRefusal), only that app's own marked region +// is edited, and the commit is a compare-and-swap on the exact block that was +// judged, so a deploy that rewrites it in between is not undone. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/signal" + "strings" + + "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/caddy" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/domains" + "github.com/useteploy/teploy/internal/events" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +var errRouteRefused = errors.New("route remove refused") + +type routeRemoveResult struct { + MachineInterface int `json:"machine_interface"` + App string `json:"app"` + Server string `json:"server"` + Route string `json:"route"` // removed | absent; refused/failed appear only in event detail + Hosts []string `json:"hosts,omitempty"` + Upstreams []string `json:"upstreams,omitempty"` +} + +func newRouteCmd(flags *Flags) *cobra.Command { + cmd := &cobra.Command{Use: "route", Short: "Manage Caddy routes"} + var app string + remove := &cobra.Command{ + Use: "remove", + Short: "Remove a managed route that nothing serves", + Long: `Removes one app's managed Caddy block, reloads Caddy, and records it in the +app's event history. Hand-written blocks and other apps' blocks are never +touched. An absent block is success. + +Refuses a route belonging to a live app, and any route not proven dead: an +upstream that resolves or cannot be checked, files that still exist, app state +or containers that cannot be read, or a block carrying the autodeploy webhook.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runRouteRemove(flags, app, cmd.OutOrStdout()) + }, + } + remove.Flags().StringVar(&app, "app", "", "the app whose managed block to remove") + _ = remove.MarkFlagRequired("app") + cmd.AddCommand(remove) + return cmd +} + +func runRouteRemove(flags *Flags, app string, out io.Writer) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + serverName, exec, err := connectAppServer(ctx, flags, app) + if err != nil { + return err + } + defer exec.Close() + res, err := executeRouteRemove(ctx, exec, app) + if err != nil { + return err + } + res.Server = serverName + if flags.JSON { + return json.NewEncoder(out).Encode(res) + } + if res.Route == "absent" { + fmt.Fprintf(out, "%s has no managed route on %s\n", app, serverName) + } else { + fmt.Fprintf(out, "Removed %s's route on %s (%s)\n", app, serverName, strings.Join(res.Hosts, ", ")) + } + return nil +} + +// routeLockApp is the app whose lock serializes an edit of this route. A +// preview's route has no directory of its own, so its parent's lock guards it. +func routeLockApp(ctx context.Context, exec ssh.Executor, app string) (string, error) { + exists, err := appDirExists(ctx, exec, app) + if err != nil || exists { + return app, err + } + parent, _, ok := strings.Cut(app, "-preview-") + if !ok || config.ValidateName(parent) != nil { + return "", nil + } + if exists, err = appDirExists(ctx, exec, parent); err != nil || !exists { + return "", err + } + return parent, nil +} + +// appDirExists separates "no directory" from "could not ask". +func appDirExists(ctx context.Context, exec ssh.Executor, app string) (bool, error) { + out, err := exec.Run(ctx, "if [ -d "+ssh.ShellQuote("/deployments/"+app)+" ]; then echo yes; else echo no; fi") + if err != nil { + return false, err + } + return strings.TrimSpace(out) == "yes", nil +} + +func executeRouteRemove(ctx context.Context, exec ssh.Executor, app string) (*routeRemoveResult, error) { + res := &routeRemoveResult{MachineInterface: MachineInterface, App: app, Server: exec.Host(), Route: "absent"} + record := func(outcome string, err error, reason string) { + events.Record(ctx, exec, events.Event{App: app, Kind: events.KindRouteRemove, Outcome: outcome, + Detail: events.ErrDetail(map[string]any{"route": res.Route, "hosts": res.Hosts, "upstreams": res.Upstreams, "reason": reason}, err)}) + } + + lockApp, err := routeLockApp(ctx, exec, app) + if err != nil { + return nil, err + } + var lk *state.Lock + if lockApp != "" { + if lk, err = state.AcquireLockFenced(ctx, exec, lockApp); err != nil { + return nil, fmt.Errorf("acquiring deploy lock: %w", err) + } + defer state.ReleaseLockFenced(exec, lk, lockApp) + } + + cd := caddy.NewClient(exec) + content, present, err := cd.ReadCaddyfile(ctx) + if err != nil { + return nil, fmt.Errorf("reading the Caddyfile: %w", err) + } + var region *caddy.ManagedRegion + if present { + regions, err := caddy.ManagedRegions(content) + if err != nil { + return nil, fmt.Errorf("reading the Caddyfile: %w", err) + } + for i := range regions { + if regions[i].App == app { + region = ®ions[i] + } + } + } + if region == nil { + record(events.Succeeded, nil, "") + return res, nil + } + res.Hosts = siteHosts(region.Sites) + for _, site := range region.Sites { + res.Upstreams = append(res.Upstreams, site.Upstreams...) + } + + facts := routeFacts{app: app, region: region.Text, sites: region.Sites, resolves: map[string]string{}, roots: map[string]string{}} + st, stErr := state.Read(ctx, exec, app) + facts.st, facts.stateKnown = st, stErr == nil + containers, listErr := docker.NewClient(exec).ListAllContainers(ctx) + facts.containers, facts.known = containers, listErr == nil + facts.resolves = resolveRegionDials(ctx, exec, region.Sites, containers) + for _, root := range caddy.RegionRoots(region.Text) { + facts.roots[root], _ = readRootExists(ctx, exec, root) + } + if reason := removalRefusal(facts); reason != "" { + err := fmt.Errorf("%w: %s: %s", errRouteRefused, app, reason) + res.Route = "refused" + record(events.Refused, nil, reason) + return nil, err + } + + generation, _ := caddy.RegionGeneration(region.Text) + guard := "" + if lk != nil { + guard = lk.GuardPrefix() + } + err = cd.WithCommitGuard(guard).WithRouteCAS(app, []string{caddy.ManagedRegionHash(region.Text)}, generation).RemoveRoute(ctx, app) + if err != nil { + res.Route = "failed" + record(events.Failed, err, "") + return nil, fmt.Errorf("removing %s's route: %w", app, err) + } + res.Route = "removed" + record(events.Succeeded, nil, "") + return res, nil +} + +// resolveRegionDials asks the running Caddy about a region's upstreams. No +// answer leaves a name unknown, which refuses the removal. +func resolveRegionDials(ctx context.Context, exec ssh.Executor, sites []caddy.RouteSite, containers []docker.Container) map[string]string { + running := false + for _, c := range containers { + if c.Name == "caddy" && c.State == "running" { + running = true + } + } + var names []string + for _, site := range sites { + for _, dial := range site.Upstreams { + if name, _ := domains.UpstreamHost(dial); name != "" { + names = append(names, name) + } + } + } + if !running || len(names) == 0 { + return map[string]string{} + } + answers, err := domains.ResolveNames(ctx, exec, names) + if err != nil { + return map[string]string{} + } + return answers +} diff --git a/internal/cli/route_remove_test.go b/internal/cli/route_remove_test.go new file mode 100644 index 0000000..406b807 --- /dev/null +++ b/internal/cli/route_remove_test.go @@ -0,0 +1,281 @@ +package cli + +// `teploy route remove` tests: only a block proven dead goes, the commit is a +// compare-and-swap on the exact block that was judged, and every refusal path +// leaves the Caddyfile untouched. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/caddy" + "github.com/useteploy/teploy/internal/ssh" +) + +const routeRemoveCaddyfile = `example.com { + reverse_proxy other:80 +} + +# TEPLOY BEGIN ghost +ghost.example.com { + reverse_proxy ghost-web-v1:3000 +} +# TEPLOY END ghost +` + +// routeRemoveExecutor models a server whose ghost app is entirely gone: a +// directory with a lock, no state, no containers, a dead upstream. Extra +// registrations are consulted first, so a test can override any read. +func routeRemoveExecutor(caddyfile string, extra ...ssh.MockCommand) *ssh.MockExecutor { + cmds := append([]ssh.MockCommand{}, extra...) + cmds = append(cmds, + ssh.MockCommand{Match: "if [ -d '/deployments/ghost' ]", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/ghost/.lock", Output: ""}, + ssh.MockCommand{Match: "docker ps --all ", Output: routeRemoveContainers()}, + ssh.MockCommand{Match: "printf '%s\\n'", Output: "2 ghost-web-v1\n"}, + ssh.MockCommand{Match: "mkdir /deployments/caddy/.lock", Output: ""}, + ssh.MockCommand{Match: "docker exec caddy caddy reload", Output: ""}, + ssh.MockCommand{Match: "a=$(docker exec caddy md5sum", Output: "TEPLOY_CADDY_OK"}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq": 1, "ts": "2026-11-05T12:00:00Z"}`}, + ) + m := ssh.NewMockExecutor("192.0.2.10", cmds...) + if caddyfile != "" { + m.Files["/deployments/caddy/Caddyfile"] = []byte(caddyfile) + } + return m +} + +func routeRemoveContainers() string { + return auditContainer("caddy", "caddy:2", "running", "2026-05-01 00:00:00 +0000 UTC", nil) +} + +func recordedEvents(t *testing.T, inputs []string) []map[string]any { + t.Helper() + var out []map[string]any + for _, input := range inputs { + var ev map[string]any + if err := json.Unmarshal([]byte(input), &ev); err != nil { + continue // a non-event stdin payload (the caddy adapt check) + } + if ev["kind"] == "route.remove" { + out = append(out, ev) + } + } + return out +} + +func TestRouteRemove_RemovesProvenDeadBlock(t *testing.T) { + m := routeRemoveExecutor(routeRemoveCaddyfile) + res, err := executeRouteRemove(context.Background(), m, "ghost") + if err != nil { + t.Fatalf("executeRouteRemove: %v", err) + } + if res.Route != "removed" { + t.Errorf("route = %s, want removed", res.Route) + } + if fmt.Sprint(res.Hosts) != fmt.Sprint([]string{"ghost.example.com"}) { + t.Errorf("hosts = %v", res.Hosts) + } + written := string(m.Files["/deployments/caddy/Caddyfile"]) + if strings.Contains(written, "TEPLOY BEGIN ghost") || strings.Contains(written, "ghost.example.com") { + t.Errorf("the ghost block survived: %q", written) + } + if !strings.Contains(written, "example.com {") { + t.Errorf("an unrelated block was dropped: %q", written) + } + // The removal is committed under the exact-block CAS on the judged block. + var casCommit bool + for _, call := range m.Calls { + if strings.Contains(call, "TEPLOY_ROUTE_CAS_MISMATCH") && strings.Contains(call, "mv -fT -- ") { + casCommit = true + } + } + if !casCommit { + t.Errorf("the commit was not guarded by the route CAS; calls: %v", m.Calls) + } + // Recorded in the app's event history. + evs := recordedEvents(t, m.Inputs) + if len(evs) != 1 || evs[0]["outcome"] != "succeeded" { + t.Fatalf("events = %v", evs) + } + if evs[0]["detail"].(map[string]any)["route"] != "removed" { + t.Errorf("event detail = %v", evs[0]["detail"]) + } +} + +func TestRouteRemove_AbsentBlockIsSuccess(t *testing.T) { + m := routeRemoveExecutor("example.com {\n\treverse_proxy other:80\n}\n") + res, err := executeRouteRemove(context.Background(), m, "ghost") + if err != nil { + t.Fatalf("an absent block must succeed: %v", err) + } + if res.Route != "absent" { + t.Errorf("route = %s, want absent", res.Route) + } +} + +func TestRouteRemove_Refusals(t *testing.T) { + tests := []struct { + name string + expect string + edited string // expected Caddyfile after the refusal ("" = unchanged) + extra []ssh.MockCommand + files map[string][]byte + }{ + { + name: "live app", + expect: "belongs to a live app", + extra: []ssh.MockCommand{{Match: "docker ps --all ", Output: routeRemoveContainers() + "\n" + + auditContainer("ghost-web-v1", "ghost:v1", "running", "2026-11-04 10:00:00 +0000 UTC", + map[string]string{"teploy.app": "ghost", "teploy.process": "web", "teploy.version": "v1"})}}, + files: map[string][]byte{"/deployments/ghost/state.json": []byte(auditStateJSON("container", "caddy", "v1", "", nil))}, + }, + { + name: "container still carries the name", + expect: "still carries this app's name", + extra: []ssh.MockCommand{{Match: "docker ps --all ", Output: routeRemoveContainers() + "\n" + + auditContainer("ghost-web-v1", "ghost:v1", "exited", "2026-11-04 10:00:00 +0000 UTC", + map[string]string{"teploy.app": "ghost", "teploy.process": "web", "teploy.version": "v1"})}}, + }, + { + name: "upstream resolves", + expect: "still resolves", + extra: []ssh.MockCommand{{Match: "printf '%s\\n'", Output: "0 ghost-web-v1\n"}}, + }, + { + name: "upstream unknowable", + expect: "could not be proven dead", + extra: []ssh.MockCommand{{Match: "printf '%s\\n'", Err: errors.New("getent: temporary failure")}}, + }, + { + name: "unreadable state", + expect: "state could not be read", + extra: []ssh.MockCommand{{Match: "if [ ! -e '/deployments/ghost/state.json'", Err: errors.New("permission denied")}}, + }, + { + name: "webhook block", + expect: "autodeploy webhook", + edited: "# TEPLOY BEGIN ghost\nghost.example.com {\n\treverse_proxy ghost-web-v1:3000\n\t# TEPLOY WEBHOOK BEGIN ghost\n\t# TEPLOY WEBHOOK END ghost\n}\n# TEPLOY END ghost\n", + }, + { + name: "serving files that exist", + expect: "serves files from", + edited: "# TEPLOY BEGIN ghost\nghost.example.com {\n\troot * /deployments/ghost/site\n\tfile_server\n}\n# TEPLOY END ghost\n", + extra: []ssh.MockCommand{{Match: "if [ -e '/deployments/ghost/site'", Output: "yes"}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + before := routeRemoveCaddyfile + if tt.edited != "" { + before = tt.edited + } + m := routeRemoveExecutor(before, tt.extra...) + for p, data := range tt.files { + m.Files[p] = data + } + _, err := executeRouteRemove(context.Background(), m, "ghost") + if !errors.Is(err, errRouteRefused) { + t.Fatalf("error = %v, want a route remove refusal", err) + } + if !strings.Contains(err.Error(), tt.expect) { + t.Errorf("refusal = %q, want it to name %q", err, tt.expect) + } + if written := string(m.Files["/deployments/caddy/Caddyfile"]); written != before { + t.Errorf("a refusal must not edit the Caddyfile:\n got: %q\nwant: %q", written, before) + } + for _, call := range m.Calls { + if strings.Contains(call, "mv -fT -- ") { + t.Errorf("a refusal must not commit: %s", call) + } + } + evs := recordedEvents(t, m.Inputs) + if len(evs) != 1 || evs[0]["outcome"] != "refused" { + t.Errorf("refusal events = %v", evs) + } + }) + } +} + +// concurrentRewriteExecutor models a deploy rewriting the app's block between +// `route remove` judging it and committing: the second read of the Caddyfile +// (inside the caddy client's mutate) answers different content. +type concurrentRewriteExecutor struct { + *ssh.MockExecutor + rewrite string + catSeen int + catCaddyfile string + eventAppended bool +} + +func (e *concurrentRewriteExecutor) Run(ctx context.Context, cmd string) (string, error) { + if cmd == e.catCaddyfile { + e.catSeen++ + if e.catSeen == 1 { + // A successor deploys a new block for the app between the + // judgement (the framed read) and the commit (this plain cat + // inside the caddy client's mutate). + e.Files["/deployments/caddy/Caddyfile"] = []byte(e.rewrite) + } + } + if strings.HasPrefix(cmd, "python3 -c") { + e.eventAppended = true + } + return e.MockExecutor.Run(ctx, cmd) +} + +// RunStream records the event append too: the ssh package's structured +// capture falls back to RunStream for executors it does not know, so the +// history append arrives here rather than through Run. +func (e *concurrentRewriteExecutor) RunStream(ctx context.Context, cmd string, stdout, stderr io.Writer) error { + if strings.HasPrefix(cmd, "python3 -c") { + e.eventAppended = true + } + return e.MockExecutor.RunStream(ctx, cmd, stdout, stderr) +} + +func TestRouteRemove_ConcurrentRewriteIsRefused(t *testing.T) { + base := routeRemoveExecutor(routeRemoveCaddyfile) + rewritten := `example.com { + reverse_proxy other:80 +} + +# TEPLOY BEGIN ghost +ghost.example.com { + reverse_proxy ghost-web-v9:3000 +} +# TEPLOY END ghost +` + m := &concurrentRewriteExecutor{MockExecutor: base, rewrite: rewritten, catCaddyfile: "cat /deployments/caddy/Caddyfile"} + _, err := executeRouteRemove(context.Background(), m, "ghost") + if err == nil { + t.Fatal("a block that changed under the removal must refuse") + } + var casErr *caddy.ErrRouteCAS + if !errors.As(err, &casErr) { + t.Fatalf("error = %v, want a route CAS refusal", err) + } + after := string(m.Files["/deployments/caddy/Caddyfile"]) + if after != rewritten { + t.Errorf("a refused commit must leave the successor's block exactly as it is: %q", after) + } + if !strings.Contains(after, "TEPLOY BEGIN ghost") { + t.Errorf("the app's block was removed despite the refusal: %q", after) + } + for path := range m.Files { + if strings.Contains(path, ".tmp-") { + t.Errorf("the staged caddyfile was not cleaned up: %s", path) + } + } + // The wrapper's structured capture falls back to RunStream, so the event + // payload is not captured in Inputs; the failure is still recorded — the + // append call itself must have run. + if !m.eventAppended { + t.Error("the refused removal was not recorded in the event history") + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index d0dd928..9930a9b 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -29,6 +29,8 @@ func newServerCmd(flags *Flags) *cobra.Command { cmd.AddCommand(newServerStatusCmd(flags)) cmd.AddCommand(newServerBackupTargetCmd(flags)) cmd.AddCommand(newServerBackupPolicyCmd(flags)) + cmd.AddCommand(newServerAuditCmd(flags)) + cmd.AddCommand(newServerPruneCmd(flags)) return cmd } diff --git a/internal/cli/server_audit.go b/internal/cli/server_audit.go new file mode 100644 index 0000000..dfd6f50 --- /dev/null +++ b/internal/cli/server_audit.go @@ -0,0 +1,1033 @@ +package cli + +// `teploy server audit `: what is left over on a server, with evidence, +// and the one command that fixes it where a command can do so safely. Read-only: +// every remote command here only reads (a test pins that). What could not be +// read is named in checks[] and yields no finding: it is never reported clean. + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "os" + "os/signal" + "regexp" + "slices" + "sort" + "strconv" + "strings" + "time" + + "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/backup" + "github.com/useteploy/teploy/internal/caddy" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/deploy" + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/domains" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +// Finding kinds. Stable identifiers: consumers code against them. +const ( + findingStaleRoute = "stale_route" + findingEmptyRegion = "empty_managed_region" + findingDeadUpstream = "managed_route_dead_upstream" + findingOrphanState = "orphan_state" + findingManualDead = "manual_route_dead_upstream" + findingManualNonLocal = "manual_route_nonlocal_address" + findingVersionsBeyond = "versions_beyond_policy" + findingBuildArtifacts = "reclaimable_build_artifacts" + findingLargeLayer = "large_writable_layer" + findingUnmanagedLarge = "unmanaged_container" + findingDanglingVolumes = "dangling_volumes" + findingCaddyfileNoCaddy = "caddyfile_without_caddy" + findingSchedulerMissing = "scheduler_missing" + findingDiskPressure = "disk_pressure" +) + +const ( + severityCritical = "critical" + severityWarning = "warning" + severityInfo = "info" + + auditCheckOK = "ok" + auditCheckUnknown = "unknown" + + auditStateOK = "ok" + auditStatePartial = "partial" + auditStateUnreachable = "unreachable" + + auditLargeLayerBytes = 5_000_000_000 + auditLargeUnmanagedBytes = 1_000_000_000 + auditLargeCacheBytes = 1_000_000_000 + auditTimeout = 5 * time.Minute +) + +// auditRemedy is what fixes a finding. Argv is null when nothing here can do it +// safely (report only); argv[0] is "teploy". Dash adds its own --actor and --json. +type auditRemedy struct { + Argv []string `json:"argv"` + Destructive bool `json:"destructive"` + Confirm bool `json:"confirm"` +} + +// auditFinding is one thing left over. ID is kind:subject, stable across runs +// so a finding can be dismissed; sizes and evidence may change under it. +type auditFinding struct { + ID string `json:"id"` + Kind string `json:"kind"` + Severity string `json:"severity"` + Subject string `json:"subject"` + // App is set when the finding belongs to one app: the name a destructive + // confirmation asks for. + App string `json:"app,omitempty"` + Evidence string `json:"evidence"` + SizeBytes int64 `json:"size_bytes,omitempty"` + // Related names the findings that would free space on a disk_pressure. + Related []string `json:"related,omitempty"` + Remedy auditRemedy `json:"remedy"` +} + +// auditCheck says whether one source of evidence was read. An unknown check +// means findings depending on it are missing, not absent. +type auditCheck struct { + Name string `json:"name"` + Status string `json:"status"` + Detail string `json:"detail,omitempty"` +} + +type auditDiskUse struct { + Name string `json:"name"` + SizeBytes int64 `json:"size_bytes"` +} + +// auditDisk is the root filesystem and what takes the space. +type auditDisk struct { + Path string `json:"path"` + AvailableBytes int64 `json:"available_bytes"` + UsedPercent int `json:"used_percent"` + Attribution []auditDiskUse `json:"attribution"` +} + +// serverAuditEnvelope is `server audit --json` (contracts/schema/ +// server-audit-envelope.schema.json). +type serverAuditEnvelope struct { + MachineInterface int `json:"machine_interface"` + Server string `json:"server"` + Host string `json:"host"` + AuditedAt time.Time `json:"audited_at"` + State string `json:"state"` + Error string `json:"error,omitempty"` + Checks []auditCheck `json:"checks"` + Disk *auditDisk `json:"disk"` + Findings []auditFinding `json:"findings"` +} + +func newServerAuditCmd(flags *Flags) *cobra.Command { + return &cobra.Command{ + Use: "audit ", + Short: "Report what is left over on a server, and what fixes it (read-only)", + Long: `Reports stale and dead routes, state with no containers, versions beyond +keep_versions, build cache, dangling images and volumes, large writable layers, +large containers teploy does not manage, a Caddyfile with no Caddy, a missing +cron daemon and disk pressure, each with its evidence. + +Changes nothing. Where a command can fix a finding safely, remedy.argv is that +command; hand-written Caddy blocks, volumes and other people's containers are +report only. A source that could not be read shows as unknown in checks[], and +the findings that depend on it are missing rather than clean. With --json the +command exits 0 whenever the envelope was produced; read state.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runServerAudit(flags, args[0], cmd.OutOrStdout()) + }, + } +} + +func runServerAudit(flags *Flags, server string, out io.Writer) error { + host, user, key, err := config.ResolveServer(server, flags.Host, flags.User, flags.Key) + if err != nil { + return fmt.Errorf("resolving server %q: %w", server, err) + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + ctx, stop := context.WithTimeout(ctx, auditTimeout) + defer stop() + + now := time.Now().UTC() + var env serverAuditEnvelope + dialCtx, dialCancel := context.WithTimeout(ctx, domainsConnectTimeout) + exec, err := ssh.Connect(dialCtx, ssh.ConnectConfig{Host: host, User: user, KeyPath: key}) + dialCancel() + if err != nil { + env = serverAuditEnvelope{MachineInterface: MachineInterface, Server: server, Host: host, AuditedAt: now, + State: auditStateUnreachable, Error: strings.Join(strings.Fields(err.Error()), " "), Checks: []auditCheck{}, Findings: []auditFinding{}} + } else { + defer exec.Close() + env = auditServer(ctx, exec, server, host, now) + } + if flags.JSON { + return json.NewEncoder(out).Encode(env) + } + return printServerAudit(out, env) +} + +func printServerAudit(out io.Writer, env serverAuditEnvelope) error { + if env.State == auditStateUnreachable { + return fmt.Errorf("%s could not be audited: %s", env.Server, env.Error) + } + for _, f := range env.Findings { + fmt.Fprintf(out, "%-8s %s\n %s\n", f.Severity, f.ID, f.Evidence) + if len(f.Remedy.Argv) > 0 { + fmt.Fprintf(out, " fix: %s\n", strings.Join(f.Remedy.Argv, " ")) + } + } + if len(env.Findings) == 0 { + fmt.Fprintf(out, "No findings on %s\n", env.Server) + } + for _, c := range env.Checks { + if c.Status != auditCheckOK { + fmt.Fprintf(out, "\nUnknown, not clean: %s: %s\n", c.Name, c.Detail) + } + } + return nil +} + +// auditFacts is everything read from a server. Each *Known flag separates "read, +// and there is none" from "could not read". +type auditFacts struct { + server string + now time.Time + + apps map[string]*state.AppState // present and not tombstoned + appErrs map[string]string // an app whose state could not be read + appsKnown bool + + caddyfile string // present | absent | unknown + regions []caddy.ManagedRegion + manualSites []caddy.RouteSite + + containers []docker.Container + containersKnown bool + sizes map[string]int64 + sizesKnown bool + + resolves map[string]string + roots map[string]string + addrs []string + addrsKnown bool + + pins map[string][]string + pinErrs map[string]string + notices map[string]time.Time + noticeErrs map[string]string + df *auditDisk + usage map[string]dockerUsage + usageKnown bool + danglingIDs []string + imagesKnown bool + volumes []string + volsKnown bool + scheduler *backup.SchedulerStatus + + checks []auditCheck +} + +type dockerUsage struct { + size, reclaimable int64 +} + +func (f *auditFacts) check(name string, err error) { + if err != nil { + f.checks = append(f.checks, auditCheck{Name: name, Status: auditCheckUnknown, Detail: strings.Join(strings.Fields(err.Error()), " ")}) + return + } + f.checks = append(f.checks, auditCheck{Name: name, Status: auditCheckOK}) +} + +// auditServer runs every read on one connected server and judges the result. +func auditServer(ctx context.Context, exec ssh.Executor, server, host string, now time.Time) serverAuditEnvelope { + f := collectAuditFacts(ctx, exec, server, now) + env := serverAuditEnvelope{MachineInterface: MachineInterface, Server: server, Host: host, AuditedAt: now, + State: auditStateOK, Checks: f.checks, Disk: f.df, Findings: analyzeAudit(f)} + for _, c := range f.checks { + if c.Status != auditCheckOK { + env.State = auditStatePartial + } + } + return env +} + +func collectAuditFacts(ctx context.Context, exec ssh.Executor, server string, now time.Time) *auditFacts { + f := &auditFacts{server: server, now: now, apps: map[string]*state.AppState{}, appErrs: map[string]string{}, + sizes: map[string]int64{}, resolves: map[string]string{}, roots: map[string]string{}, + pins: map[string][]string{}, pinErrs: map[string]string{}, notices: map[string]time.Time{}, noticeErrs: map[string]string{}, + usage: map[string]dockerUsage{}} + + f.check("apps", f.readApps(ctx, exec)) + f.check("caddyfile", f.readCaddyfile(ctx, exec)) + containers, err := docker.NewClient(exec).ListAllContainers(ctx) + f.containers, f.containersKnown = containers, err == nil + f.check("containers", err) + f.check("container_sizes", f.readSizes(ctx, exec)) + f.check("upstreams", f.readUpstreams(ctx, exec)) + f.check("static_roots", f.readRoots(ctx, exec)) + f.check("addresses", f.readAddrs(ctx, exec)) + f.check("docker_usage", f.readUsage(ctx, exec)) + f.check("dangling_images", f.readDanglingImages(ctx, exec)) + f.check("dangling_volumes", f.readDanglingVolumes(ctx, exec)) + f.check("disk", f.readDisk(ctx, exec)) + st, err := backup.ProbeScheduler(ctx, exec) + if err == nil { + f.scheduler = &st + } + f.check("scheduler", err) + f.check("retention", f.readRetention(ctx, exec)) + return f +} + +func (f *auditFacts) readApps(ctx context.Context, exec ssh.Executor) error { + out, err := exec.Run(ctx, domains.AppListCommand) + if err != nil { + return err + } + var names []string + for _, n := range strings.Fields(out) { + if config.ValidateName(n) == nil { + names = append(names, n) + } + } + snaps, err := state.ReadAppSnapshots(ctx, exec, names) + if err != nil { + return err + } + f.appsKnown = true + var bad []string + for _, snap := range snaps { + switch { + case snap.StateErr != nil: + f.appErrs[snap.App] = snap.StateErr.Error() + bad = append(bad, snap.App) + case snap.State != nil && !snap.State.Tombstone: + f.apps[snap.App] = snap.State + } + } + if len(bad) > 0 { + return fmt.Errorf("could not read the state of %s", strings.Join(bad, ", ")) + } + return nil +} + +func (f *auditFacts) readCaddyfile(ctx context.Context, exec ssh.Executor) error { + f.caddyfile = "unknown" + content, present, err := caddy.NewClient(exec).ReadCaddyfile(ctx) + if err != nil { + return err + } + if !present { + f.caddyfile = "absent" + return nil + } + regions, err := caddy.ManagedRegions(content) + if err != nil { + return err + } + sites, err := caddy.InspectRoutes(content) + if err != nil { + return err + } + f.caddyfile, f.regions = "present", regions + for _, site := range sites { + if site.Owner == "" { + f.manualSites = append(f.manualSites, site) + } + } + return nil +} + +// auditSizesCommand is bounded: --size walks every writable layer. +const auditSizesCommand = `t=; command -v timeout >/dev/null 2>&1 && t='timeout 120'; $t docker ps --all --size --format '{"Names":{{json .Names}},"Size":{{json .Size}}}'` + +func (f *auditFacts) readSizes(ctx context.Context, exec ssh.Executor) error { + out, err := exec.Run(ctx, auditSizesCommand) + if err != nil { + return err + } + for _, line := range doctorLines(out) { + var row struct{ Names, Size string } + if err := json.Unmarshal([]byte(line), &row); err != nil { + return fmt.Errorf("unrecognized docker ps output: %w", err) + } + size, ok := parseDockerSize(strings.SplitN(row.Size, " (", 2)[0]) + if !ok { + return fmt.Errorf("unrecognized container size %q", row.Size) + } + f.sizes[row.Names] = size + } + f.sizesKnown = true + return nil +} + +var dockerSizeRE = regexp.MustCompile(`^([0-9.]+)\s*([A-Za-z]*)$`) + +// parseDockerSize reads docker's human sizes (powers of 1000: go-units). +func parseDockerSize(s string) (int64, bool) { + m := dockerSizeRE.FindStringSubmatch(strings.TrimSpace(s)) + if m == nil { + return 0, false + } + n, err := strconv.ParseFloat(m[1], 64) + if err != nil { + return 0, false + } + mult := map[string]float64{"": 1, "B": 1, "kB": 1e3, "KB": 1e3, "MB": 1e6, "GB": 1e9, "TB": 1e12, "PB": 1e15, + "KiB": 1 << 10, "MiB": 1 << 20, "GiB": 1 << 30, "TiB": 1 << 40}[m[2]] + if mult == 0 { + return 0, false + } + return int64(n * mult), true +} + +func (f *auditFacts) caddyRunning() bool { + return f.containersKnown && f.caddyContainerState() == "running" +} + +// caddyContainerState is the caddy container's state, "" when there is none. +func (f *auditFacts) caddyContainerState() string { + for _, c := range f.containers { + if c.Name == "caddy" { + return c.State + } + } + return "" +} + +func (f *auditFacts) readUpstreams(ctx context.Context, exec ssh.Executor) error { + wanted := map[string]bool{} + add := func(sites []caddy.RouteSite) { + for _, site := range sites { + for _, dial := range site.Upstreams { + if name, _ := domains.UpstreamHost(dial); name != "" { + wanted[name] = true + } + } + } + } + add(f.manualSites) + for _, r := range f.regions { + add(r.Sites) + } + if len(wanted) == 0 { + return nil + } + if !f.caddyRunning() { + return fmt.Errorf("no running caddy container to resolve %d upstream name(s) in", len(wanted)) + } + names := make([]string, 0, len(wanted)) + for n := range wanted { + names = append(names, n) + } + answers, err := domains.ResolveNames(ctx, exec, names) + f.resolves = answers + if err != nil { + return err + } + var missing []string + for _, n := range names { + if answers[n] == "" { + missing = append(missing, n) + } + } + if len(missing) > 0 { + sort.Strings(missing) + return fmt.Errorf("could not resolve %s", strings.Join(missing, ", ")) + } + return nil +} + +// readRoots checks the file roots of managed regions, the only thing besides an +// upstream that can make a block with no app state still serve something. +func (f *auditFacts) readRoots(ctx context.Context, exec ssh.Executor) error { + var firstErr error + for _, r := range f.regions { + for _, root := range caddy.RegionRoots(r.Text) { + if _, seen := f.roots[root]; seen { + continue + } + verdict, err := readRootExists(ctx, exec, root) + if err != nil && firstErr == nil { + firstErr = err + } + f.roots[root] = verdict + } + } + return firstErr +} + +func readRootExists(ctx context.Context, exec ssh.Executor, root string) (string, error) { + if strings.Contains(root, "{") { + return domains.ResolvesUnknown, nil // a placeholder: only Caddy can expand it + } + out, err := exec.Run(ctx, "if [ -e "+ssh.ShellQuote(root)+" ]; then echo yes; else echo no; fi") + if err != nil { + return domains.ResolvesUnknown, err + } + return strings.TrimSpace(out), nil +} + +func (f *auditFacts) readAddrs(ctx context.Context, exec ssh.Executor) error { + out, err := exec.Run(ctx, "hostname -I") + if err != nil { + return err + } + f.addrs = append(strings.Fields(out), exec.Host()) + f.addrsKnown = true + return nil +} + +func (f *auditFacts) readUsage(ctx context.Context, exec ssh.Executor) error { + out, err := exec.Run(ctx, `docker system df --format '{{json .}}'`) + if err != nil { + return err + } + for _, line := range doctorLines(out) { + var row struct{ Type, Size, Reclaimable string } + if err := json.Unmarshal([]byte(line), &row); err != nil { + return fmt.Errorf("unrecognized docker system df output: %w", err) + } + size, ok1 := parseDockerSize(row.Size) + reclaim, ok2 := parseDockerSize(strings.SplitN(row.Reclaimable, " (", 2)[0]) + if !ok1 || !ok2 { + return fmt.Errorf("unrecognized docker system df row %q", line) + } + f.usage[row.Type] = dockerUsage{size, reclaim} + } + f.usageKnown = len(f.usage) > 0 + if !f.usageKnown { + return fmt.Errorf("docker system df returned nothing") + } + return nil +} + +func (f *auditFacts) readDanglingImages(ctx context.Context, exec ssh.Executor) error { + out, err := exec.Run(ctx, "docker images -q -f dangling=true") + if err != nil { + return err + } + f.danglingIDs, f.imagesKnown = doctorLines(out), true + return nil +} + +func (f *auditFacts) readDanglingVolumes(ctx context.Context, exec ssh.Executor) error { + out, err := exec.Run(ctx, "docker volume ls -q -f dangling=true") + if err != nil { + return err + } + f.volumes, f.volsKnown = doctorLines(out), true + return nil +} + +func (f *auditFacts) readDisk(ctx context.Context, exec ssh.Executor) error { + out, err := exec.Run(ctx, "df -B1 -P /") + if err != nil { + return err + } + avail, pct, ok := parseDoctorDisk(out) + if !ok { + return fmt.Errorf("could not parse df output") + } + disk := &auditDisk{Path: "/", AvailableBytes: int64(avail), UsedPercent: pct, Attribution: []auditDiskUse{}} + for _, row := range []struct{ name, key string }{{"images", "Images"}, {"container_layers", "Containers"}, {"volumes", "Local Volumes"}, {"build_cache", "Build Cache"}} { + if u, ok := f.usage[row.key]; ok { + disk.Attribution = append(disk.Attribution, auditDiskUse{row.name, u.size}) + } + } + // Only when it matters: du walks the tree. + if diskUnderPressure(disk) { + if out, err := exec.Run(ctx, `t=; command -v timeout >/dev/null 2>&1 && t='timeout 60'; $t du -sxb /deployments 2>/dev/null`); err == nil { + if fields := strings.Fields(out); len(fields) == 2 { + if n, err := strconv.ParseInt(fields[0], 10, 64); err == nil { + disk.Attribution = append(disk.Attribution, auditDiskUse{"deployments", n}) + } + } + } + } + f.df = disk + return nil +} + +func diskUnderPressure(d *auditDisk) bool { + return d.AvailableBytes < doctorDiskWarnAvailable || d.UsedPercent >= doctorDiskWarnUsedPct +} + +// readRetention reads pins and notices for the apps with versions beyond policy. +func (f *auditFacts) readRetention(ctx context.Context, exec ssh.Executor) error { + if !f.containersKnown { + return fmt.Errorf("container list unavailable") + } + var firstErr error + for app, st := range f.apps { + keep, explicit, keepAll := appliedKeepPolicy(st) + if keepAll || st.DeploymentType == "static" { + continue + } + if len(docker.PrunableVersions(docker.ContainersByApp(f.containers, app), keep, retentionProtected(st, nil, docker.ContainersByApp(f.containers, app))...)) == 0 { + continue + } + pins, err := state.ReadPins(ctx, exec, app) + if err != nil { + f.pinErrs[app] = err.Error() + if firstErr == nil { + firstErr = err + } + continue + } + f.pins[app] = pins + if explicit { + continue + } + data, present, err := state.ReadRemoteFile(ctx, exec, deploy.PruneNoticePath(app)) + if err != nil { + f.noticeErrs[app] = err.Error() + if firstErr == nil { + firstErr = err + } + continue + } + if t, ok := deploy.ParsePruneNotice(data); present && ok { + f.notices[app] = t + } + } + return firstErr +} + +// appliedKeepPolicy reads the retention the running release was deployed with. +// A manifest with no keep_versions, or the 0 an unset field records, is the +// default: an explicit 0 records keep_all_versions. +func appliedKeepPolicy(st *state.AppState) (keep int, explicit, keepAll bool) { + var m struct { + Container struct { + KeepVersions int `json:"keep_versions"` + KeepAll bool `json:"keep_all_versions"` + } `json:"container"` + } + if len(st.AppliedManifest) > 0 { + _ = json.Unmarshal(st.AppliedManifest, &m) + } + switch { + case m.Container.KeepAll: + return 0, true, true + case m.Container.KeepVersions > 0: + return m.Container.KeepVersions, true, false + } + return config.DefaultKeepVersions, false, false +} + +// retentionProtected is what no prune may touch: current, previous, pinned, and +// any version with a running container. +func retentionProtected(st *state.AppState, pins []string, containers []docker.Container) []string { + protected := append([]string{st.CurrentHash, st.PreviousHash}, pins...) + for _, c := range containers { + if v := c.Labels["teploy.version"]; v != "" && c.State == "running" && !slices.Contains(protected, v) { + protected = append(protected, v) + } + } + return protected +} + +// ownedContainers are the containers that make an app's name still in use: its +// own, its previews', and unlabelled ones named after it (older deploys). +func ownedContainers(containers []docker.Container, app string) []docker.Container { + var owned []docker.Container + for _, c := range containers { + label := c.Labels["teploy.app"] + switch { + case label == app, strings.HasPrefix(label, app+"-preview-"): + owned = append(owned, c) + case label == "" && strings.HasPrefix(c.Name, app+"-"): + owned = append(owned, c) + } + } + return owned +} + +// routeFacts is what deciding on one managed route needs: the same judgement for +// the audit and for `route remove`, so the remedy offered is the one that runs. +type routeFacts struct { + app string + region string + sites []caddy.RouteSite + stateKnown bool + st *state.AppState + known bool // containers were listed + containers []docker.Container + resolves map[string]string + roots map[string]string +} + +// removalRefusal says why a route must not be removed, "" when it may be. Only +// what is proven dead can go: an unreadable source refuses like a live app does. +func removalRefusal(r routeFacts) string { + if !r.stateKnown { + return "its app state could not be read" + } + if !r.known { + return "its containers could not be listed" + } + if caddy.RegionHasWebhook(r.region) { + return "it carries the autodeploy webhook route; the next deploy retires it" + } + if r.st != nil && routedByCaddy(r.st) { + if r.st.DeploymentType == "static" { + return "it belongs to a live static app" + } + for _, c := range ownedContainers(r.containers, r.app) { + if c.Labels["teploy.role"] != "accessory" { + return fmt.Sprintf("it belongs to a live app: %s has container %s (%s)", r.app, c.Name, c.State) + } + } + } + if r.st == nil { + for _, c := range ownedContainers(r.containers, r.app) { + return fmt.Sprintf("container %s (%s) still carries this app's name", c.Name, c.State) + } + } + for _, site := range r.sites { + for _, dial := range site.Upstreams { + switch dialVerdict(dial, r.resolves) { + case domains.ResolvesNo: + case domains.ResolvesYes: + return "upstream " + dial + " still resolves" + default: + return "upstream " + dial + " could not be proven dead" + } + } + } + for _, root := range caddy.RegionRoots(r.region) { + switch r.roots[root] { + case "no": + case "yes": + return "it serves files from " + root + ", which exists" + default: + return "whether " + root + " exists could not be read" + } + } + return "" +} + +func routedByCaddy(st *state.AppState) bool { + return st.IngressMode == "" || st.IngressMode == config.IngressCaddy +} + +func dialVerdict(dial string, resolves map[string]string) string { + name, verdict := domains.UpstreamHost(dial) + if name == "" { + return verdict + } + if v := resolves[name]; v != "" { + return v + } + return domains.ResolvesUnknown +} + +func dialsByVerdict(sites []caddy.RouteSite, resolves map[string]string, verdict string) []string { + var dials []string + for _, site := range sites { + for _, dial := range site.Upstreams { + if dialVerdict(dial, resolves) == verdict && !slices.Contains(dials, dial) { + dials = append(dials, dial) + } + } + } + return dials +} + +func siteHosts(sites []caddy.RouteSite) []string { + var hosts []string + for _, site := range sites { + hosts = append(hosts, site.Hosts...) + } + return hosts +} + +func analyzeAudit(f *auditFacts) []auditFinding { + findings := []auditFinding{} + add := func(kind, severity, subject, app, evidence string, size int64, remedy auditRemedy) { + findings = append(findings, auditFinding{ID: kind + ":" + subject, Kind: kind, Severity: severity, Subject: subject, + App: app, Evidence: evidence, SizeBytes: size, Remedy: remedy}) + } + teploy := func(args ...string) []string { return append([]string{"teploy"}, args...) } + + // Managed routes. An app whose state could not be read is not judged. + if f.appsKnown { + for _, r := range f.regions { + if _, bad := f.appErrs[r.App]; bad { + continue + } + st := f.apps[r.App] + facts := routeFacts{app: r.App, region: r.Text, sites: r.Sites, stateKnown: true, st: st, + known: f.containersKnown, containers: f.containers, resolves: f.resolves, roots: f.roots} + refusal := removalRefusal(facts) + removeRemedy := auditRemedy{} + note := "" + if refusal == "" { + removeRemedy.Argv = teploy("route", "remove", "--app", r.App, "--host", f.server) + } else { + note = "; not removable: " + refusal + } + hosts := strings.Join(siteHosts(r.Sites), ", ") + switch { + case len(r.Sites) == 0: + add(findingEmptyRegion, severityInfo, r.App, r.App, "the managed region holds no site block"+note, 0, removeRemedy) + case st == nil: + dead := dialsByVerdict(r.Sites, f.resolves, domains.ResolvesNo) + live := dialsByVerdict(r.Sites, f.resolves, domains.ResolvesYes) + // A live preview has a route and no app state of its own. + if strings.Contains(r.App, "-preview-") && len(live) > 0 && len(dead) == 0 { + continue + } + evidence := fmt.Sprintf("no app %s on this server, but its managed block serves %s", r.App, hosts) + if len(dead) > 0 { + evidence += " and routes to " + strings.Join(dead, ", ") + ", which does not resolve" + } + add(findingStaleRoute, severityWarning, r.App, r.App, evidence+note, 0, removeRemedy) + case !routedByCaddy(st): + add(findingStaleRoute, severityWarning, r.App, r.App, + fmt.Sprintf("%s is deployed with ingress %s, but a managed block still serves %s%s", r.App, st.IngressMode, hosts, note), 0, removeRemedy) + default: + dead := dialsByVerdict(r.Sites, f.resolves, domains.ResolvesNo) + if len(dead) == 0 { + continue + } + severity, remedy, evidence := deadRouteVerdict(f, r, st, dead, hosts) + add(findingDeadUpstream, severity, r.App, r.App, evidence, 0, remedy) + } + } + } + + // Hand-written blocks are never edited: report only. + for _, site := range f.manualSites { + subject := strings.Join(site.Hosts, ",") + if dead := dialsByVerdict([]caddy.RouteSite{site}, f.resolves, domains.ResolvesNo); len(dead) > 0 { + add(findingManualDead, severityWarning, subject, "", fmt.Sprintf("hand-written block for %s routes to %s, which does not resolve", subject, strings.Join(dead, ", ")), 0, auditRemedy{}) + } + if !f.addrsKnown { + continue + } + for _, host := range site.Hosts { + if ip := net.ParseIP(host); ip != nil && !ip.IsLoopback() && !slices.Contains(f.addrs, ip.String()) { + add(findingManualNonLocal, severityInfo, subject, "", fmt.Sprintf("hand-written block is addressed to %s, which is not an address of this server", host), 0, auditRemedy{}) + break + } + } + } + + if f.caddyfile == "present" && f.containersKnown && f.caddyContainerState() != "running" { + how := "no caddy container" + if s := f.caddyContainerState(); s != "" { + how = "the caddy container is " + s + } + severity := severityInfo + routes := len(f.regions) + len(f.manualSites) + if routes > 0 { + severity = severityWarning + } + add(findingCaddyfileNoCaddy, severity, "caddy", "", fmt.Sprintf("/deployments/caddy/Caddyfile exists with %d route block(s), but %s", routes, how), 0, auditRemedy{}) + } + + // State with no containers. + if f.appsKnown && f.containersKnown { + for _, app := range sortedAppNames(f.apps) { + st := f.apps[app] + if st.DeploymentType == "static" || len(ownedContainers(f.containers, app)) > 0 { + continue + } + add(findingOrphanState, severityWarning, app, app, + fmt.Sprintf("%s has deploy state (release %s) but no containers; removing keeps its volumes, accessory data and credentials", app, st.CurrentHash), 0, + auditRemedy{Argv: teploy("remove", "--app", app, "--host", f.server, "-y"), Destructive: true, Confirm: true}) + } + } + + // Versions beyond policy. + if f.appsKnown && f.containersKnown { + for _, app := range sortedAppNames(f.apps) { + st := f.apps[app] + keep, explicit, keepAll := appliedKeepPolicy(st) + if keepAll || st.DeploymentType == "static" { + continue + } + if _, bad := f.pinErrs[app]; bad { + continue + } + owned := docker.ContainersByApp(f.containers, app) + prunable := docker.PrunableVersions(owned, keep, retentionProtected(st, f.pins[app], owned)...) + if len(prunable) == 0 { + continue + } + var size int64 + for _, c := range owned { + if slices.Contains(prunable, c.Labels["teploy.version"]) { + size += f.sizes[c.Name] + } + } + evidence := fmt.Sprintf("%d version(s) beyond keep_versions %d: %s", len(prunable), keep, strings.Join(prunable, ", ")) + severity := severityInfo + if explicit { + evidence += "; the next deploy removes them" + } else { + severity = severityWarning + evidence += "; keep_versions is not set, so " + defaultPruneWarning(f, app) + } + add(findingVersionsBeyond, severity, app, app, evidence, size, + auditRemedy{Argv: teploy("releases", "prune", "--app", app, "--host", f.server, "--keep", strconv.Itoa(keep)), Destructive: true, Confirm: true}) + } + } + + // Build cache and dangling images: rebuildable, so no confirmation. + if f.usageKnown || f.imagesKnown { + cache := f.usage["Build Cache"].reclaimable + if cache >= auditLargeCacheBytes || len(f.danglingIDs) > 0 { + evidence := fmt.Sprintf("%d dangling image(s)", len(f.danglingIDs)) + if f.usageKnown { + evidence = fmt.Sprintf("%s of unused build cache and %s", humanBytes(cache), evidence) + } else { + evidence += " (the build cache size could not be read)" + } + add(findingBuildArtifacts, severityInfo, "docker", "", evidence, cache, + auditRemedy{Argv: teploy("server", "prune", "--host", f.server, "--build-cache", "--dangling")}) + } + } + + // Large containers: never touched, whoever owns them. + if f.sizesKnown && f.containersKnown { + for _, c := range f.containers { + size := f.sizes[c.Name] + managed := c.Labels["teploy.app"] != "" || c.Name == "caddy" || belongsToKnownApp(c.Name, f.apps) + switch { + case managed && size >= auditLargeLayerBytes: + add(findingLargeLayer, severityWarning, c.Name, c.Labels["teploy.app"], + fmt.Sprintf("%s (%s) has written %s to its container layer, which a redeploy discards: move that data to a volume", c.Name, c.State, humanBytes(size)), size, auditRemedy{}) + case !managed && size >= auditLargeUnmanagedBytes: + add(findingUnmanagedLarge, severityWarning, c.Name, "", + fmt.Sprintf("%s (%s) is not managed by teploy and holds %s in its container layer", c.Name, c.State, humanBytes(size)), size, auditRemedy{}) + } + } + } + + // Volumes may hold the only copy of something: there is never a remedy. + if f.volsKnown && len(f.volumes) > 0 { + add(findingDanglingVolumes, severityInfo, "docker", "", + fmt.Sprintf("%d volume(s) attached to no container, %s reclaimable: %s", len(f.volumes), humanBytes(f.usage["Local Volumes"].reclaimable), doctorSample(f.volumes)), + f.usage["Local Volumes"].reclaimable, auditRemedy{}) + } + + if f.scheduler != nil && !f.scheduler.Daemon && f.scheduler.HasJobs() { + severity := severityWarning + if f.scheduler.HasTeployJobs() { + severity = severityCritical + } + add(findingSchedulerMissing, severity, "cron", "", "jobs are scheduled but no cron daemon is running, so none of them run: "+backup.SchedulerRemedy(), 0, auditRemedy{}) + } + + if f.df != nil && diskUnderPressure(f.df) { + severity := severityWarning + if f.df.AvailableBytes < doctorDiskFailAvailable { + severity = severityCritical + } + uses := make([]string, 0, len(f.df.Attribution)) + attribution := append([]auditDiskUse{}, f.df.Attribution...) + sort.Slice(attribution, func(i, j int) bool { return attribution[i].SizeBytes > attribution[j].SizeBytes }) + for _, u := range attribution { + uses = append(uses, u.Name+" "+humanBytes(u.SizeBytes)) + } + var related []string + for _, x := range findings { + switch x.Kind { + case findingVersionsBeyond, findingBuildArtifacts, findingLargeLayer, findingUnmanagedLarge, findingDanglingVolumes: + related = append(related, x.ID) + } + } + sort.Strings(related) + findings = append(findings, auditFinding{ID: findingDiskPressure + ":/", Kind: findingDiskPressure, Severity: severity, Subject: "/", + Evidence: fmt.Sprintf("%s available on / (%d%% used); %s", humanBytes(f.df.AvailableBytes), f.df.UsedPercent, strings.Join(uses, ", ")), Related: related}) + } + + rank := map[string]int{severityCritical: 0, severityWarning: 1, severityInfo: 2} + sort.SliceStable(findings, func(i, j int) bool { + if rank[findings[i].Severity] != rank[findings[j].Severity] { + return rank[findings[i].Severity] < rank[findings[j].Severity] + } + return findings[i].ID < findings[j].ID + }) + return findings +} + +// deadRouteVerdict words a caddy-routed app's dead upstream. Redeploy is offered +// only when it can help: a static app has no containers to bring back, and an +// upstream named after another app (a caddy_extra proxy) stays dead through this +// app's redeploy. +func deadRouteVerdict(f *auditFacts, r caddy.ManagedRegion, st *state.AppState, dead []string, hosts string) (string, auditRemedy, string) { + live := len(dialsByVerdict(r.Sites, f.resolves, domains.ResolvesYes)) > 0 + evidence := fmt.Sprintf("%s routes to %s, which does not resolve in Caddy's network", hosts, strings.Join(dead, ", ")) + ownDead := false + for _, dial := range dead { + if name, _ := domains.UpstreamHost(dial); strings.HasPrefix(name, r.App+"-") { + ownDead = true + } + } + severity := severityCritical + if live || st.DeploymentType == "static" { + severity = severityWarning // part of the site still serves + } + switch { + case st.DeploymentType == "static": + return severity, auditRemedy{}, evidence + fmt.Sprintf("; %s itself serves files, so only those paths fail. A redeploy cannot help: fix the app's caddy_extra or deploy what it proxies to", r.App) + case !ownDead: + return severity, auditRemedy{}, evidence + "; that upstream is not this app's, so a redeploy cannot help: fix the app's caddy_extra or deploy what it proxies to" + } + return severity, auditRemedy{Argv: []string{"teploy", "redeploy", "--app", r.App, "--host", f.server}, Confirm: true}, evidence +} + +func defaultPruneWarning(f *auditFacts, app string) string { + if detail, bad := f.noticeErrs[app]; bad { + return "when automatic pruning starts could not be read (" + detail + ")" + } + if t, ok := f.notices[app]; ok { + return "deploys after " + t.Add(deploy.DefaultPruneNotice).Format("2006-01-02") + " remove them automatically; set keep_versions (0 keeps everything) or pin what you need" + } + return "the next deploy starts a 7-day notice, then deploys remove them automatically; set keep_versions (0 keeps everything) or pin what you need" +} + +func belongsToKnownApp(name string, apps map[string]*state.AppState) bool { + for app := range apps { + if strings.HasPrefix(name, app+"-") { + return true + } + } + return false +} + +func sortedAppNames(apps map[string]*state.AppState) []string { + names := make([]string, 0, len(apps)) + for n := range apps { + names = append(names, n) + } + sort.Strings(names) + return names +} + +func humanBytes(n int64) string { + switch { + case n >= 1e12: + return fmt.Sprintf("%.1f TB", float64(n)/1e12) + case n >= 1e9: + return fmt.Sprintf("%.1f GB", float64(n)/1e9) + case n >= 1e6: + return fmt.Sprintf("%.1f MB", float64(n)/1e6) + } + return fmt.Sprintf("%d B", n) +} diff --git a/internal/cli/server_audit_test.go b/internal/cli/server_audit_test.go new file mode 100644 index 0000000..5663f09 --- /dev/null +++ b/internal/cli/server_audit_test.go @@ -0,0 +1,846 @@ +package cli + +// `teploy server audit` tests. Every fixture models one server through the +// ssh MockExecutor the way domains_test.go does: the responses are synthetic, +// every parse and judgement stage is the real one. The read-only pin at the +// bottom is the audit's safety property: no command it issues may mutate. + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/ssh" +) + +var auditFixtureNow = time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC) + +// auditFixture builds the mock executor for one audited server. Zero values +// mean "the read succeeds and reports nothing" wherever that is a shape the +// real server can produce (no dangling images, no dangling volumes); a read +// that must FAIL is registered explicitly through extra. +type auditFixture struct { + server, host string + now time.Time + + apps []string // /deployments/*/state.json listing + state map[string]string // app -> state JSON; "!" means unreadable + badRC map[string]bool // apps whose state read fails (S rc=1) + + caddyfile string // "" = no Caddyfile on the server + + containers string // docker ps --all --format output + sizes string // docker ps --all --size output + resolves string // getent answers inside caddy, "rc name" lines + roots map[string]string + addrs string // hostname -I + usage string // docker system df rows + danglingIm string + danglingVo string + df string + du string + scheduler string + + files map[string][]byte + extra []ssh.MockCommand // registered first: explicit wins +} + +func auditFrames(fx auditFixture) string { + var b strings.Builder + for _, app := range fx.apps { + st := fx.state[app] + rc := "0" + if fx.badRC[app] { + rc = "1" + } + fmt.Fprintf(&b, "A %s\nS %s %s\nG absent\nL \nM 0\nR 0\n", app, rc, + base64.StdEncoding.EncodeToString([]byte(st))) + } + return b.String() +} + +func (fx auditFixture) exec() *ssh.MockExecutor { + cmds := append([]ssh.MockCommand{}, fx.extra...) + if fx.apps != nil { + cmds = append(cmds, ssh.MockCommand{Match: "for f in /deployments/*/state.json", Output: strings.Join(fx.apps, "\n") + "\n"}) + cmds = append(cmds, ssh.MockCommand{Match: "for app in", Output: auditFrames(fx)}) + } + // Registered before the plain `docker ps --all` listing so the longer + // (size-carrying) command is matched first; an empty output means "the + // read succeeded and reported nothing". + cmds = append(cmds, ssh.MockCommand{Match: "t=; command -v timeout >/dev/null 2>&1 && t='timeout 120'; $t docker ps --all --size", Output: fx.sizes}) + if fx.containers != "" { + cmds = append(cmds, ssh.MockCommand{Match: "docker ps --all ", Output: fx.containers}) + } + if fx.resolves != "" { + cmds = append(cmds, ssh.MockCommand{Match: "printf '%s\\n'", Output: fx.resolves}) + } + for root, verdict := range fx.roots { + cmds = append(cmds, ssh.MockCommand{Match: "if [ -e " + ssh.ShellQuote(root) + " ]", Output: verdict}) + } + if fx.addrs != "" { + cmds = append(cmds, ssh.MockCommand{Match: "hostname -I", Output: fx.addrs}) + } + if fx.usage != "" { + cmds = append(cmds, ssh.MockCommand{Match: "docker system df", Output: fx.usage}) + } + cmds = append(cmds, ssh.MockCommand{Match: "docker images -q -f dangling=true", Output: fx.danglingIm}) + cmds = append(cmds, ssh.MockCommand{Match: "docker volume ls -q -f dangling=true", Output: fx.danglingVo}) + if fx.df != "" { + cmds = append(cmds, ssh.MockCommand{Match: "df -B1 -P /", Output: fx.df}) + } + if fx.du != "" { + cmds = append(cmds, ssh.MockCommand{Match: "t=; command -v timeout >/dev/null 2>&1 && t='timeout 60'; $t du -sxb", Output: fx.du}) + } + if fx.scheduler != "" { + cmds = append(cmds, ssh.MockCommand{Match: "sched_daemon=no; ", Output: fx.scheduler}) + } + m := ssh.NewMockExecutor(fx.host, cmds...) + for p, data := range fx.files { + m.Files[p] = data + } + if fx.caddyfile != "" { + m.Files["/deployments/caddy/Caddyfile"] = []byte(fx.caddyfile) + } + return m +} + +func (fx auditFixture) audit(t *testing.T) serverAuditEnvelope { + t.Helper() + if fx.server == "" { + fx.server = "edge" + } + if fx.host == "" { + fx.host = "192.0.2.10" + } + if fx.now.IsZero() { + fx.now = auditFixtureNow + } + return auditServer(context.Background(), fx.exec(), fx.server, fx.host, fx.now) +} + +// auditStateJSON builds a state.json the canonical parser accepts. +func auditStateJSON(deployment, ingress, current, previous string, manifest map[string]any) string { + m := map[string]any{ + "schema_version": 2, + "deployment_type": deployment, + "ingress_mode": ingress, + "updated_at": "2026-11-01T00:00:00Z", + "current_hash": current, + "generation": 2, + } + if previous != "" { + m["previous_hash"] = previous + } + if manifest != nil { + raw, _ := json.Marshal(manifest) + m["applied_manifest"] = json.RawMessage(raw) + } + b, _ := json.Marshal(m) + return string(b) +} + +// auditContainer renders one docker ps --format row. +func auditContainer(name, image, state, createdAt string, labels map[string]string) string { + c := struct { + ID string `json:"ID"` + Names string `json:"Names"` + Image string `json:"Image"` + State string `json:"State"` + Status string `json:"Status"` + CreatedAt string `json:"CreatedAt"` + Labels map[string]string `json:"Labels"` + }{"id" + name, name, image, state, "Up", createdAt, labels} + b, _ := json.Marshal(c) + return string(b) +} + +func auditSizeRow(name, size string) string { + return fmt.Sprintf(`{"Names":%q,"Size":%q}`, name, size) +} + +func findingsByKind(env serverAuditEnvelope, kind string) []auditFinding { + var out []auditFinding + for _, f := range env.Findings { + if f.Kind == kind { + out = append(out, f) + } + } + return out +} + +func oneFinding(t *testing.T, env serverAuditEnvelope, kind string) auditFinding { + t.Helper() + got := findingsByKind(env, kind) + if len(got) != 1 { + t.Fatalf("kind %s: expected exactly 1 finding, got %d: %+v", kind, len(got), env.Findings) + } + return got[0] +} + +// kitchenSink is a server with one of everything: every finding kind, a pin, +// a prune notice, disk pressure (so the du read fires) and an app whose state +// cannot be read (so a check reads unknown). The expected findings are +// asserted piecewise by the tests below and pinned as the contracts fixture. +func auditKitchenSink() auditFixture { + ghostBlock := "# TEPLOY BEGIN ghost\nghost.example.com {\n\treverse_proxy ghost-web-v1:3000\n}\n# TEPLOY END ghost\n" + hollowBlock := "# TEPLOY BEGIN hollow\n# TEPLOY END hollow\n" + blogBlock := "# TEPLOY BEGIN blog\nblog.example.com {\n\treverse_proxy blog-web-v3:3000\n}\n# TEPLOY END blog\n" + filesBlock := "# TEPLOY BEGIN files\nfiles.example.com {\n\troot * /deployments/files/site\n\tfile_server\n}\n# TEPLOY END files\n" + preview := "# TEPLOY BEGIN blog-preview-p-08e81639\npreview.example.com {\n\treverse_proxy blog-preview-p-08e81639-abc1234:3000\n}\n# TEPLOY END blog-preview-p-08e81639\n" + manual := "hand.example.com {\n\treverse_proxy gone-svc:80\n}\n\nhttp://203.0.113.9 {\n\treverse_proxy 127.0.0.1:9000\n}\n" + + keep := func(n int) map[string]any { + return map[string]any{"container": map[string]any{"keep_versions": n}} + } + manyV := func(v, when string, running bool) string { + state := "exited" + if running { + state = "running" + } + return auditContainer("many-web-"+v, "many:"+v, state, when, + map[string]string{"teploy.app": "many", "teploy.process": "web", "teploy.version": v}) + } + unsetV := func(v, when string, running bool) string { + state := "exited" + if running { + state = "running" + } + return auditContainer("unset-web-"+v, "unset:"+v, state, when, + map[string]string{"teploy.app": "unset", "teploy.process": "web", "teploy.version": v}) + } + containers := []string{ + auditContainer("caddy", "caddy:2", "running", "2026-05-01 00:00:00 +0000 UTC", nil), + auditContainer("blog-web-v2", "blog:v2", "running", "2026-11-04 10:00:00 +0000 UTC", + map[string]string{"teploy.app": "blog", "teploy.process": "web", "teploy.version": "v2"}), + manyV("v4", "2026-11-04 09:00:00 +0000 UTC", true), + manyV("v3", "2026-11-03 09:00:00 +0000 UTC", false), + manyV("v2", "2026-11-02 09:00:00 +0000 UTC", false), + manyV("v1", "2026-11-01 09:00:00 +0000 UTC", false), + unsetV("u7", "2026-11-04 08:00:00 +0000 UTC", true), + unsetV("u6", "2026-11-03 08:00:00 +0000 UTC", false), + unsetV("u5", "2026-11-02 08:00:00 +0000 UTC", false), + unsetV("u4", "2026-11-01 08:00:00 +0000 UTC", false), + unsetV("u3", "2026-10-31 08:00:00 +0000 UTC", false), + unsetV("u2", "2026-10-30 08:00:00 +0000 UTC", false), + unsetV("u1", "2026-10-29 08:00:00 +0000 UTC", false), + auditContainer("reacher-web-1f2e3d", "reacher:1", "running", "2026-08-01 00:00:00 +0000 UTC", + map[string]string{"teploy.app": "reacher", "teploy.process": "web", "teploy.version": "1f2e3d"}), + auditContainer("portainer", "portainer/portainer:2", "running", "2026-05-02 00:00:00 +0000 UTC", nil), + } + sizes := []string{ + auditSizeRow("caddy", "100MB (virtual 200MB)"), + auditSizeRow("blog-web-v2", "50MB (virtual 1GB)"), + auditSizeRow("many-web-v4", "10MB (virtual 1GB)"), + auditSizeRow("many-web-v3", "10MB (virtual 1GB)"), + auditSizeRow("many-web-v2", "10MB (virtual 1GB)"), + auditSizeRow("many-web-v1", "10MB (virtual 1GB)"), + auditSizeRow("unset-web-u7", "10MB (virtual 1GB)"), + auditSizeRow("unset-web-u6", "10MB (virtual 1GB)"), + auditSizeRow("unset-web-u5", "10MB (virtual 1GB)"), + auditSizeRow("unset-web-u4", "10MB (virtual 1GB)"), + auditSizeRow("unset-web-u3", "10MB (virtual 1GB)"), + auditSizeRow("unset-web-u2", "10MB (virtual 1GB)"), + auditSizeRow("unset-web-u1", "10MB (virtual 1GB)"), + auditSizeRow("reacher-web-1f2e3d", "35.7GB (virtual 1.1GB)"), + auditSizeRow("portainer", "2GB (virtual 500MB)"), + } + volumes := make([]string, 21) + for i := range volumes { + volumes[i] = fmt.Sprintf("%031dxxxxxxxdeadbeef", i) + } + return auditFixture{ + apps: []string{"blog", "many", "unset", "orphan", "static-site", "broken"}, + state: map[string]string{ + "blog": auditStateJSON("container", "caddy", "v3", "v2", keep(3)), + "many": auditStateJSON("container", "caddy", "v4", "v3", keep(2)), + "unset": auditStateJSON("container", "caddy", "u7", "u6", map[string]any{"container": map[string]any{"keep_versions": 0}}), + "orphan": auditStateJSON("container", "caddy", "o1", "", nil), + "static-site": auditStateJSON("static", "caddy", "s1", "", nil), + "broken": "", + }, + badRC: map[string]bool{"broken": true}, + caddyfile: ghostBlock + "\n" + hollowBlock + "\n" + blogBlock + "\n" + filesBlock + "\n" + preview + "\n" + manual, + containers: strings.Join(containers, "\n"), + sizes: strings.Join(sizes, "\n"), + resolves: "0 blog-preview-p-08e81639-abc1234\n" + + "2 ghost-web-v1\n" + + "2 blog-web-v3\n" + + "2 gone-svc\n", + roots: map[string]string{"/deployments/files/site": "yes"}, + addrs: "192.0.2.10 172.17.0.1\n", + usage: `{"Type":"Images","Size":"20GB","Reclaimable":"1GB (5%)"}` + "\n" + `{"Type":"Containers","Size":"5GB","Reclaimable":"0B (0%)"}` + "\n" + `{"Type":"Local Volumes","Size":"4GB","Reclaimable":"3.15GB (78%)"}` + "\n" + `{"Type":"Build Cache","Size":"2GB","Reclaimable":"2GB (100%)"}`, + danglingIm: "sha256:1111111111111111111111111111111111111111111111111111111111111111\nsha256:2222222222222222222222222222222222222222222222222222222222222222\nsha256:3333333333333333333333333333333333333333333333333333333333333333\n", + danglingVo: strings.Join(volumes, "\n") + "\n", + df: "Filesystem 1-blocks Used Available Capacity Mounted on\n/dev/vda1 10000000000 9600000000 800000000 96% /\n", + du: "53687091200\t/deployments\n", + scheduler: "daemon=no\nenabled=disabled\ncrontab=yes\ncrontab_entries=2\nteploy_entries=1\n", + files: map[string][]byte{ + "/deployments/many/pinned": []byte("v1\n"), + "/deployments/unset/.prune-notice": []byte("2026-11-01T00:00:00Z\n"), + }, + } +} + +func auditCheckStatus(env serverAuditEnvelope, name string) (string, bool) { + for _, c := range env.Checks { + if c.Name == name { + return c.Status, true + } + } + return "", false +} + +func TestServerAudit_KitchenSinkFindings(t *testing.T) { + env := auditKitchenSink().audit(t) + + if env.State != auditStatePartial { + t.Fatalf("state = %s, want partial (one app state unreadable)", env.State) + } + if st, ok := auditCheckStatus(env, "apps"); !ok || st != auditCheckUnknown { + t.Errorf("apps check = %q %v, want unknown: %+v", st, ok, env.Checks) + } + + var kinds []string + for _, f := range env.Findings { + kinds = append(kinds, f.Kind) + } + got := strings.Join(kinds, ",") + expected := []string{ + "dangling_volumes", + "disk_pressure", + "empty_managed_region", + "large_writable_layer", + "managed_route_dead_upstream", + "manual_route_dead_upstream", + "manual_route_nonlocal_address", + "orphan_state", + "reclaimable_build_artifacts", + "scheduler_missing", + "stale_route", + "unmanaged_container", + "versions_beyond_policy", + } + for _, k := range expected { + if strings.Count(got, k) == 0 { + t.Errorf("expected at least one %s finding; got: %s", k, got) + } + } + if strings.Count(got, "versions_beyond_policy") != 2 { + t.Errorf("expected two versions_beyond_policy findings (explicit + default), got: %s", got) + } + if strings.Count(got, "stale_route") != 2 { + t.Errorf("expected two stale_route findings (ghost + files), got: %s", got) + } + // Findings are ordered by severity then id; a critical must not sort after an info. + rank := map[string]int{severityCritical: 0, severityWarning: 1, severityInfo: 2} + for i := 1; i < len(env.Findings); i++ { + a, b := env.Findings[i-1], env.Findings[i] + if rank[a.Severity] > rank[b.Severity] { + t.Errorf("findings not sorted by severity: %s after %s", b.ID, a.ID) + } + } +} + +func TestServerAudit_StaleRouteAndEmptyRegion(t *testing.T) { + env := auditKitchenSink().audit(t) + + var ghost, files *auditFinding + for _, f := range findingsByKind(env, findingStaleRoute) { + switch f.Subject { + case "ghost": + ghost = &f + case "files": + files = &f + } + } + if ghost == nil || files == nil { + t.Fatalf("missing stale_route findings: %+v", env.Findings) + } + if ghost.App != "ghost" || !strings.Contains(ghost.Evidence, "no app ghost") { + t.Errorf("ghost finding: %+v", ghost) + } + wantArgv := []string{"teploy", "route", "remove", "--app", "ghost", "--host", "edge"} + if fmt.Sprint(ghost.Remedy.Argv) != fmt.Sprint(wantArgv) { + t.Errorf("ghost remedy argv = %v, want %v", ghost.Remedy.Argv, wantArgv) + } + if ghost.Remedy.Destructive || ghost.Remedy.Confirm { + t.Errorf("a proven-dead route needs no confirmation: %+v", ghost.Remedy) + } + + // The files app's block still serves files that exist: reported, remedy null. + if files.Remedy.Argv != nil { + t.Errorf("a route still serving files must be report-only, got argv %v", files.Remedy.Argv) + } + if !strings.Contains(files.Evidence, "not removable") || !strings.Contains(files.Evidence, "/deployments/files/site") { + t.Errorf("files evidence must name the refusal: %q", files.Evidence) + } + + hollow := oneFinding(t, env, findingEmptyRegion) + if hollow.Subject != "hollow" || hollow.Remedy.Argv == nil { + t.Errorf("empty region should be removable: %+v", hollow) + } +} + +func TestServerAudit_ManagedDeadUpstreamOffersRedeploy(t *testing.T) { + env := auditKitchenSink().audit(t) + f := oneFinding(t, env, findingDeadUpstream) + if f.Subject != "blog" { + t.Fatalf("dead upstream subject = %s, want blog", f.Subject) + } + if f.Severity != severityCritical { + t.Errorf("a fully dead route is critical, got %s", f.Severity) + } + if fmt.Sprint(f.Remedy.Argv) != fmt.Sprint([]string{"teploy", "redeploy", "--app", "blog", "--host", "edge"}) { + t.Errorf("remedy argv = %v", f.Remedy.Argv) + } + if !f.Remedy.Confirm || f.Remedy.Destructive { + t.Errorf("redeploy needs confirmation but is not destructive: %+v", f.Remedy) + } + if !strings.Contains(f.Evidence, "blog-web-v3:3000") { + t.Errorf("evidence must name the dead upstream: %q", f.Evidence) + } + + // A dead upstream that is NOT this app's (caddy_extra proxy to another + // app) cannot be fixed by this app's redeploy: report only. + fx := auditKitchenSink() + fx.caddyfile = "# TEPLOY BEGIN blog\nblog.example.com {\n\treverse_proxy other-svc:3000\n}\n# TEPLOY END blog\n" + fx.resolves = "2 other-svc\n" + f = oneFinding(t, fx.audit(t), findingDeadUpstream) + if f.Remedy.Argv != nil { + t.Errorf("someone else's dead upstream must be report-only: %+v", f.Remedy) + } + if !strings.Contains(f.Evidence, "redeploy cannot help") { + t.Errorf("evidence should say a redeploy cannot help: %q", f.Evidence) + } +} + +func TestServerAudit_OrphanState(t *testing.T) { + env := auditKitchenSink().audit(t) + f := oneFinding(t, env, findingOrphanState) + if f.Subject != "orphan" { + t.Fatalf("orphan subject = %s", f.Subject) + } + if fmt.Sprint(f.Remedy.Argv) != fmt.Sprint([]string{"teploy", "remove", "--app", "orphan", "--host", "edge", "-y"}) { + t.Errorf("remedy argv = %v", f.Remedy.Argv) + } + if !f.Remedy.Destructive || !f.Remedy.Confirm { + t.Errorf("removing state is destructive and needs confirmation: %+v", f.Remedy) + } + + // A static app with no containers is normal, not an orphan. + for _, f := range env.Findings { + if f.Kind == findingOrphanState && f.Subject == "static-site" { + t.Errorf("static app reported as orphan: %+v", f) + } + } +} + +func TestServerAudit_VersionsBeyondPolicy(t *testing.T) { + env := auditKitchenSink().audit(t) + var many, unset *auditFinding + for _, f := range findingsByKind(env, findingVersionsBeyond) { + switch f.Subject { + case "many": + many = &f + case "unset": + unset = &f + } + } + if many == nil || unset == nil { + t.Fatalf("missing versions findings: %+v", env.Findings) + } + + // many: keep 2, current v4 + previous v3 protected, v1 pinned → only v2. + if !strings.Contains(many.Evidence, "beyond keep_versions 2: v2") { + t.Errorf("many evidence = %q", many.Evidence) + } + if strings.Contains(many.Evidence, "v1") { + t.Errorf("pinned v1 must not be listed as beyond policy: %q", many.Evidence) + } + if many.Severity != severityInfo || !strings.Contains(many.Evidence, "the next deploy removes them") { + t.Errorf("an explicit keep_versions is housekeeping, not a warning: %+v", many) + } + if fmt.Sprint(many.Remedy.Argv) != fmt.Sprint([]string{"teploy", "releases", "prune", "--app", "many", "--host", "edge", "--keep", "2"}) { + t.Errorf("many remedy argv = %v", many.Remedy.Argv) + } + if !many.Remedy.Destructive || !many.Remedy.Confirm { + t.Errorf("pruning versions is destructive: %+v", many.Remedy) + } + + // unset: keep_versions not set → default 5, warning + the notice date. + if !strings.Contains(unset.Evidence, "beyond keep_versions 5") { + t.Errorf("unset evidence = %q", unset.Evidence) + } + if unset.Severity != severityWarning { + t.Errorf("an unset keep_versions pruning by default is a warning: %+v", unset) + } + if !strings.Contains(unset.Evidence, "2026-11-08") { // notice 2026-11-01 + 7d + t.Errorf("unset evidence should name when automatic pruning starts: %q", unset.Evidence) + } + + // No notice file yet: the warning says the next deploy starts the notice. + fx := auditKitchenSink() + delete(fx.files, "/deployments/unset/.prune-notice") + env = fx.audit(t) + for _, f := range findingsByKind(env, findingVersionsBeyond) { + if f.Subject == "unset" && !strings.Contains(f.Evidence, "7-day notice") { + t.Errorf("unset evidence without a notice file = %q", f.Evidence) + } + } +} + +func TestServerAudit_ManualBlocksAreReportOnly(t *testing.T) { + env := auditKitchenSink().audit(t) + dead := oneFinding(t, env, findingManualDead) + if dead.Remedy.Argv != nil { + t.Errorf("hand-written blocks must never carry a remedy: %+v", dead) + } + if !strings.Contains(dead.Evidence, "gone-svc") || !strings.Contains(dead.Evidence, "hand-written") { + t.Errorf("dead manual evidence = %q", dead.Evidence) + } + nonlocal := oneFinding(t, env, findingManualNonLocal) + if nonlocal.Remedy.Argv != nil { + t.Errorf("hand-written blocks must never carry a remedy: %+v", nonlocal) + } + if !strings.Contains(nonlocal.Evidence, "203.0.113.9") { + t.Errorf("nonlocal evidence = %q", nonlocal.Evidence) + } +} + +func TestServerAudit_HygieneFindings(t *testing.T) { + env := auditKitchenSink().audit(t) + + artifacts := oneFinding(t, env, findingBuildArtifacts) + if fmt.Sprint(artifacts.Remedy.Argv) != fmt.Sprint([]string{"teploy", "server", "prune", "--host", "edge", "--build-cache", "--dangling"}) { + t.Errorf("build artifacts remedy = %v", artifacts.Remedy.Argv) + } + if artifacts.Remedy.Destructive || artifacts.Remedy.Confirm { + t.Errorf("rebuildable cache needs no confirmation: %+v", artifacts.Remedy) + } + + layer := oneFinding(t, env, findingLargeLayer) + if layer.Subject != "reacher-web-1f2e3d" || layer.Remedy.Argv != nil { + t.Errorf("large writable layer is report-only: %+v", layer) + } + if layer.SizeBytes < 35_000_000_000 { + t.Errorf("layer size = %d", layer.SizeBytes) + } + + unmanaged := oneFinding(t, env, findingUnmanagedLarge) + if unmanaged.Subject != "portainer" || unmanaged.Remedy.Argv != nil { + t.Errorf("unmanaged container is report-only: %+v", unmanaged) + } + + volumes := oneFinding(t, env, findingDanglingVolumes) + if volumes.Remedy.Argv != nil { + t.Errorf("volumes may hold the only copy — report only: %+v", volumes) + } + if !strings.Contains(volumes.Evidence, "21 volume(s)") || !strings.Contains(volumes.Evidence, "3.1 GB") { + t.Errorf("volumes evidence = %q", volumes.Evidence) + } + + sched := oneFinding(t, env, findingSchedulerMissing) + if sched.Severity != severityCritical || sched.Remedy.Argv != nil { + t.Errorf("teploy jobs with no cron daemon is critical, report-only: %+v", sched) + } +} + +func TestServerAudit_CaddyfileWithoutCaddy(t *testing.T) { + fx := auditKitchenSink() + fx.containers = strings.ReplaceAll(fx.containers, + auditContainer("caddy", "caddy:2", "running", "2026-05-01 00:00:00 +0000 UTC", nil), + auditContainer("caddy", "caddy:2", "exited", "2026-05-01 00:00:00 +0000 UTC", nil)) + env := fx.audit(t) + f := oneFinding(t, env, findingCaddyfileNoCaddy) + if f.Severity != severityWarning || !strings.Contains(f.Evidence, "the caddy container is exited") { + t.Errorf("caddyfile without caddy: %+v", f) + } + // Without a running caddy nothing can be proven dead: the stale route + // loses its one-click remedy instead of being reported removable. + var ghost *auditFinding + for _, f := range findingsByKind(env, findingStaleRoute) { + if f.Subject == "ghost" { + ghost = &f + } + } + if ghost == nil { + t.Fatalf("no stale_route finding for ghost: %+v", env.Findings) + } + if ghost.Remedy.Argv != nil || !strings.Contains(ghost.Evidence, "not removable") { + t.Errorf("an unprovable route must be report-only: %+v", ghost) + } + if st, ok := auditCheckStatus(env, "upstreams"); !ok || st != auditCheckUnknown { + t.Errorf("upstreams check = %q %v, want unknown without a running caddy", st, ok) + } +} + +func TestServerAudit_DiskPressureAttributesAndLinks(t *testing.T) { + env := auditKitchenSink().audit(t) + f := oneFinding(t, env, findingDiskPressure) + if f.Severity != severityCritical { // 800MB < 2GiB + t.Errorf("800MB free is critical, got %s", f.Severity) + } + if !strings.Contains(f.Evidence, "images 20.0 GB") || !strings.Contains(f.Evidence, "deployments") { + t.Errorf("disk evidence must attribute the space: %q", f.Evidence) + } + var relatedKinds []string + for _, id := range f.Related { + for _, x := range env.Findings { + if x.ID == id { + relatedKinds = append(relatedKinds, x.Kind) + } + } + } + for _, want := range []string{findingVersionsBeyond, findingBuildArtifacts, findingDanglingVolumes, findingLargeLayer} { + found := false + for _, k := range relatedKinds { + if k == want { + found = true + } + } + if !found { + t.Errorf("disk pressure related findings missing %s: %v", want, f.Related) + } + } + if env.Disk == nil || env.Disk.AvailableBytes != 800000000 || env.Disk.UsedPercent != 96 { + t.Errorf("disk = %+v", env.Disk) + } +} + +func TestServerAudit_LivePreviewIsNotAFinding(t *testing.T) { + env := auditKitchenSink().audit(t) + for _, f := range env.Findings { + if strings.Contains(f.ID, "preview") { + t.Errorf("a live preview route is not stale: %+v", f) + } + } +} + +func TestServerAudit_UnreadableYieldsCheckNotFinding(t *testing.T) { + fx := auditKitchenSink() + // The container listing fails: orphan/large-layer/unmanaged findings are + // impossible to judge, and routes cannot be proven dead. + fx.extra = append([]ssh.MockCommand{{Match: "docker ps --all ", Err: errors.New("docker: cannot connect to the docker daemon")}}, fx.extra...) + env := fx.audit(t) + if env.State != auditStatePartial { + t.Fatalf("state = %s, want partial", env.State) + } + if st, ok := auditCheckStatus(env, "containers"); !ok || st != auditCheckUnknown { + t.Fatalf("containers check = %q %v, want unknown: %+v", st, ok, env.Checks) + } + for _, kind := range []string{findingOrphanState, findingLargeLayer, findingUnmanagedLarge, findingCaddyfileNoCaddy} { + if got := findingsByKind(env, kind); len(got) != 0 { + t.Errorf("with containers unreadable there must be no %s finding: %+v", kind, got) + } + } + var ghost *auditFinding + for _, f := range findingsByKind(env, findingStaleRoute) { + if f.Subject == "ghost" { + ghost = &f + } + } + if ghost == nil { + t.Fatalf("no stale_route finding for ghost: %+v", env.Findings) + } + if ghost.Remedy.Argv != nil { + t.Errorf("removal cannot be proven safe without the container list: %+v", ghost.Remedy) + } + // An app whose state could not be read is never judged; the check names it. + if st, ok := auditCheckStatus(env, "apps"); !ok || st != auditCheckUnknown { + t.Errorf("apps check = %q %v, want unknown", st, ok) + } +} + +func TestServerAudit_HealthyServerReportsNothing(t *testing.T) { + fx := auditFixture{ + apps: []string{"blog"}, + state: map[string]string{"blog": auditStateJSON("container", "caddy", "v3", "v2", map[string]any{"container": map[string]any{"keep_versions": 3}})}, + caddyfile: "# TEPLOY BEGIN blog\nblog.example.com {\n\treverse_proxy blog-web-v3:3000\n}\n# TEPLOY END blog\n", + containers: auditContainer("caddy", "caddy:2", "running", "2026-05-01 00:00:00 +0000 UTC", nil) + "\n" + + auditContainer("blog-web-v3", "blog:v3", "running", "2026-11-04 10:00:00 +0000 UTC", + map[string]string{"teploy.app": "blog", "teploy.process": "web", "teploy.version": "v3"}), + sizes: auditSizeRow("caddy", "100MB (virtual 200MB)") + "\n" + auditSizeRow("blog-web-v3", "50MB (virtual 1GB)"), + resolves: "0 blog-web-v3\n", + addrs: "192.0.2.10\n", + usage: `{"Type":"Images","Size":"20GB","Reclaimable":"1GB (5%)"}` + "\n" + `{"Type":"Containers","Size":"5GB","Reclaimable":"0B (0%)"}` + "\n" + `{"Type":"Local Volumes","Size":"4GB","Reclaimable":"0B (0%)"}` + "\n" + `{"Type":"Build Cache","Size":"2GB","Reclaimable":"100MB (5%)"}`, + df: "Filesystem 1-blocks Used Available Capacity Mounted on\n/dev/vda1 100000000000 30000000000 70000000000 30% /\n", + scheduler: "daemon=yes\nenabled=enabled\ncrontab=no\n", + } + env := fx.audit(t) + if env.State != auditStateOK { + t.Fatalf("state = %s, want ok; checks: %+v", env.State, env.Checks) + } + if len(env.Findings) != 0 { + t.Fatalf("a healthy server must report no findings: %+v", env.Findings) + } +} + +func TestServerAudit_JSONEncodesRemedyNull(t *testing.T) { + env := auditKitchenSink().audit(t) + data, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + findings, _ := doc["findings"].([]any) + if len(findings) == 0 { + t.Fatal("no findings in JSON") + } + nullArgv := 0 + for _, f := range findings { + remedy := f.(map[string]any)["remedy"].(map[string]any) + argv, _ := remedy["argv"].([]any) + if remedy["argv"] == nil { + nullArgv++ + } else if len(argv) == 0 || argv[0] != "teploy" { + t.Errorf("remedy argv must be null or start with teploy: %v", remedy["argv"]) + } + if _, ok := remedy["destructive"].(bool); !ok { + t.Errorf("destructive must always be present: %v", remedy) + } + if _, ok := remedy["confirm"].(bool); !ok { + t.Errorf("confirm must always be present: %v", remedy) + } + } + if nullArgv == 0 { + t.Error("no finding with remedy argv null in the fixture") + } + if _, ok := doc["disk"].(map[string]any); !ok { + t.Error("disk must be an object on a reachable server") + } + if _, ok := doc["checks"].([]any); !ok { + t.Error("checks must be an array") + } +} + +// The read-only pin: run the full kitchen sink (every read fires, including +// the pressure-only du and the retention reads) through a recording wrapper +// and refuse any mutating verb, docker subcommand outside the read set, or +// file upload. +func TestServerAudit_EveryCommandOnlyReads(t *testing.T) { + fx := auditKitchenSink() + base := fx.exec() + rec := &recordingExecutor{Executor: base} + env := auditServer(context.Background(), rec, "edge", "192.0.2.10", auditFixtureNow) + if env.State != auditStatePartial { + t.Fatalf("fixture stopped exercising a read: %+v", env.Checks) + } + sawDU := false + for _, cmd := range rec.commands { + if strings.Contains(cmd, "du -sxb") { + sawDU = true + } + } + if !sawDU { + t.Errorf("the disk-pressure attribution read never ran; recorded: %v", rec.commands) + } + if len(rec.uploads) != 0 { + t.Errorf("the audit uploaded files: %v", rec.uploads) + } + if len(rec.inputs) != 0 { + t.Errorf("the audit streamed stdin to commands: %v", rec.inputs) + } + if len(rec.commands) == 0 { + t.Fatal("no commands recorded") + } + banned := []*regexp.Regexp{ + regexp.MustCompile(`\brm\b`), regexp.MustCompile(`\bmv\b`), regexp.MustCompile(`\bmkdir\b`), + regexp.MustCompile(`\brmdir\b`), regexp.MustCompile(`\bcp\b`), regexp.MustCompile(`\btouch\b`), + regexp.MustCompile(`\btruncate\b`), regexp.MustCompile(`\btee\b`), regexp.MustCompile(`\bchmod\b`), + regexp.MustCompile(`\bchown\b`), regexp.MustCompile(`\bkill\b`), regexp.MustCompile(`\bpkill\b`), + regexp.MustCompile(`\breboot\b`), regexp.MustCompile(`\bshutdown\b`), regexp.MustCompile(`\bsed\b`), + regexp.MustCompile(`\bdd\b`), regexp.MustCompile(`\bmkfs\b`), + regexp.MustCompile(`\bmount\b`), regexp.MustCompile(`\bumount\b`), regexp.MustCompile(`\bcurl\b`), + regexp.MustCompile(`\bwget\b`), regexp.MustCompile(`\bapt\b`), regexp.MustCompile(`\byum\b`), + regexp.MustCompile(`\bdnf\b`), regexp.MustCompile(`\bpip\b`), regexp.MustCompile(`\buseradd\b`), + regexp.MustCompile(`\bpasswd\b`), + regexp.MustCompile(`\bsystemctl\s+(start|stop|restart|enable|disable|mask)\b`), + regexp.MustCompile(`>>`), + } + allowedDocker := map[string]bool{"ps": true, "images": true, "volume": true, "system": true, "exec": true, "inspect": true, "version": true} + for _, cmd := range rec.commands { + for _, re := range banned { + if re.MatchString(cmd) { + t.Errorf("audit ran a mutating command (%s): %s", re, cmd) + } + } + if strings.Contains(cmd, "crontab") && !strings.Contains(cmd, "crontab -l") { + t.Errorf("audit touched crontab beyond -l: %s", cmd) + } + for _, tok := range strings.Fields(cmd) { + if strings.HasPrefix(tok, ">") && tok != ">/dev/null" { + t.Errorf("audit redirects stdout (%s): %s", tok, cmd) + } + } + if i := strings.Index(cmd, "docker "); i >= 0 { + fields := strings.Fields(cmd[i:]) + if len(fields) > 1 && !allowedDocker[fields[1]] { + t.Errorf("audit ran docker %s: %s", fields[1], cmd) + } + if len(fields) > 1 && fields[1] == "exec" && !strings.Contains(cmd, "getent") { + t.Errorf("audit docker exec is not the read-only getent probe: %s", cmd) + } + } + } +} + +// recordingExecutor wraps an Executor and records everything asked of it, so +// a test can pin WHAT a read-only code path may do to a server. +type recordingExecutor struct { + ssh.Executor + commands []string + inputs []string + uploads []string +} + +func (r *recordingExecutor) Run(ctx context.Context, cmd string) (string, error) { + r.commands = append(r.commands, cmd) + return r.Executor.Run(ctx, cmd) +} + +func (r *recordingExecutor) RunStream(ctx context.Context, cmd string, stdout, stderr io.Writer) error { + r.commands = append(r.commands, cmd) + return r.Executor.RunStream(ctx, cmd, stdout, stderr) +} + +func (r *recordingExecutor) RunInput(ctx context.Context, cmd string, stdin io.Reader) error { + r.inputs = append(r.inputs, cmd) + return r.Executor.RunInput(ctx, cmd, stdin) +} + +func (r *recordingExecutor) Upload(ctx context.Context, content io.Reader, remotePath string, mode string) error { + r.uploads = append(r.uploads, remotePath) + return r.Executor.Upload(ctx, content, remotePath, mode) +} + +func TestParseDockerSize(t *testing.T) { + for _, tt := range []struct { + in string + want int64 + ok bool + }{ + {"12B", 12, true}, + {"50MB", 50_000_000, true}, + {"2.5GB", 2_500_000_000, true}, + {"1KiB", 1024, true}, + {"35.7GB", 35_700_000_000, true}, // the "(virtual …)" tail is split off by the caller + {"", 0, false}, + {"lots", 0, false}, + } { + got, ok := parseDockerSize(tt.in) + if ok != tt.ok || got != tt.want { + t.Errorf("parseDockerSize(%q) = %d,%v want %d,%v", tt.in, got, ok, tt.want, tt.ok) + } + } +} diff --git a/internal/config/app.go b/internal/config/app.go index e130d61..2efb9e0 100644 --- a/internal/config/app.go +++ b/internal/config/app.go @@ -689,11 +689,12 @@ type AppConfig struct { // never plaintext on disk). Later files and explicit env: keys win. EnvFiles []string `yaml:"env_files,omitempty" toml:"env_files"` // KeepVersions caps the number of past app versions retained after a - // successful deploy (containers + images). Zero (default) keeps - // everything — historical behavior. Set to 2 or 3 to enable auto-prune - // with a rollback window. Container-deploy only; static deploy uses - // KeepReleases instead. - KeepVersions int `yaml:"keep_versions,omitempty" toml:"keep_versions"` + // successful deploy (containers + images). Unset means DefaultKeepVersions, + // and automatic pruning under that default starts only after a notice + // period; 0 keeps everything; `teploy init` writes 3. Pinned, current and + // previous versions are never pruned. Container-deploy only; static deploy + // uses KeepReleases instead. + KeepVersions *int `yaml:"keep_versions,omitempty" toml:"keep_versions,omitempty"` Hooks HooksConfig `yaml:"hooks,omitempty" toml:"hooks"` // BackupBeforeHooks backs every database accessory up before a deploy whose // pre_deploy hook is about to run (migrations rewrite data with no way down). @@ -1369,7 +1370,7 @@ func (c *AppConfig) validate() error { // rather than silently ignore. return fmt.Errorf("'tls' is not yet supported for type:static") } - if c.KeepVersions != 0 { + if c.KeepVersions != nil && *c.KeepVersions != 0 { return fmt.Errorf("'keep_versions' is container-deploy only; static deploys use 'keep_releases'") } default: @@ -1489,8 +1490,8 @@ func (c *AppConfig) validate() error { return fmt.Errorf("healthcheck refers to unknown process %q (must match a key in processes)", name) } } - if c.KeepVersions < 0 { - return fmt.Errorf("'keep_versions' must be >= 0 (got %d)", c.KeepVersions) + if c.KeepVersions != nil && *c.KeepVersions < 0 { + return fmt.Errorf("'keep_versions' must be >= 0 (got %d)", *c.KeepVersions) } if c.DrainSeconds < 0 || c.DrainSeconds > 600 { return fmt.Errorf("'drain_seconds' must be in 0..600 (got %d)", c.DrainSeconds) @@ -1881,7 +1882,7 @@ func mergeConfigs(base, overlay *AppConfig) { if overlay.Scan { base.Scan = true } - if overlay.KeepVersions != 0 { + if overlay.KeepVersions != nil { base.KeepVersions = overlay.KeepVersions } if overlay.Replicas != 0 { diff --git a/internal/config/keep_versions_test.go b/internal/config/keep_versions_test.go index 664937b..3cf0a10 100644 --- a/internal/config/keep_versions_test.go +++ b/internal/config/keep_versions_test.go @@ -1,6 +1,7 @@ package config import ( + "encoding/json" "os" "path/filepath" "strings" @@ -20,8 +21,8 @@ keep_versions: 3 if err != nil { t.Fatalf("LoadApp: %v", err) } - if cfg.KeepVersions != 3 { - t.Errorf("KeepVersions = %d, want 3", cfg.KeepVersions) + if cfg.KeepVersions == nil || *cfg.KeepVersions != 3 { + t.Errorf("KeepVersions = %v, want 3", cfg.KeepVersions) } } @@ -38,8 +39,8 @@ keep_versions = 2 if err != nil { t.Fatalf("LoadApp: %v", err) } - if cfg.KeepVersions != 2 { - t.Errorf("KeepVersions = %d, want 2", cfg.KeepVersions) + if cfg.KeepVersions == nil || *cfg.KeepVersions != 2 { + t.Errorf("KeepVersions = %v, want 2", cfg.KeepVersions) } } @@ -82,3 +83,54 @@ keep_versions: 2 t.Errorf("expected 'container-deploy only' in error, got: %v", err) } } + +// D3: unset means the default 5 (not explicit), any set value — including 0, +// which keeps everything — is the operator's choice. +func TestEffectiveKeepVersions(t *testing.T) { + if keep, explicit := EffectiveKeepVersions(nil); keep != DefaultKeepVersions || explicit { + t.Errorf("unset = %d, explicit %v, want %d not explicit", keep, explicit, DefaultKeepVersions) + } + three := 3 + if keep, explicit := EffectiveKeepVersions(&three); keep != 3 || !explicit { + t.Errorf("3 = %d, explicit %v", keep, explicit) + } + zero := 0 + if keep, explicit := EffectiveKeepVersions(&zero); keep != 0 || !explicit { + t.Errorf("0 = %d, explicit %v, want 0 explicit (keep everything)", keep, explicit) + } +} + +// The applied manifest must record unset and N exactly as before (existing +// digests hold) and distinguish an explicit 0 with its own key. +func TestManifestRecordsKeepVersions(t *testing.T) { + manifestFor := func(kv *int) map[string]any { + cfg := &AppConfig{App: "myapp", Domain: "myapp.com", Image: "example/myapp:1"} + cfg.KeepVersions = kv + raw, _, err := NormalizeAndDigest(cfg, "example/myapp:1") + if err != nil { + t.Fatalf("NormalizeAndDigest: %v", err) + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m["container"].(map[string]any) + } + if _, ok := manifestFor(nil)["keep_versions"].(float64); !ok || manifestFor(nil)["keep_versions"].(float64) != 0 { + t.Errorf("unset must record 0 (digest compatibility), got %v", manifestFor(nil)["keep_versions"]) + } + if _, ok := manifestFor(nil)["keep_all_versions"]; ok { + t.Error("unset must not record keep_all_versions") + } + three := 3 + if manifestFor(&three)["keep_versions"].(float64) != 3 { + t.Errorf("3 must record 3, got %v", manifestFor(&three)["keep_versions"]) + } + zero := 0 + if manifestFor(&zero)["keep_versions"].(float64) != 0 { + t.Errorf("explicit 0 must still record 0, got %v", manifestFor(&zero)["keep_versions"]) + } + if manifestFor(&zero)["keep_all_versions"] != true { + t.Errorf("explicit 0 must record keep_all_versions true, got %v", manifestFor(&zero)["keep_all_versions"]) + } +} diff --git a/internal/config/manifest.go b/internal/config/manifest.go index 60250ab..eb87159 100644 --- a/internal/config/manifest.go +++ b/internal/config/manifest.go @@ -41,7 +41,7 @@ func NormalizeAndDigest(cfg *AppConfig, appliedImage string) (json.RawMessage, s "ingress_mode": ingress, } if deploymentType == TypeContainer { - manifest["container"] = map[string]any{ + container := map[string]any{ "image": appliedImage, "port": port, "replicas": replicas, @@ -52,7 +52,7 @@ func NormalizeAndDigest(cfg *AppConfig, appliedImage string) (json.RawMessage, s "build_local": cfg.BuildLocal, "context": cleanOptionalPath(cfg.Context), "dockerfile": cleanOptionalPath(cfg.Dockerfile), - "keep_versions": cfg.KeepVersions, + "keep_versions": keepVersionsRecorded(cfg.KeepVersions), "processes": sortedMapKeys(cfg.Processes), "env_keys": bindingCombinedEnvKeys(cfg.Env, cfg.EnvLiteral), "env_files": append([]string(nil), cfg.EnvFiles...), @@ -68,6 +68,12 @@ func NormalizeAndDigest(cfg *AppConfig, appliedImage string) (json.RawMessage, s "asset_path": cfg.Assets.Path, "asset_keep_days": cfg.Assets.KeepDays, } + // Unset and N record what they always did, so existing digests hold; + // an explicit 0 needs its own key because unset also records 0. + if cfg.KeepVersions != nil && *cfg.KeepVersions == 0 { + container["keep_all_versions"] = true + } + manifest["container"] = container } else { keepReleases := cfg.KeepReleases if keepReleases == 0 { @@ -104,6 +110,25 @@ func NormalizeAndDigest(cfg *AppConfig, appliedImage string) (json.RawMessage, s return json.RawMessage(data), hex.EncodeToString(digest[:]), nil } +// DefaultKeepVersions is the retention of an app that does not set keep_versions. +const DefaultKeepVersions = 5 + +// EffectiveKeepVersions resolves keep_versions: the count (0 keeps everything) +// and whether the operator chose it. +func EffectiveKeepVersions(v *int) (keep int, explicit bool) { + if v == nil { + return DefaultKeepVersions, false + } + return *v, true +} + +func keepVersionsRecorded(v *int) int { + if v == nil { + return 0 + } + return *v +} + func normalizeDomains(domain string) string { if domain == "" { return "" diff --git a/internal/confighome/seed.go b/internal/confighome/seed.go index 19efb34..7e4ca0c 100644 --- a/internal/confighome/seed.go +++ b/internal/confighome/seed.go @@ -66,6 +66,9 @@ func Seed(rec *releasemeta.Record, manifest json.RawMessage) ([]byte, []string, } { copyKey(from, to) } + if all, _ := c["keep_all_versions"].(bool); all { + seed["keep_versions"] = 0 + } if m.Ingress == config.IngressHost { copyKey("bind", "bind") } diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go index fea8ff7..aaa8da3 100644 --- a/internal/deploy/deploy.go +++ b/internal/deploy/deploy.go @@ -53,15 +53,12 @@ type Config struct { // HTTP listener for the image's curl probe to hit). NoHealthcheck map[string]bool // KeepVersions caps the number of past app versions retained on the - // server after a successful deploy. Zero (default) keeps every version - // — the historical behavior, in which Teploy stopped but never removed - // old containers, letting disk usage climb across deploys. Set this to - // 2 or 3 to keep the current version + a small rollback window and - // have superseded versions auto-pruned (containers + images). - // Versions are scored by newest container creation timestamp; the - // current and immediately-previous versions are always protected even - // if older than other versions on disk. - KeepVersions int + // server after a successful deploy (containers + images). Nil means + // config.DefaultKeepVersions, pruned only after a notice period (see + // defaultPruneDue); zero keeps every version. Versions are scored by + // newest container creation timestamp; the current, immediately-previous + // and pinned versions are always protected. + KeepVersions *int // Ingress selects the routing layer; see config.IngressCaddy / // IngressExternal. Empty defaults to caddy. When external, the // deployer skips every Caddy interaction (route update, reload) — @@ -1268,11 +1265,11 @@ func (d *Deployer) deployFenced(ctx context.Context, cfg Config, lk *state.Lock) } } - // 15b. Prune superseded app versions (containers + images) if the - // operator opted in via keep_versions. Always protects the current - // version and the immediately-previous version so a rollback target - // is preserved regardless of timestamp ordering. - if cfg.KeepVersions > 0 { + // 15b. Prune superseded app versions (containers + images) per + // keep_versions. Always protects the current version and the + // immediately-previous version so a rollback target is preserved + // regardless of timestamp ordering. + if keep, explicit := config.EffectiveKeepVersions(cfg.KeepVersions); keep > 0 { // Fence (F16): pruning removes other versions' containers; a // stale holder must not delete under a new owner's feet. fenceOK := true @@ -1298,18 +1295,24 @@ func (d *Deployer) deployFenced(ctx context.Context, cfg Config, lk *state.Lock) fmt.Fprintf(d.out, "Warning: version prune skipped — pin state could not be read: %v\n", pinsErr) } else { protected = append(protected, pins...) - pruned, err := d.docker.PruneVersions(ctx, cfg.App, cfg.KeepVersions, protected...) - switch { - case err != nil: - // Partial cleanup (A25): report what was left behind — - // the old path printed nothing when any removal failed, - // or reported failed removals as pruned. - fmt.Fprintf(d.out, "Warning: version prune incomplete: %v\n", err) + if explicit || d.defaultPruneDue(ctx, cfg.App, keep, protected) { + pruned, err := d.docker.PruneVersions(ctx, cfg.App, keep, protected...) if len(pruned) > 0 { + events.Record(ctx, d.exec, events.Event{App: cfg.App, Kind: events.KindReleasesPrune, Outcome: events.Outcome(err), Release: cfg.Version, + Detail: events.ErrDetail(map[string]any{"keep": keep, "pruned": pruned, "trigger": "deploy"}, err)}) + } + switch { + case err != nil: + // Partial cleanup (A25): report what was left behind — + // the old path printed nothing when any removal failed, + // or reported failed removals as pruned. + fmt.Fprintf(d.out, "Warning: version prune incomplete: %v\n", err) + if len(pruned) > 0 { + fmt.Fprintf(d.out, "Pruned %d superseded version(s): %s\n", len(pruned), strings.Join(pruned, ", ")) + } + case len(pruned) > 0: fmt.Fprintf(d.out, "Pruned %d superseded version(s): %s\n", len(pruned), strings.Join(pruned, ", ")) } - case len(pruned) > 0: - fmt.Fprintf(d.out, "Pruned %d superseded version(s): %s\n", len(pruned), strings.Join(pruned, ", ")) } } } diff --git a/internal/deploy/retention.go b/internal/deploy/retention.go new file mode 100644 index 0000000..522ea11 --- /dev/null +++ b/internal/deploy/retention.go @@ -0,0 +1,65 @@ +package deploy + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +// DefaultPruneNotice is how long automatic pruning under the default +// keep_versions waits after a deploy first finds versions beyond it, so +// `teploy server audit` and the deploy output warn before anything goes. An +// explicit keep_versions already asked for pruning and has no notice. +const DefaultPruneNotice = 7 * 24 * time.Hour + +// PruneNoticePath records when an app's notice period started. +func PruneNoticePath(app string) string { + return "/deployments/" + app + "/.prune-notice" +} + +// ParsePruneNotice reads a notice file's timestamp. +func ParsePruneNotice(data []byte) (time.Time, bool) { + t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(data))) + return t, err == nil +} + +// defaultPruneDue says whether a default-policy prune may run now, starting the +// notice period the first time there is something to prune. Anything it cannot +// establish means no prune this deploy. +func (d *Deployer) defaultPruneDue(ctx context.Context, app string, keep int, protected []string) bool { + containers, err := d.docker.ListContainers(ctx, app) + if err != nil { + fmt.Fprintf(d.out, "Warning: version prune skipped: %v\n", err) + return false + } + prunable := docker.PrunableVersions(containers, keep, protected...) + if len(prunable) == 0 { + return false + } + data, present, err := state.ReadRemoteFile(ctx, d.exec, PruneNoticePath(app)) + if err != nil { + fmt.Fprintf(d.out, "Warning: version prune skipped: %v\n", err) + return false + } + now := time.Now().UTC() + started, ok := ParsePruneNotice(data) + if !present || !ok { + started = now + if err := ssh.UploadAtomic(ctx, d.exec, strings.NewReader(now.Format(time.RFC3339)+"\n"), PruneNoticePath(app), "0600"); err != nil { + fmt.Fprintf(d.out, "Warning: version prune skipped: could not record the notice: %v\n", err) + return false + } + } + if due := started.Add(DefaultPruneNotice); now.Before(due) { + fmt.Fprintf(d.out, "Note: %d version(s) are beyond the default keep_versions (%d): %s. Deploys after %s remove them; set keep_versions (0 keeps everything) or pin what you need.\n", + len(prunable), config.DefaultKeepVersions, strings.Join(prunable, ", "), due.Format("2006-01-02")) + return false + } + return true +} diff --git a/internal/deploy/retention_test.go b/internal/deploy/retention_test.go new file mode 100644 index 0000000..73c8954 --- /dev/null +++ b/internal/deploy/retention_test.go @@ -0,0 +1,113 @@ +package deploy + +// The default keep_versions notice period (decision D3): an app that never +// set keep_versions is pruned to DefaultKeepVersions only after a recorded +// 7-day notice, so `server audit` and the deploy output can warn first. + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/ssh" +) + +func pruneDueExecutor(containers string, notice *string) *ssh.MockExecutor { + m := ssh.NewMockExecutor("192.0.2.10", + ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='myapp'", Output: containers}, + ) + if notice != nil { + m.Files["/deployments/myapp/.prune-notice"] = []byte(*notice) + } + return m +} + +func TestDefaultPruneDue(t *testing.T) { + threeVersions := strings.Join([]string{ + `{"ID":"c1","Names":"myapp-web-v3","Image":"myapp:v3","State":"running","Status":"Up","CreatedAt":"2026-05-28 21:30:00 -0700 PDT","Labels":"teploy.app=myapp,teploy.version=v3"}`, + `{"ID":"c2","Names":"myapp-web-v2","Image":"myapp:v2","State":"exited","Status":"Up","CreatedAt":"2026-05-28 21:00:00 -0700 PDT","Labels":"teploy.app=myapp,teploy.version=v2"}`, + `{"ID":"c3","Names":"myapp-web-v1","Image":"myapp:v1","State":"exited","Status":"Up","CreatedAt":"2026-05-28 20:00:00 -0700 PDT","Labels":"teploy.app=myapp,teploy.version=v1"}`, + }, "\n") + + t.Run("nothing beyond keep never starts a notice", func(t *testing.T) { + var out bytes.Buffer + m := pruneDueExecutor(threeVersions, nil) + d := &Deployer{exec: m, docker: docker.NewClient(m), out: &out} + if d.defaultPruneDue(context.Background(), "myapp", 5, nil) { + t.Fatal("prune due with nothing beyond keep") + } + if _, uploaded := m.Files["/deployments/myapp/.prune-notice"]; uploaded { + t.Error("a notice was recorded with nothing to prune") + } + }) + + t.Run("the first finding starts the notice and waits", func(t *testing.T) { + var out bytes.Buffer + m := pruneDueExecutor(threeVersions, nil) + d := &Deployer{exec: m, docker: docker.NewClient(m), out: &out} + if d.defaultPruneDue(context.Background(), "myapp", 1, []string{"v3", "v2"}) { + t.Fatal("prune due on the first sighting") + } + recorded := string(m.Files["/deployments/myapp/.prune-notice"]) + if !strings.Contains(recorded, time.Now().UTC().Format("2006-01-02")) { + t.Errorf("notice = %q, want today's timestamp", recorded) + } + if !strings.Contains(out.String(), "remove them automatically") && !strings.Contains(out.String(), "remove them") { + t.Errorf("deploy output must warn before the first automatic prune: %q", out.String()) + } + if !strings.Contains(out.String(), "set keep_versions") { + t.Errorf("the warning must say how to opt out: %q", out.String()) + } + }) + + t.Run("a notice inside the window waits and names the date", func(t *testing.T) { + var out bytes.Buffer + threeDaysAgo := time.Now().UTC().Add(-3 * 24 * time.Hour).Format(time.RFC3339) + m := pruneDueExecutor(threeVersions, &threeDaysAgo) + d := &Deployer{exec: m, docker: docker.NewClient(m), out: &out} + if d.defaultPruneDue(context.Background(), "myapp", 1, []string{"v3", "v2"}) { + t.Fatal("prune due three days into a seven-day notice") + } + if !strings.Contains(out.String(), time.Now().UTC().Add(4*24*time.Hour).Format("2006-01-02")) { + t.Errorf("the warning should name when pruning starts: %q", out.String()) + } + }) + + t.Run("an elapsed notice allows the prune", func(t *testing.T) { + var out bytes.Buffer + eightDaysAgo := time.Now().UTC().Add(-8 * 24 * time.Hour).Format(time.RFC3339) + m := pruneDueExecutor(threeVersions, &eightDaysAgo) + d := &Deployer{exec: m, docker: docker.NewClient(m), out: &out} + if !d.defaultPruneDue(context.Background(), "myapp", 1, []string{"v3", "v2"}) { + t.Fatal("prune still refused after the notice elapsed") + } + }) + + t.Run("an unreadable notice never prunes", func(t *testing.T) { + var out bytes.Buffer + m := ssh.NewMockExecutor("192.0.2.10", + ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='myapp'", Output: threeVersions}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/.prune-notice'", Err: errors.New("stale file handle")}, + ) + d := &Deployer{exec: m, docker: docker.NewClient(m), out: &out} + if d.defaultPruneDue(context.Background(), "myapp", 1, []string{"v3", "v2"}) { + t.Fatal("a notice that cannot be read must not authorize a prune") + } + if !strings.Contains(out.String(), "skipped") { + t.Errorf("the refusal must be visible: %q", out.String()) + } + }) +} + +func TestParsePruneNotice(t *testing.T) { + if ts, ok := ParsePruneNotice([]byte("2026-11-01T00:00:00Z\n")); !ok || ts.UTC().Format(time.DateOnly) != "2026-11-01" { + t.Errorf("ParsePruneNotice = %v, %v", ts, ok) + } + if _, ok := ParsePruneNotice([]byte("yesterday")); ok { + t.Error("a non-timestamp notice must not parse") + } +} diff --git a/internal/docker/docker.go b/internal/docker/docker.go index e9f128b..8f27a8f 100644 --- a/internal/docker/docker.go +++ b/internal/docker/docker.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net" + "slices" "sort" "strconv" "strings" @@ -879,114 +880,100 @@ func ContainersByApp(containers []Container, app string) []Container { // a joined error describing every failed removal. Per-version image // removal stays best-effort (a shared image legitimately refuses). func (c *Client) PruneVersions(ctx context.Context, app string, keep int, protectedVersions ...string) ([]string, error) { - if keep < 0 { - keep = 0 - } containers, err := c.ListContainers(ctx, app) if err != nil { return nil, fmt.Errorf("listing containers for prune: %w", err) } + var pruned []string + var failures []error + for _, version := range PrunableVersions(containers, keep, protectedVersions...) { + names, images := versionArtifacts(containers, version) + // Force-remove containers in case any are still running. We + // already took ownership of cleanup; refusing to nuke a stray + // running container from an older version defeats the point. + // A version counts as pruned ONLY when every container removal + // succeeded (A25) — the old loop counted it regardless, so + // "Pruned N" could report versions whose containers were still + // running, hiding disk exhaustion and failed cleanup. + complete := true + for _, name := range names { + if _, err := c.exec.Run(ctx, "docker rm -f "+ssh.ShellQuote(name)); err != nil { + complete = false + failures = append(failures, fmt.Errorf("removing container %s (version %s): %w", name, version, err)) + } + } + // Best-effort image removal. Fails (silently) if another + // container or tag still references the image, which is the + // safe behavior. + for _, img := range images { + _, _ = c.exec.Run(ctx, "docker rmi "+ssh.ShellQuote(img)) + } + if complete { + pruned = append(pruned, version) + } + } + return pruned, errors.Join(failures...) +} - // Group containers by teploy.version label. Track newest creation - // timestamp + container names + image names per version. Containers - // without the label (e.g. caddy, postgres accessory) are ignored. - type vinfo struct { - newestCreated time.Time - containerNames []string - images map[string]struct{} +// PrunableVersions lists the versions PruneVersions would remove from one app's +// containers, newest first: everything outside the `keep` most recent and the +// protected set. Containers without a version label (accessories) never count. +func PrunableVersions(containers []Container, keep int, protectedVersions ...string) []string { + if keep < 0 { + keep = 0 } - versions := map[string]*vinfo{} + newest := map[string]time.Time{} for _, ct := range containers { v := ct.Labels["teploy.version"] - if v == "" { + if v == "" || ct.Labels["teploy.role"] == "accessory" { continue } - info, ok := versions[v] - if !ok { - info = &vinfo{images: map[string]struct{}{}} - versions[v] = info - } // Compare as instants, not strings. docker's CreatedAt is a localized // timestamp ("2006-01-02 15:04:05 -0700 MST"), so lexicographic ordering // is wrong across timezones / DST. On a parse failure, treat the version // as newest so cleanup never prunes something it can't date. created := parseDockerTime(ct.CreatedAt) - if created.After(info.newestCreated) { - info.newestCreated = created + if cur, ok := newest[v]; !ok || created.After(cur) { + newest[v] = created } - info.containerNames = append(info.containerNames, ct.Name) - if ct.Image != "" { - info.images[ct.Image] = struct{}{} - } - } - - if len(versions) == 0 { - return nil, nil } - - // Sort versions by recency (newest first). - type entry struct { - version string - info *vinfo - } - sorted := make([]entry, 0, len(versions)) - for v, info := range versions { - sorted = append(sorted, entry{v, info}) + sorted := make([]string, 0, len(newest)) + for v := range newest { + sorted = append(sorted, v) } sort.Slice(sorted, func(i, j int) bool { - if !sorted[i].info.newestCreated.Equal(sorted[j].info.newestCreated) { - return sorted[i].info.newestCreated.After(sorted[j].info.newestCreated) + if !newest[sorted[i]].Equal(newest[sorted[j]]) { + return newest[sorted[i]].After(newest[sorted[j]]) } // Deterministic tie-break when timestamps match (or both unparseable). - return sorted[i].version > sorted[j].version + return sorted[i] > sorted[j] }) - - // Build the protected set: explicit names + top-`keep` by recency. protect := map[string]bool{} for _, v := range protectedVersions { if v != "" { protect[v] = true } } - kept := 0 - for _, e := range sorted { - if kept < keep { - protect[e.version] = true - kept++ + var prunable []string + for i, v := range sorted { + if i >= keep && !protect[v] { + prunable = append(prunable, v) } } + return prunable +} - var pruned []string - var failures []error - for _, e := range sorted { - if protect[e.version] { +func versionArtifacts(containers []Container, version string) (names, images []string) { + for _, ct := range containers { + if ct.Labels["teploy.version"] != version || ct.Labels["teploy.role"] == "accessory" { continue } - // Force-remove containers in case any are still running. We - // already took ownership of cleanup; refusing to nuke a stray - // running container from an older version defeats the point. - // A version counts as pruned ONLY when every container removal - // succeeded (A25) — the old loop counted it regardless, so - // "Pruned N" could report versions whose containers were still - // running, hiding disk exhaustion and failed cleanup. - complete := true - for _, name := range e.info.containerNames { - if _, err := c.exec.Run(ctx, "docker rm -f "+ssh.ShellQuote(name)); err != nil { - complete = false - failures = append(failures, fmt.Errorf("removing container %s (version %s): %w", name, e.version, err)) - } - } - // Best-effort image removal. Fails (silently) if another - // container or tag still references the image, which is the - // safe behavior. - for img := range e.info.images { - _, _ = c.exec.Run(ctx, "docker rmi "+ssh.ShellQuote(img)) - } - if complete { - pruned = append(pruned, e.version) + names = append(names, ct.Name) + if ct.Image != "" && !slices.Contains(images, ct.Image) { + images = append(images, ct.Image) } } - return pruned, errors.Join(failures...) + return names, images } // EnsureNetwork creates the "teploy" Docker network if it doesn't already exist. diff --git a/internal/docker/prune_test.go b/internal/docker/prune_test.go index 0835c8b..6c72577 100644 --- a/internal/docker/prune_test.go +++ b/internal/docker/prune_test.go @@ -151,3 +151,53 @@ func mkPS(id, name, image, createdAt, version, process string) string { id, name, image, createdAt, process, version, ) } + +// TestPrunableVersions_Selection pins the selection PruneVersions and +// `releases prune`/`server audit` share: the `keep` most recent by newest +// container creation time, minus the protected set, accessories never counted. +func TestPrunableVersions_Selection(t *testing.T) { + ct := func(name, version, createdAt string) Container { + return Container{Name: name, Image: "myapp:" + version, CreatedAt: createdAt, + Labels: map[string]string{"teploy.app": "myapp", "teploy.version": version}} + } + containers := []Container{ + ct("myapp-web-v3", "v3", "2026-05-28 21:30:00 -0700 PDT"), + ct("myapp-web-v2", "v2", "2026-05-28 21:00:00 -0700 PDT"), + ct("myapp-web-v1", "v1", "2026-05-28 20:00:00 -0700 PDT"), + // An accessory carrying a version label must never be a candidate. + {Name: "myapp-db", Image: "postgres:16", CreatedAt: "2026-05-20 20:00:00 -0700 PDT", + Labels: map[string]string{"teploy.app": "myapp", "teploy.role": "accessory", "teploy.version": "v0"}}, + } + got := PrunableVersions(containers, 2) + if fmt.Sprint(got) != fmt.Sprint([]string{"v1"}) { + t.Errorf("keep 2 = %v, want [v1]", got) + } + got = PrunableVersions(containers, 0) + if fmt.Sprint(got) != fmt.Sprint([]string{"v3", "v2", "v1"}) { + t.Errorf("keep 0 = %v, want everything but accessories, newest first", got) + } + got = PrunableVersions(containers, 1, "v3") + if fmt.Sprint(got) != fmt.Sprint([]string{"v2", "v1"}) { + t.Errorf("keep 1 with v3 protected = %v, want [v2 v1]", got) + } + // Everything protected: nothing prunable whatever keep says. + if got = PrunableVersions(containers, 0, "v1", "v2", "v3"); len(got) != 0 { + t.Errorf("fully protected = %v, want nothing", got) + } +} + +// TestPrunableVersions_UnparseableDatesAreNewest pins the fail-safe ordering: +// a version whose creation time cannot be parsed is treated as newest, so +// cleanup never deletes something it cannot date. +func TestPrunableVersions_UnparseableDatesAreNewest(t *testing.T) { + containers := []Container{ + {Name: "myapp-web-v1", Image: "myapp:v1", CreatedAt: "not a date", + Labels: map[string]string{"teploy.app": "myapp", "teploy.version": "v1"}}, + {Name: "myapp-web-v2", Image: "myapp:v2", CreatedAt: "2026-05-28 21:00:00 -0700 PDT", + Labels: map[string]string{"teploy.app": "myapp", "teploy.version": "v2"}}, + } + got := PrunableVersions(containers, 1) + if fmt.Sprint(got) != fmt.Sprint([]string{"v2"}) { + t.Errorf("keep 1 = %v, want [v2]: the undatable v1 sorts newest and is kept", got) + } +} diff --git a/internal/domains/domains.go b/internal/domains/domains.go index a8ca8b6..fa3812a 100644 --- a/internal/domains/domains.go +++ b/internal/domains/domains.go @@ -306,10 +306,13 @@ func CheckServer(ctx context.Context, exec ssh.Executor, name string, opts Optio return srv, rows } +// AppListCommand prints the name of every app with state on a server. +const AppListCommand = `for f in /deployments/*/state.json /deployments/*/state; do [ -f "$f" ] && basename "$(dirname "$f")"; done | sort -u` + // readApps lists the deployed apps with their ingress and domain. known=false // means the app list itself could not be read, so ownership cannot be judged. func readApps(ctx context.Context, exec ssh.Executor, srv *Server, issue func(string, error)) (apps []AppInfo, known bool) { - out, err := exec.Run(ctx, `for f in /deployments/*/state.json /deployments/*/state; do [ -f "$f" ] && basename "$(dirname "$f")"; done | sort -u`) + out, err := exec.Run(ctx, AppListCommand) if err != nil { issue("apps", err) return nil, false @@ -834,10 +837,10 @@ func skipFor(row Domain, host string) string { var upstreamHostRE = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`) -// upstreamHost returns the name to resolve for a dial, or why not: an IP literal +// UpstreamHost returns the name to resolve for a dial, or why not: an IP literal // has nothing to resolve; a placeholder or odd token cannot be checked. Names // are validated before they reach a shell, and quoted there. -func upstreamHost(dial string) (name, verdict string) { +func UpstreamHost(dial string) (name, verdict string) { host := dial if h, _, err := net.SplitHostPort(dial); err == nil { host = h @@ -859,7 +862,7 @@ func resolveUpstreams(ctx context.Context, exec ssh.Executor, rows []Domain, srv wanted := map[string]bool{} for i := range rows { for _, u := range rows[i].Route.Upstreams { - if name, _ := upstreamHost(u.Dial); name != "" { + if name, _ := UpstreamHost(u.Dial); name != "" { wanted[name] = true } } @@ -868,32 +871,18 @@ func resolveUpstreams(ctx context.Context, exec ssh.Executor, rows []Domain, srv if len(wanted) > 0 && srv.Caddy.Container == "present" { names := make([]string, 0, len(wanted)) for n := range wanted { - names = append(names, ssh.ShellQuote(n)) + names = append(names, n) } - sort.Strings(names) - cmd := "printf '%s\\n' " + strings.Join(names, " ") + ` | docker exec -i caddy sh -c 'while read -r h; do getent hosts "$h" >/dev/null 2>&1; echo "$? $h"; done'` - out, err := exec.Run(ctx, cmd) - if err != nil { + var err error + if answers, err = ResolveNames(ctx, exec, names); err != nil { issue("upstreams", err) } - for _, line := range strings.Split(out, "\n") { - rc, name, ok := strings.Cut(strings.TrimSpace(line), " ") - if !ok || !wanted[name] { - continue - } - switch rc { - case "0": - answers[name] = ResolvesYes - case "2": - answers[name] = ResolvesNo - } - } } for i := range rows { r := &rows[i].Route yes, no, unknown := 0, 0, 0 for j := range r.Upstreams { - name, verdict := upstreamHost(r.Upstreams[j].Dial) + name, verdict := UpstreamHost(r.Upstreams[j].Dial) if name != "" { if verdict = answers[name]; verdict == "" { verdict = ResolvesUnknown @@ -928,6 +917,36 @@ func resolveUpstreams(ctx context.Context, exec ssh.Executor, rows []Domain, srv } } +// ResolveNames asks the running caddy container, in one docker exec, whether +// each name resolves: yes or no. A name missing from the result is unknown, +// and so is everything when the error is set. Names must pass UpstreamHost. +func ResolveNames(ctx context.Context, exec ssh.Executor, names []string) (map[string]string, error) { + answers := map[string]string{} + if len(names) == 0 { + return answers, nil + } + quoted := make([]string, 0, len(names)) + for _, n := range names { + quoted = append(quoted, ssh.ShellQuote(n)) + } + sort.Strings(quoted) + cmd := "printf '%s\\n' " + strings.Join(quoted, " ") + ` | docker exec -i caddy sh -c 'while read -r h; do getent hosts "$h" >/dev/null 2>&1; echo "$? $h"; done'` + out, err := exec.Run(ctx, cmd) + for _, line := range strings.Split(out, "\n") { + rc, name, ok := strings.Cut(strings.TrimSpace(line), " ") + if !ok { + continue + } + switch rc { + case "0": + answers[name] = ResolvesYes + case "2": + answers[name] = ResolvesNo + } + } + return answers, err +} + // applyTLS fills certificate evidence on rows Caddy would obtain for. Unknown // when the logs cannot back a claim; zero failures only on logs long enough. func applyTLS(rows []Domain, stats map[string]*ObtainStats, status CaddyStatus) { diff --git a/internal/events/events.go b/internal/events/events.go index a534529..d4d514b 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -117,6 +117,9 @@ const ( KindBackupOwners = "backup_target.owners_clear" KindBackupAll = "backup.all" KindBackupPolicy = "backup_policy.set" + KindRouteRemove = "route.remove" + KindReleasesPrune = "releases.prune" + KindServerPrune = "server.prune" ) // detailAllow is the per-kind allowlist of detail keys. Env and secret @@ -154,6 +157,9 @@ var detailAllow = map[string][]string{ KindBackupOwners: {"fingerprint", "force", "removed", "error"}, KindBackupAll: {"status", "targets_ok", "targets_failed", "omissions", "encrypted", "duration_ms", "heartbeats_ok", "heartbeats_failed", "warnings", "error"}, KindBackupPolicy: {"interval", "encrypted", "volumes", "adopted", "error"}, + KindRouteRemove: {"route", "hosts", "upstreams", "reason", "error"}, + KindReleasesPrune: {"keep", "pruned", "protected", "trigger", "error"}, + KindServerPrune: {"build_cache", "dangling", "output", "error"}, } // maxErrorDetail bounds the "error" detail so a command's full stderr