From d06df80e55b701425e91a867e346c05dbede25a8 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:38:43 -0700 Subject: [PATCH 1/6] feat(retention): keep_versions becomes optional (D3) with a prunable-version selection Unset keep_versions now means DefaultKeepVersions (5), pruned by deploys only after a 7-day notice recorded server-side (.prune-notice) so the deploy output and `server audit` can warn before anything goes; `teploy init` writes 3; an explicit 0 keeps everything and records keep_all_versions in the applied manifest (unset and N record what they always did, so existing digests hold; confighome seeding maps it back). docker.PrunableVersions is extracted from PruneVersions so the audit and `releases prune` can reason about the selection without running it: the `keep` most recent versions by newest container creation time, minus the protected set, accessories never candidates, undatable versions sorted newest (cleanup never deletes what it cannot date). Deploy-triggered prunes are recorded as releases.prune events. --- internal/cli/init.go | 11 +++ internal/cli/init_test.go | 16 ++++ internal/config/app.go | 19 ++-- internal/config/keep_versions_test.go | 60 +++++++++++- internal/config/manifest.go | 29 +++++- internal/confighome/seed.go | 3 + internal/deploy/deploy.go | 49 +++++----- internal/deploy/retention.go | 65 +++++++++++++ internal/deploy/retention_test.go | 113 ++++++++++++++++++++++ internal/docker/docker.go | 133 ++++++++++++-------------- internal/docker/prune_test.go | 50 ++++++++++ internal/events/events.go | 2 + 12 files changed, 439 insertions(+), 111 deletions(-) create mode 100644 internal/deploy/retention.go create mode 100644 internal/deploy/retention_test.go diff --git a/internal/cli/init.go b/internal/cli/init.go index 89c643d..a1f27ea 100644 --- a/internal/cli/init.go +++ b/internal/cli/init.go @@ -43,6 +43,11 @@ func newInitCmd(flags *Flags) *cobra.Command { return cmd } +// initKeepVersions is the keep_versions a generated config starts with +// (decision D3): a deliberate, small rollback window rather than the +// unset default. +const initKeepVersions = 3 + // initFlow prompts for the minimal config and writes it to dir. Shared by // `teploy init` and the zero-config first run inside `teploy deploy`. // Returns the path of the file written. @@ -109,6 +114,12 @@ func initFlow(reader *bufio.Reader, dir string, useTOML, force bool) (string, er } cfg.Server = prompt(reader, "Server (name from servers.yml or IP)", defaultServer) + // New apps start with a small rollback window rather than the default. + if cfg.KeepVersions == nil && !cfg.IsStatic() { + keep := initKeepVersions + cfg.KeepVersions = &keep + } + // Write config. var out []byte var err error diff --git a/internal/cli/init_test.go b/internal/cli/init_test.go index 2609968..5397acd 100644 --- a/internal/cli/init_test.go +++ b/internal/cli/init_test.go @@ -40,6 +40,22 @@ func TestInitFlowWritesValidConfigWithDomain(t *testing.T) { } } +// D3: a generated container config starts with keep_versions 3 — a small +// rollback window rather than the unset default. +func TestInitFlowWritesKeepVersionsThree(t *testing.T) { + dir := t.TempDir() + if _, err := initFlow(scripted("myapp", "myapp.com", "1.2.3.4"), dir, false, false); err != nil { + t.Fatalf("initFlow: %v", err) + } + cfg, err := config.LoadApp(dir) + if err != nil { + t.Fatalf("generated config does not load: %v", err) + } + if cfg.KeepVersions == nil || *cfg.KeepVersions != initKeepVersions { + t.Errorf("keep_versions = %v, want %d", cfg.KeepVersions, initKeepVersions) + } +} + func TestInitFlowEmptyDomainBecomesHostIngress(t *testing.T) { dir := t.TempDir() diff --git a/internal/config/app.go b/internal/config/app.go index e130d61..2efb9e0 100644 --- a/internal/config/app.go +++ b/internal/config/app.go @@ -689,11 +689,12 @@ type AppConfig struct { // never plaintext on disk). Later files and explicit env: keys win. EnvFiles []string `yaml:"env_files,omitempty" toml:"env_files"` // KeepVersions caps the number of past app versions retained after a - // successful deploy (containers + images). Zero (default) keeps - // everything — historical behavior. Set to 2 or 3 to enable auto-prune - // with a rollback window. Container-deploy only; static deploy uses - // KeepReleases instead. - KeepVersions int `yaml:"keep_versions,omitempty" toml:"keep_versions"` + // successful deploy (containers + images). Unset means DefaultKeepVersions, + // and automatic pruning under that default starts only after a notice + // period; 0 keeps everything; `teploy init` writes 3. Pinned, current and + // previous versions are never pruned. Container-deploy only; static deploy + // uses KeepReleases instead. + KeepVersions *int `yaml:"keep_versions,omitempty" toml:"keep_versions,omitempty"` Hooks HooksConfig `yaml:"hooks,omitempty" toml:"hooks"` // BackupBeforeHooks backs every database accessory up before a deploy whose // pre_deploy hook is about to run (migrations rewrite data with no way down). @@ -1369,7 +1370,7 @@ func (c *AppConfig) validate() error { // rather than silently ignore. return fmt.Errorf("'tls' is not yet supported for type:static") } - if c.KeepVersions != 0 { + if c.KeepVersions != nil && *c.KeepVersions != 0 { return fmt.Errorf("'keep_versions' is container-deploy only; static deploys use 'keep_releases'") } default: @@ -1489,8 +1490,8 @@ func (c *AppConfig) validate() error { return fmt.Errorf("healthcheck refers to unknown process %q (must match a key in processes)", name) } } - if c.KeepVersions < 0 { - return fmt.Errorf("'keep_versions' must be >= 0 (got %d)", c.KeepVersions) + if c.KeepVersions != nil && *c.KeepVersions < 0 { + return fmt.Errorf("'keep_versions' must be >= 0 (got %d)", *c.KeepVersions) } if c.DrainSeconds < 0 || c.DrainSeconds > 600 { return fmt.Errorf("'drain_seconds' must be in 0..600 (got %d)", c.DrainSeconds) @@ -1881,7 +1882,7 @@ func mergeConfigs(base, overlay *AppConfig) { if overlay.Scan { base.Scan = true } - if overlay.KeepVersions != 0 { + if overlay.KeepVersions != nil { base.KeepVersions = overlay.KeepVersions } if overlay.Replicas != 0 { diff --git a/internal/config/keep_versions_test.go b/internal/config/keep_versions_test.go index 664937b..3cf0a10 100644 --- a/internal/config/keep_versions_test.go +++ b/internal/config/keep_versions_test.go @@ -1,6 +1,7 @@ package config import ( + "encoding/json" "os" "path/filepath" "strings" @@ -20,8 +21,8 @@ keep_versions: 3 if err != nil { t.Fatalf("LoadApp: %v", err) } - if cfg.KeepVersions != 3 { - t.Errorf("KeepVersions = %d, want 3", cfg.KeepVersions) + if cfg.KeepVersions == nil || *cfg.KeepVersions != 3 { + t.Errorf("KeepVersions = %v, want 3", cfg.KeepVersions) } } @@ -38,8 +39,8 @@ keep_versions = 2 if err != nil { t.Fatalf("LoadApp: %v", err) } - if cfg.KeepVersions != 2 { - t.Errorf("KeepVersions = %d, want 2", cfg.KeepVersions) + if cfg.KeepVersions == nil || *cfg.KeepVersions != 2 { + t.Errorf("KeepVersions = %v, want 2", cfg.KeepVersions) } } @@ -82,3 +83,54 @@ keep_versions: 2 t.Errorf("expected 'container-deploy only' in error, got: %v", err) } } + +// D3: unset means the default 5 (not explicit), any set value — including 0, +// which keeps everything — is the operator's choice. +func TestEffectiveKeepVersions(t *testing.T) { + if keep, explicit := EffectiveKeepVersions(nil); keep != DefaultKeepVersions || explicit { + t.Errorf("unset = %d, explicit %v, want %d not explicit", keep, explicit, DefaultKeepVersions) + } + three := 3 + if keep, explicit := EffectiveKeepVersions(&three); keep != 3 || !explicit { + t.Errorf("3 = %d, explicit %v", keep, explicit) + } + zero := 0 + if keep, explicit := EffectiveKeepVersions(&zero); keep != 0 || !explicit { + t.Errorf("0 = %d, explicit %v, want 0 explicit (keep everything)", keep, explicit) + } +} + +// The applied manifest must record unset and N exactly as before (existing +// digests hold) and distinguish an explicit 0 with its own key. +func TestManifestRecordsKeepVersions(t *testing.T) { + manifestFor := func(kv *int) map[string]any { + cfg := &AppConfig{App: "myapp", Domain: "myapp.com", Image: "example/myapp:1"} + cfg.KeepVersions = kv + raw, _, err := NormalizeAndDigest(cfg, "example/myapp:1") + if err != nil { + t.Fatalf("NormalizeAndDigest: %v", err) + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + return m["container"].(map[string]any) + } + if _, ok := manifestFor(nil)["keep_versions"].(float64); !ok || manifestFor(nil)["keep_versions"].(float64) != 0 { + t.Errorf("unset must record 0 (digest compatibility), got %v", manifestFor(nil)["keep_versions"]) + } + if _, ok := manifestFor(nil)["keep_all_versions"]; ok { + t.Error("unset must not record keep_all_versions") + } + three := 3 + if manifestFor(&three)["keep_versions"].(float64) != 3 { + t.Errorf("3 must record 3, got %v", manifestFor(&three)["keep_versions"]) + } + zero := 0 + if manifestFor(&zero)["keep_versions"].(float64) != 0 { + t.Errorf("explicit 0 must still record 0, got %v", manifestFor(&zero)["keep_versions"]) + } + if manifestFor(&zero)["keep_all_versions"] != true { + t.Errorf("explicit 0 must record keep_all_versions true, got %v", manifestFor(&zero)["keep_all_versions"]) + } +} diff --git a/internal/config/manifest.go b/internal/config/manifest.go index 60250ab..eb87159 100644 --- a/internal/config/manifest.go +++ b/internal/config/manifest.go @@ -41,7 +41,7 @@ func NormalizeAndDigest(cfg *AppConfig, appliedImage string) (json.RawMessage, s "ingress_mode": ingress, } if deploymentType == TypeContainer { - manifest["container"] = map[string]any{ + container := map[string]any{ "image": appliedImage, "port": port, "replicas": replicas, @@ -52,7 +52,7 @@ func NormalizeAndDigest(cfg *AppConfig, appliedImage string) (json.RawMessage, s "build_local": cfg.BuildLocal, "context": cleanOptionalPath(cfg.Context), "dockerfile": cleanOptionalPath(cfg.Dockerfile), - "keep_versions": cfg.KeepVersions, + "keep_versions": keepVersionsRecorded(cfg.KeepVersions), "processes": sortedMapKeys(cfg.Processes), "env_keys": bindingCombinedEnvKeys(cfg.Env, cfg.EnvLiteral), "env_files": append([]string(nil), cfg.EnvFiles...), @@ -68,6 +68,12 @@ func NormalizeAndDigest(cfg *AppConfig, appliedImage string) (json.RawMessage, s "asset_path": cfg.Assets.Path, "asset_keep_days": cfg.Assets.KeepDays, } + // Unset and N record what they always did, so existing digests hold; + // an explicit 0 needs its own key because unset also records 0. + if cfg.KeepVersions != nil && *cfg.KeepVersions == 0 { + container["keep_all_versions"] = true + } + manifest["container"] = container } else { keepReleases := cfg.KeepReleases if keepReleases == 0 { @@ -104,6 +110,25 @@ func NormalizeAndDigest(cfg *AppConfig, appliedImage string) (json.RawMessage, s return json.RawMessage(data), hex.EncodeToString(digest[:]), nil } +// DefaultKeepVersions is the retention of an app that does not set keep_versions. +const DefaultKeepVersions = 5 + +// EffectiveKeepVersions resolves keep_versions: the count (0 keeps everything) +// and whether the operator chose it. +func EffectiveKeepVersions(v *int) (keep int, explicit bool) { + if v == nil { + return DefaultKeepVersions, false + } + return *v, true +} + +func keepVersionsRecorded(v *int) int { + if v == nil { + return 0 + } + return *v +} + func normalizeDomains(domain string) string { if domain == "" { return "" diff --git a/internal/confighome/seed.go b/internal/confighome/seed.go index 19efb34..7e4ca0c 100644 --- a/internal/confighome/seed.go +++ b/internal/confighome/seed.go @@ -66,6 +66,9 @@ func Seed(rec *releasemeta.Record, manifest json.RawMessage) ([]byte, []string, } { copyKey(from, to) } + if all, _ := c["keep_all_versions"].(bool); all { + seed["keep_versions"] = 0 + } if m.Ingress == config.IngressHost { copyKey("bind", "bind") } diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go index fea8ff7..aaa8da3 100644 --- a/internal/deploy/deploy.go +++ b/internal/deploy/deploy.go @@ -53,15 +53,12 @@ type Config struct { // HTTP listener for the image's curl probe to hit). NoHealthcheck map[string]bool // KeepVersions caps the number of past app versions retained on the - // server after a successful deploy. Zero (default) keeps every version - // — the historical behavior, in which Teploy stopped but never removed - // old containers, letting disk usage climb across deploys. Set this to - // 2 or 3 to keep the current version + a small rollback window and - // have superseded versions auto-pruned (containers + images). - // Versions are scored by newest container creation timestamp; the - // current and immediately-previous versions are always protected even - // if older than other versions on disk. - KeepVersions int + // server after a successful deploy (containers + images). Nil means + // config.DefaultKeepVersions, pruned only after a notice period (see + // defaultPruneDue); zero keeps every version. Versions are scored by + // newest container creation timestamp; the current, immediately-previous + // and pinned versions are always protected. + KeepVersions *int // Ingress selects the routing layer; see config.IngressCaddy / // IngressExternal. Empty defaults to caddy. When external, the // deployer skips every Caddy interaction (route update, reload) — @@ -1268,11 +1265,11 @@ func (d *Deployer) deployFenced(ctx context.Context, cfg Config, lk *state.Lock) } } - // 15b. Prune superseded app versions (containers + images) if the - // operator opted in via keep_versions. Always protects the current - // version and the immediately-previous version so a rollback target - // is preserved regardless of timestamp ordering. - if cfg.KeepVersions > 0 { + // 15b. Prune superseded app versions (containers + images) per + // keep_versions. Always protects the current version and the + // immediately-previous version so a rollback target is preserved + // regardless of timestamp ordering. + if keep, explicit := config.EffectiveKeepVersions(cfg.KeepVersions); keep > 0 { // Fence (F16): pruning removes other versions' containers; a // stale holder must not delete under a new owner's feet. fenceOK := true @@ -1298,18 +1295,24 @@ func (d *Deployer) deployFenced(ctx context.Context, cfg Config, lk *state.Lock) fmt.Fprintf(d.out, "Warning: version prune skipped — pin state could not be read: %v\n", pinsErr) } else { protected = append(protected, pins...) - pruned, err := d.docker.PruneVersions(ctx, cfg.App, cfg.KeepVersions, protected...) - switch { - case err != nil: - // Partial cleanup (A25): report what was left behind — - // the old path printed nothing when any removal failed, - // or reported failed removals as pruned. - fmt.Fprintf(d.out, "Warning: version prune incomplete: %v\n", err) + if explicit || d.defaultPruneDue(ctx, cfg.App, keep, protected) { + pruned, err := d.docker.PruneVersions(ctx, cfg.App, keep, protected...) if len(pruned) > 0 { + events.Record(ctx, d.exec, events.Event{App: cfg.App, Kind: events.KindReleasesPrune, Outcome: events.Outcome(err), Release: cfg.Version, + Detail: events.ErrDetail(map[string]any{"keep": keep, "pruned": pruned, "trigger": "deploy"}, err)}) + } + switch { + case err != nil: + // Partial cleanup (A25): report what was left behind — + // the old path printed nothing when any removal failed, + // or reported failed removals as pruned. + fmt.Fprintf(d.out, "Warning: version prune incomplete: %v\n", err) + if len(pruned) > 0 { + fmt.Fprintf(d.out, "Pruned %d superseded version(s): %s\n", len(pruned), strings.Join(pruned, ", ")) + } + case len(pruned) > 0: fmt.Fprintf(d.out, "Pruned %d superseded version(s): %s\n", len(pruned), strings.Join(pruned, ", ")) } - case len(pruned) > 0: - fmt.Fprintf(d.out, "Pruned %d superseded version(s): %s\n", len(pruned), strings.Join(pruned, ", ")) } } } diff --git a/internal/deploy/retention.go b/internal/deploy/retention.go new file mode 100644 index 0000000..522ea11 --- /dev/null +++ b/internal/deploy/retention.go @@ -0,0 +1,65 @@ +package deploy + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +// DefaultPruneNotice is how long automatic pruning under the default +// keep_versions waits after a deploy first finds versions beyond it, so +// `teploy server audit` and the deploy output warn before anything goes. An +// explicit keep_versions already asked for pruning and has no notice. +const DefaultPruneNotice = 7 * 24 * time.Hour + +// PruneNoticePath records when an app's notice period started. +func PruneNoticePath(app string) string { + return "/deployments/" + app + "/.prune-notice" +} + +// ParsePruneNotice reads a notice file's timestamp. +func ParsePruneNotice(data []byte) (time.Time, bool) { + t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(data))) + return t, err == nil +} + +// defaultPruneDue says whether a default-policy prune may run now, starting the +// notice period the first time there is something to prune. Anything it cannot +// establish means no prune this deploy. +func (d *Deployer) defaultPruneDue(ctx context.Context, app string, keep int, protected []string) bool { + containers, err := d.docker.ListContainers(ctx, app) + if err != nil { + fmt.Fprintf(d.out, "Warning: version prune skipped: %v\n", err) + return false + } + prunable := docker.PrunableVersions(containers, keep, protected...) + if len(prunable) == 0 { + return false + } + data, present, err := state.ReadRemoteFile(ctx, d.exec, PruneNoticePath(app)) + if err != nil { + fmt.Fprintf(d.out, "Warning: version prune skipped: %v\n", err) + return false + } + now := time.Now().UTC() + started, ok := ParsePruneNotice(data) + if !present || !ok { + started = now + if err := ssh.UploadAtomic(ctx, d.exec, strings.NewReader(now.Format(time.RFC3339)+"\n"), PruneNoticePath(app), "0600"); err != nil { + fmt.Fprintf(d.out, "Warning: version prune skipped: could not record the notice: %v\n", err) + return false + } + } + if due := started.Add(DefaultPruneNotice); now.Before(due) { + fmt.Fprintf(d.out, "Note: %d version(s) are beyond the default keep_versions (%d): %s. Deploys after %s remove them; set keep_versions (0 keeps everything) or pin what you need.\n", + len(prunable), config.DefaultKeepVersions, strings.Join(prunable, ", "), due.Format("2006-01-02")) + return false + } + return true +} diff --git a/internal/deploy/retention_test.go b/internal/deploy/retention_test.go new file mode 100644 index 0000000..73c8954 --- /dev/null +++ b/internal/deploy/retention_test.go @@ -0,0 +1,113 @@ +package deploy + +// The default keep_versions notice period (decision D3): an app that never +// set keep_versions is pruned to DefaultKeepVersions only after a recorded +// 7-day notice, so `server audit` and the deploy output can warn first. + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/ssh" +) + +func pruneDueExecutor(containers string, notice *string) *ssh.MockExecutor { + m := ssh.NewMockExecutor("192.0.2.10", + ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='myapp'", Output: containers}, + ) + if notice != nil { + m.Files["/deployments/myapp/.prune-notice"] = []byte(*notice) + } + return m +} + +func TestDefaultPruneDue(t *testing.T) { + threeVersions := strings.Join([]string{ + `{"ID":"c1","Names":"myapp-web-v3","Image":"myapp:v3","State":"running","Status":"Up","CreatedAt":"2026-05-28 21:30:00 -0700 PDT","Labels":"teploy.app=myapp,teploy.version=v3"}`, + `{"ID":"c2","Names":"myapp-web-v2","Image":"myapp:v2","State":"exited","Status":"Up","CreatedAt":"2026-05-28 21:00:00 -0700 PDT","Labels":"teploy.app=myapp,teploy.version=v2"}`, + `{"ID":"c3","Names":"myapp-web-v1","Image":"myapp:v1","State":"exited","Status":"Up","CreatedAt":"2026-05-28 20:00:00 -0700 PDT","Labels":"teploy.app=myapp,teploy.version=v1"}`, + }, "\n") + + t.Run("nothing beyond keep never starts a notice", func(t *testing.T) { + var out bytes.Buffer + m := pruneDueExecutor(threeVersions, nil) + d := &Deployer{exec: m, docker: docker.NewClient(m), out: &out} + if d.defaultPruneDue(context.Background(), "myapp", 5, nil) { + t.Fatal("prune due with nothing beyond keep") + } + if _, uploaded := m.Files["/deployments/myapp/.prune-notice"]; uploaded { + t.Error("a notice was recorded with nothing to prune") + } + }) + + t.Run("the first finding starts the notice and waits", func(t *testing.T) { + var out bytes.Buffer + m := pruneDueExecutor(threeVersions, nil) + d := &Deployer{exec: m, docker: docker.NewClient(m), out: &out} + if d.defaultPruneDue(context.Background(), "myapp", 1, []string{"v3", "v2"}) { + t.Fatal("prune due on the first sighting") + } + recorded := string(m.Files["/deployments/myapp/.prune-notice"]) + if !strings.Contains(recorded, time.Now().UTC().Format("2006-01-02")) { + t.Errorf("notice = %q, want today's timestamp", recorded) + } + if !strings.Contains(out.String(), "remove them automatically") && !strings.Contains(out.String(), "remove them") { + t.Errorf("deploy output must warn before the first automatic prune: %q", out.String()) + } + if !strings.Contains(out.String(), "set keep_versions") { + t.Errorf("the warning must say how to opt out: %q", out.String()) + } + }) + + t.Run("a notice inside the window waits and names the date", func(t *testing.T) { + var out bytes.Buffer + threeDaysAgo := time.Now().UTC().Add(-3 * 24 * time.Hour).Format(time.RFC3339) + m := pruneDueExecutor(threeVersions, &threeDaysAgo) + d := &Deployer{exec: m, docker: docker.NewClient(m), out: &out} + if d.defaultPruneDue(context.Background(), "myapp", 1, []string{"v3", "v2"}) { + t.Fatal("prune due three days into a seven-day notice") + } + if !strings.Contains(out.String(), time.Now().UTC().Add(4*24*time.Hour).Format("2006-01-02")) { + t.Errorf("the warning should name when pruning starts: %q", out.String()) + } + }) + + t.Run("an elapsed notice allows the prune", func(t *testing.T) { + var out bytes.Buffer + eightDaysAgo := time.Now().UTC().Add(-8 * 24 * time.Hour).Format(time.RFC3339) + m := pruneDueExecutor(threeVersions, &eightDaysAgo) + d := &Deployer{exec: m, docker: docker.NewClient(m), out: &out} + if !d.defaultPruneDue(context.Background(), "myapp", 1, []string{"v3", "v2"}) { + t.Fatal("prune still refused after the notice elapsed") + } + }) + + t.Run("an unreadable notice never prunes", func(t *testing.T) { + var out bytes.Buffer + m := ssh.NewMockExecutor("192.0.2.10", + ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='myapp'", Output: threeVersions}, + ssh.MockCommand{Match: "if [ ! -e '/deployments/myapp/.prune-notice'", Err: errors.New("stale file handle")}, + ) + d := &Deployer{exec: m, docker: docker.NewClient(m), out: &out} + if d.defaultPruneDue(context.Background(), "myapp", 1, []string{"v3", "v2"}) { + t.Fatal("a notice that cannot be read must not authorize a prune") + } + if !strings.Contains(out.String(), "skipped") { + t.Errorf("the refusal must be visible: %q", out.String()) + } + }) +} + +func TestParsePruneNotice(t *testing.T) { + if ts, ok := ParsePruneNotice([]byte("2026-11-01T00:00:00Z\n")); !ok || ts.UTC().Format(time.DateOnly) != "2026-11-01" { + t.Errorf("ParsePruneNotice = %v, %v", ts, ok) + } + if _, ok := ParsePruneNotice([]byte("yesterday")); ok { + t.Error("a non-timestamp notice must not parse") + } +} diff --git a/internal/docker/docker.go b/internal/docker/docker.go index e9f128b..8f27a8f 100644 --- a/internal/docker/docker.go +++ b/internal/docker/docker.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net" + "slices" "sort" "strconv" "strings" @@ -879,114 +880,100 @@ func ContainersByApp(containers []Container, app string) []Container { // a joined error describing every failed removal. Per-version image // removal stays best-effort (a shared image legitimately refuses). func (c *Client) PruneVersions(ctx context.Context, app string, keep int, protectedVersions ...string) ([]string, error) { - if keep < 0 { - keep = 0 - } containers, err := c.ListContainers(ctx, app) if err != nil { return nil, fmt.Errorf("listing containers for prune: %w", err) } + var pruned []string + var failures []error + for _, version := range PrunableVersions(containers, keep, protectedVersions...) { + names, images := versionArtifacts(containers, version) + // Force-remove containers in case any are still running. We + // already took ownership of cleanup; refusing to nuke a stray + // running container from an older version defeats the point. + // A version counts as pruned ONLY when every container removal + // succeeded (A25) — the old loop counted it regardless, so + // "Pruned N" could report versions whose containers were still + // running, hiding disk exhaustion and failed cleanup. + complete := true + for _, name := range names { + if _, err := c.exec.Run(ctx, "docker rm -f "+ssh.ShellQuote(name)); err != nil { + complete = false + failures = append(failures, fmt.Errorf("removing container %s (version %s): %w", name, version, err)) + } + } + // Best-effort image removal. Fails (silently) if another + // container or tag still references the image, which is the + // safe behavior. + for _, img := range images { + _, _ = c.exec.Run(ctx, "docker rmi "+ssh.ShellQuote(img)) + } + if complete { + pruned = append(pruned, version) + } + } + return pruned, errors.Join(failures...) +} - // Group containers by teploy.version label. Track newest creation - // timestamp + container names + image names per version. Containers - // without the label (e.g. caddy, postgres accessory) are ignored. - type vinfo struct { - newestCreated time.Time - containerNames []string - images map[string]struct{} +// PrunableVersions lists the versions PruneVersions would remove from one app's +// containers, newest first: everything outside the `keep` most recent and the +// protected set. Containers without a version label (accessories) never count. +func PrunableVersions(containers []Container, keep int, protectedVersions ...string) []string { + if keep < 0 { + keep = 0 } - versions := map[string]*vinfo{} + newest := map[string]time.Time{} for _, ct := range containers { v := ct.Labels["teploy.version"] - if v == "" { + if v == "" || ct.Labels["teploy.role"] == "accessory" { continue } - info, ok := versions[v] - if !ok { - info = &vinfo{images: map[string]struct{}{}} - versions[v] = info - } // Compare as instants, not strings. docker's CreatedAt is a localized // timestamp ("2006-01-02 15:04:05 -0700 MST"), so lexicographic ordering // is wrong across timezones / DST. On a parse failure, treat the version // as newest so cleanup never prunes something it can't date. created := parseDockerTime(ct.CreatedAt) - if created.After(info.newestCreated) { - info.newestCreated = created + if cur, ok := newest[v]; !ok || created.After(cur) { + newest[v] = created } - info.containerNames = append(info.containerNames, ct.Name) - if ct.Image != "" { - info.images[ct.Image] = struct{}{} - } - } - - if len(versions) == 0 { - return nil, nil } - - // Sort versions by recency (newest first). - type entry struct { - version string - info *vinfo - } - sorted := make([]entry, 0, len(versions)) - for v, info := range versions { - sorted = append(sorted, entry{v, info}) + sorted := make([]string, 0, len(newest)) + for v := range newest { + sorted = append(sorted, v) } sort.Slice(sorted, func(i, j int) bool { - if !sorted[i].info.newestCreated.Equal(sorted[j].info.newestCreated) { - return sorted[i].info.newestCreated.After(sorted[j].info.newestCreated) + if !newest[sorted[i]].Equal(newest[sorted[j]]) { + return newest[sorted[i]].After(newest[sorted[j]]) } // Deterministic tie-break when timestamps match (or both unparseable). - return sorted[i].version > sorted[j].version + return sorted[i] > sorted[j] }) - - // Build the protected set: explicit names + top-`keep` by recency. protect := map[string]bool{} for _, v := range protectedVersions { if v != "" { protect[v] = true } } - kept := 0 - for _, e := range sorted { - if kept < keep { - protect[e.version] = true - kept++ + var prunable []string + for i, v := range sorted { + if i >= keep && !protect[v] { + prunable = append(prunable, v) } } + return prunable +} - var pruned []string - var failures []error - for _, e := range sorted { - if protect[e.version] { +func versionArtifacts(containers []Container, version string) (names, images []string) { + for _, ct := range containers { + if ct.Labels["teploy.version"] != version || ct.Labels["teploy.role"] == "accessory" { continue } - // Force-remove containers in case any are still running. We - // already took ownership of cleanup; refusing to nuke a stray - // running container from an older version defeats the point. - // A version counts as pruned ONLY when every container removal - // succeeded (A25) — the old loop counted it regardless, so - // "Pruned N" could report versions whose containers were still - // running, hiding disk exhaustion and failed cleanup. - complete := true - for _, name := range e.info.containerNames { - if _, err := c.exec.Run(ctx, "docker rm -f "+ssh.ShellQuote(name)); err != nil { - complete = false - failures = append(failures, fmt.Errorf("removing container %s (version %s): %w", name, e.version, err)) - } - } - // Best-effort image removal. Fails (silently) if another - // container or tag still references the image, which is the - // safe behavior. - for img := range e.info.images { - _, _ = c.exec.Run(ctx, "docker rmi "+ssh.ShellQuote(img)) - } - if complete { - pruned = append(pruned, e.version) + names = append(names, ct.Name) + if ct.Image != "" && !slices.Contains(images, ct.Image) { + images = append(images, ct.Image) } } - return pruned, errors.Join(failures...) + return names, images } // EnsureNetwork creates the "teploy" Docker network if it doesn't already exist. diff --git a/internal/docker/prune_test.go b/internal/docker/prune_test.go index 0835c8b..6c72577 100644 --- a/internal/docker/prune_test.go +++ b/internal/docker/prune_test.go @@ -151,3 +151,53 @@ func mkPS(id, name, image, createdAt, version, process string) string { id, name, image, createdAt, process, version, ) } + +// TestPrunableVersions_Selection pins the selection PruneVersions and +// `releases prune`/`server audit` share: the `keep` most recent by newest +// container creation time, minus the protected set, accessories never counted. +func TestPrunableVersions_Selection(t *testing.T) { + ct := func(name, version, createdAt string) Container { + return Container{Name: name, Image: "myapp:" + version, CreatedAt: createdAt, + Labels: map[string]string{"teploy.app": "myapp", "teploy.version": version}} + } + containers := []Container{ + ct("myapp-web-v3", "v3", "2026-05-28 21:30:00 -0700 PDT"), + ct("myapp-web-v2", "v2", "2026-05-28 21:00:00 -0700 PDT"), + ct("myapp-web-v1", "v1", "2026-05-28 20:00:00 -0700 PDT"), + // An accessory carrying a version label must never be a candidate. + {Name: "myapp-db", Image: "postgres:16", CreatedAt: "2026-05-20 20:00:00 -0700 PDT", + Labels: map[string]string{"teploy.app": "myapp", "teploy.role": "accessory", "teploy.version": "v0"}}, + } + got := PrunableVersions(containers, 2) + if fmt.Sprint(got) != fmt.Sprint([]string{"v1"}) { + t.Errorf("keep 2 = %v, want [v1]", got) + } + got = PrunableVersions(containers, 0) + if fmt.Sprint(got) != fmt.Sprint([]string{"v3", "v2", "v1"}) { + t.Errorf("keep 0 = %v, want everything but accessories, newest first", got) + } + got = PrunableVersions(containers, 1, "v3") + if fmt.Sprint(got) != fmt.Sprint([]string{"v2", "v1"}) { + t.Errorf("keep 1 with v3 protected = %v, want [v2 v1]", got) + } + // Everything protected: nothing prunable whatever keep says. + if got = PrunableVersions(containers, 0, "v1", "v2", "v3"); len(got) != 0 { + t.Errorf("fully protected = %v, want nothing", got) + } +} + +// TestPrunableVersions_UnparseableDatesAreNewest pins the fail-safe ordering: +// a version whose creation time cannot be parsed is treated as newest, so +// cleanup never deletes something it cannot date. +func TestPrunableVersions_UnparseableDatesAreNewest(t *testing.T) { + containers := []Container{ + {Name: "myapp-web-v1", Image: "myapp:v1", CreatedAt: "not a date", + Labels: map[string]string{"teploy.app": "myapp", "teploy.version": "v1"}}, + {Name: "myapp-web-v2", Image: "myapp:v2", CreatedAt: "2026-05-28 21:00:00 -0700 PDT", + Labels: map[string]string{"teploy.app": "myapp", "teploy.version": "v2"}}, + } + got := PrunableVersions(containers, 1) + if fmt.Sprint(got) != fmt.Sprint([]string{"v2"}) { + t.Errorf("keep 1 = %v, want [v2]: the undatable v1 sorts newest and is kept", got) + } +} diff --git a/internal/events/events.go b/internal/events/events.go index a534529..28851d1 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -117,6 +117,7 @@ const ( KindBackupOwners = "backup_target.owners_clear" KindBackupAll = "backup.all" KindBackupPolicy = "backup_policy.set" + KindReleasesPrune = "releases.prune" ) // detailAllow is the per-kind allowlist of detail keys. Env and secret @@ -154,6 +155,7 @@ var detailAllow = map[string][]string{ KindBackupOwners: {"fingerprint", "force", "removed", "error"}, KindBackupAll: {"status", "targets_ok", "targets_failed", "omissions", "encrypted", "duration_ms", "heartbeats_ok", "heartbeats_failed", "warnings", "error"}, KindBackupPolicy: {"interval", "encrypted", "volumes", "adopted", "error"}, + KindReleasesPrune: {"keep", "pruned", "protected", "trigger", "error"}, } // maxErrorDetail bounds the "error" detail so a command's full stderr From 98c3878a6d25b6572751ab6f6ddffbce0a74aa92 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:38:44 -0700 Subject: [PATCH 2/6] =?UTF-8?q?feat(audit):=20teploy=20server=20audit=20?= =?UTF-8?q?=E2=80=94=20read-only=20findings=20with=20evidence=20and=20reme?= =?UTF-8?q?dies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `teploy server audit --json` reports what is left over on a server: stale and empty managed routes, managed routes with dead upstreams, orphan state, hand-written blocks (dead upstream, non-local address), versions beyond keep_versions, reclaimable build cache and dangling images, large writable layers, unmanaged containers, dangling volumes, a Caddyfile with no Caddy, scheduled jobs with no cron daemon, and disk pressure with attribution. Every finding carries kind:subject (stable), severity, evidence, and a remedy {argv, destructive, confirm} — argv null where nothing the CLI does is safe (hand-written config, other people's containers, volumes). Remedies are offered only where a command exists: route remove for a proven-dead route, redeploy (confirm) for an app's own dead upstream, remove -y (confirm) for orphan state, releases prune (confirm) for versions beyond policy, server prune for build cache/dangling images. The audit only reads: every remote command is read-only (pinned by a test over a recording executor: no mutating verb, no docker subcommand outside the read set, no upload). A source that could not be read shows as unknown in checks[] and yields no finding — never a clean bill; with --json the exit code is 0 whenever the envelope was produced, read state. caddy.ManagedRegions/RegionHasWebhook/RegionRoots expose the managed region view InspectRoutes already had; domains.ResolveNames is the extracted in-Caddy-network resolver, shared with domains check; doctor's disk remediations now point at server audit instead of a releases prune command that did not exist. --- internal/caddy/inspect.go | 56 ++ internal/cli/doctor.go | 4 +- internal/cli/server.go | 1 + internal/cli/server_audit.go | 1033 +++++++++++++++++++++++++++++ internal/cli/server_audit_test.go | 846 +++++++++++++++++++++++ internal/domains/domains.go | 63 +- 6 files changed, 1979 insertions(+), 24 deletions(-) create mode 100644 internal/cli/server_audit.go create mode 100644 internal/cli/server_audit_test.go diff --git a/internal/caddy/inspect.go b/internal/caddy/inspect.go index 8e5cab0..f16189d 100644 --- a/internal/caddy/inspect.go +++ b/internal/caddy/inspect.go @@ -98,6 +98,62 @@ func (e *unterminatedRegionError) Error() string { return "the managed block for " + e.app + " has no end marker" } +// ManagedRegion is one app's marker-delimited region of a Caddyfile. +type ManagedRegion struct { + App string + // Text is the region including its markers, as ManagedRegionHash expects. + Text string + // Sites is empty for a region that holds no site block. + Sites []RouteSite +} + +// ManagedRegions lists a Caddyfile's managed regions in file order. An +// unparseable file is an error, as for InspectRoutes. +func ManagedRegions(content string) ([]ManagedRegion, error) { + sites, err := InspectRoutes(content) + if err != nil { + return nil, err + } + var regions []ManagedRegion + seen := map[string]bool{} + for _, line := range strings.Split(content, "\n") { + app, ok := beginMarkerApp(line) + if !ok || seen[app] { + continue + } + seen[app] = true + region := ManagedRegion{App: app, Text: extractManagedRegion(content, app)} + for _, site := range sites { + if site.Owner == app { + region.Sites = append(region.Sites, site) + } + } + regions = append(regions, region) + } + return regions, nil +} + +// RegionHasWebhook reports whether a region carries the autodeploy webhook route. +func RegionHasWebhook(region string) bool { + return strings.Contains(region, "# TEPLOY WEBHOOK BEGIN") +} + +// RegionRoots lists the paths a region's root directives serve files from. +func RegionRoots(region string) []string { + var roots []string + for _, line := range strings.Split(region, "\n") { + f := strings.Fields(line) + if len(f) < 2 || f[0] != "root" { + continue + } + path := f[len(f)-1] + if !slices.Contains(roots, path) { + roots = append(roots, path) + } + } + return roots +} + // RegionHosts lists the hosts a managed region serves. func RegionHosts(region string) []string { body := make([]string, 0) diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index 7cb6acb..c4bd413 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -451,12 +451,12 @@ func doctorDiskCheck(ctx context.Context, exec ssh.Executor) doctorCheck { case avail < doctorDiskFailAvailable: return doctorCheck{ Name: "disk", Result: doctorFail, Detail: detail, - Remediation: "free disk space on / (docker system prune, teploy releases/preview prune, old backups) — deploys fail mid-flight below 2 GiB", + Remediation: "free disk space on / — deploys fail mid-flight below 2 GiB. `teploy server audit ` names what takes the space and what removes it", } case avail < doctorDiskWarnAvailable || pct >= doctorDiskWarnUsedPct: return doctorCheck{ Name: "disk", Result: doctorWarn, Detail: detail, - Remediation: "free disk space on / before the next deploy (docker system prune, teploy releases prune)", + Remediation: "free disk space on / before the next deploy. `teploy server audit ` names what takes the space and what removes it", } } return doctorCheck{Name: "disk", Result: doctorOK, Detail: detail} diff --git a/internal/cli/server.go b/internal/cli/server.go index d0dd928..6037a64 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -29,6 +29,7 @@ func newServerCmd(flags *Flags) *cobra.Command { cmd.AddCommand(newServerStatusCmd(flags)) cmd.AddCommand(newServerBackupTargetCmd(flags)) cmd.AddCommand(newServerBackupPolicyCmd(flags)) + cmd.AddCommand(newServerAuditCmd(flags)) return cmd } diff --git a/internal/cli/server_audit.go b/internal/cli/server_audit.go new file mode 100644 index 0000000..dfd6f50 --- /dev/null +++ b/internal/cli/server_audit.go @@ -0,0 +1,1033 @@ +package cli + +// `teploy server audit `: what is left over on a server, with evidence, +// and the one command that fixes it where a command can do so safely. Read-only: +// every remote command here only reads (a test pins that). What could not be +// read is named in checks[] and yields no finding: it is never reported clean. + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "os" + "os/signal" + "regexp" + "slices" + "sort" + "strconv" + "strings" + "time" + + "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/backup" + "github.com/useteploy/teploy/internal/caddy" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/deploy" + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/domains" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +// Finding kinds. Stable identifiers: consumers code against them. +const ( + findingStaleRoute = "stale_route" + findingEmptyRegion = "empty_managed_region" + findingDeadUpstream = "managed_route_dead_upstream" + findingOrphanState = "orphan_state" + findingManualDead = "manual_route_dead_upstream" + findingManualNonLocal = "manual_route_nonlocal_address" + findingVersionsBeyond = "versions_beyond_policy" + findingBuildArtifacts = "reclaimable_build_artifacts" + findingLargeLayer = "large_writable_layer" + findingUnmanagedLarge = "unmanaged_container" + findingDanglingVolumes = "dangling_volumes" + findingCaddyfileNoCaddy = "caddyfile_without_caddy" + findingSchedulerMissing = "scheduler_missing" + findingDiskPressure = "disk_pressure" +) + +const ( + severityCritical = "critical" + severityWarning = "warning" + severityInfo = "info" + + auditCheckOK = "ok" + auditCheckUnknown = "unknown" + + auditStateOK = "ok" + auditStatePartial = "partial" + auditStateUnreachable = "unreachable" + + auditLargeLayerBytes = 5_000_000_000 + auditLargeUnmanagedBytes = 1_000_000_000 + auditLargeCacheBytes = 1_000_000_000 + auditTimeout = 5 * time.Minute +) + +// auditRemedy is what fixes a finding. Argv is null when nothing here can do it +// safely (report only); argv[0] is "teploy". Dash adds its own --actor and --json. +type auditRemedy struct { + Argv []string `json:"argv"` + Destructive bool `json:"destructive"` + Confirm bool `json:"confirm"` +} + +// auditFinding is one thing left over. ID is kind:subject, stable across runs +// so a finding can be dismissed; sizes and evidence may change under it. +type auditFinding struct { + ID string `json:"id"` + Kind string `json:"kind"` + Severity string `json:"severity"` + Subject string `json:"subject"` + // App is set when the finding belongs to one app: the name a destructive + // confirmation asks for. + App string `json:"app,omitempty"` + Evidence string `json:"evidence"` + SizeBytes int64 `json:"size_bytes,omitempty"` + // Related names the findings that would free space on a disk_pressure. + Related []string `json:"related,omitempty"` + Remedy auditRemedy `json:"remedy"` +} + +// auditCheck says whether one source of evidence was read. An unknown check +// means findings depending on it are missing, not absent. +type auditCheck struct { + Name string `json:"name"` + Status string `json:"status"` + Detail string `json:"detail,omitempty"` +} + +type auditDiskUse struct { + Name string `json:"name"` + SizeBytes int64 `json:"size_bytes"` +} + +// auditDisk is the root filesystem and what takes the space. +type auditDisk struct { + Path string `json:"path"` + AvailableBytes int64 `json:"available_bytes"` + UsedPercent int `json:"used_percent"` + Attribution []auditDiskUse `json:"attribution"` +} + +// serverAuditEnvelope is `server audit --json` (contracts/schema/ +// server-audit-envelope.schema.json). +type serverAuditEnvelope struct { + MachineInterface int `json:"machine_interface"` + Server string `json:"server"` + Host string `json:"host"` + AuditedAt time.Time `json:"audited_at"` + State string `json:"state"` + Error string `json:"error,omitempty"` + Checks []auditCheck `json:"checks"` + Disk *auditDisk `json:"disk"` + Findings []auditFinding `json:"findings"` +} + +func newServerAuditCmd(flags *Flags) *cobra.Command { + return &cobra.Command{ + Use: "audit ", + Short: "Report what is left over on a server, and what fixes it (read-only)", + Long: `Reports stale and dead routes, state with no containers, versions beyond +keep_versions, build cache, dangling images and volumes, large writable layers, +large containers teploy does not manage, a Caddyfile with no Caddy, a missing +cron daemon and disk pressure, each with its evidence. + +Changes nothing. Where a command can fix a finding safely, remedy.argv is that +command; hand-written Caddy blocks, volumes and other people's containers are +report only. A source that could not be read shows as unknown in checks[], and +the findings that depend on it are missing rather than clean. With --json the +command exits 0 whenever the envelope was produced; read state.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runServerAudit(flags, args[0], cmd.OutOrStdout()) + }, + } +} + +func runServerAudit(flags *Flags, server string, out io.Writer) error { + host, user, key, err := config.ResolveServer(server, flags.Host, flags.User, flags.Key) + if err != nil { + return fmt.Errorf("resolving server %q: %w", server, err) + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + ctx, stop := context.WithTimeout(ctx, auditTimeout) + defer stop() + + now := time.Now().UTC() + var env serverAuditEnvelope + dialCtx, dialCancel := context.WithTimeout(ctx, domainsConnectTimeout) + exec, err := ssh.Connect(dialCtx, ssh.ConnectConfig{Host: host, User: user, KeyPath: key}) + dialCancel() + if err != nil { + env = serverAuditEnvelope{MachineInterface: MachineInterface, Server: server, Host: host, AuditedAt: now, + State: auditStateUnreachable, Error: strings.Join(strings.Fields(err.Error()), " "), Checks: []auditCheck{}, Findings: []auditFinding{}} + } else { + defer exec.Close() + env = auditServer(ctx, exec, server, host, now) + } + if flags.JSON { + return json.NewEncoder(out).Encode(env) + } + return printServerAudit(out, env) +} + +func printServerAudit(out io.Writer, env serverAuditEnvelope) error { + if env.State == auditStateUnreachable { + return fmt.Errorf("%s could not be audited: %s", env.Server, env.Error) + } + for _, f := range env.Findings { + fmt.Fprintf(out, "%-8s %s\n %s\n", f.Severity, f.ID, f.Evidence) + if len(f.Remedy.Argv) > 0 { + fmt.Fprintf(out, " fix: %s\n", strings.Join(f.Remedy.Argv, " ")) + } + } + if len(env.Findings) == 0 { + fmt.Fprintf(out, "No findings on %s\n", env.Server) + } + for _, c := range env.Checks { + if c.Status != auditCheckOK { + fmt.Fprintf(out, "\nUnknown, not clean: %s: %s\n", c.Name, c.Detail) + } + } + return nil +} + +// auditFacts is everything read from a server. Each *Known flag separates "read, +// and there is none" from "could not read". +type auditFacts struct { + server string + now time.Time + + apps map[string]*state.AppState // present and not tombstoned + appErrs map[string]string // an app whose state could not be read + appsKnown bool + + caddyfile string // present | absent | unknown + regions []caddy.ManagedRegion + manualSites []caddy.RouteSite + + containers []docker.Container + containersKnown bool + sizes map[string]int64 + sizesKnown bool + + resolves map[string]string + roots map[string]string + addrs []string + addrsKnown bool + + pins map[string][]string + pinErrs map[string]string + notices map[string]time.Time + noticeErrs map[string]string + df *auditDisk + usage map[string]dockerUsage + usageKnown bool + danglingIDs []string + imagesKnown bool + volumes []string + volsKnown bool + scheduler *backup.SchedulerStatus + + checks []auditCheck +} + +type dockerUsage struct { + size, reclaimable int64 +} + +func (f *auditFacts) check(name string, err error) { + if err != nil { + f.checks = append(f.checks, auditCheck{Name: name, Status: auditCheckUnknown, Detail: strings.Join(strings.Fields(err.Error()), " ")}) + return + } + f.checks = append(f.checks, auditCheck{Name: name, Status: auditCheckOK}) +} + +// auditServer runs every read on one connected server and judges the result. +func auditServer(ctx context.Context, exec ssh.Executor, server, host string, now time.Time) serverAuditEnvelope { + f := collectAuditFacts(ctx, exec, server, now) + env := serverAuditEnvelope{MachineInterface: MachineInterface, Server: server, Host: host, AuditedAt: now, + State: auditStateOK, Checks: f.checks, Disk: f.df, Findings: analyzeAudit(f)} + for _, c := range f.checks { + if c.Status != auditCheckOK { + env.State = auditStatePartial + } + } + return env +} + +func collectAuditFacts(ctx context.Context, exec ssh.Executor, server string, now time.Time) *auditFacts { + f := &auditFacts{server: server, now: now, apps: map[string]*state.AppState{}, appErrs: map[string]string{}, + sizes: map[string]int64{}, resolves: map[string]string{}, roots: map[string]string{}, + pins: map[string][]string{}, pinErrs: map[string]string{}, notices: map[string]time.Time{}, noticeErrs: map[string]string{}, + usage: map[string]dockerUsage{}} + + f.check("apps", f.readApps(ctx, exec)) + f.check("caddyfile", f.readCaddyfile(ctx, exec)) + containers, err := docker.NewClient(exec).ListAllContainers(ctx) + f.containers, f.containersKnown = containers, err == nil + f.check("containers", err) + f.check("container_sizes", f.readSizes(ctx, exec)) + f.check("upstreams", f.readUpstreams(ctx, exec)) + f.check("static_roots", f.readRoots(ctx, exec)) + f.check("addresses", f.readAddrs(ctx, exec)) + f.check("docker_usage", f.readUsage(ctx, exec)) + f.check("dangling_images", f.readDanglingImages(ctx, exec)) + f.check("dangling_volumes", f.readDanglingVolumes(ctx, exec)) + f.check("disk", f.readDisk(ctx, exec)) + st, err := backup.ProbeScheduler(ctx, exec) + if err == nil { + f.scheduler = &st + } + f.check("scheduler", err) + f.check("retention", f.readRetention(ctx, exec)) + return f +} + +func (f *auditFacts) readApps(ctx context.Context, exec ssh.Executor) error { + out, err := exec.Run(ctx, domains.AppListCommand) + if err != nil { + return err + } + var names []string + for _, n := range strings.Fields(out) { + if config.ValidateName(n) == nil { + names = append(names, n) + } + } + snaps, err := state.ReadAppSnapshots(ctx, exec, names) + if err != nil { + return err + } + f.appsKnown = true + var bad []string + for _, snap := range snaps { + switch { + case snap.StateErr != nil: + f.appErrs[snap.App] = snap.StateErr.Error() + bad = append(bad, snap.App) + case snap.State != nil && !snap.State.Tombstone: + f.apps[snap.App] = snap.State + } + } + if len(bad) > 0 { + return fmt.Errorf("could not read the state of %s", strings.Join(bad, ", ")) + } + return nil +} + +func (f *auditFacts) readCaddyfile(ctx context.Context, exec ssh.Executor) error { + f.caddyfile = "unknown" + content, present, err := caddy.NewClient(exec).ReadCaddyfile(ctx) + if err != nil { + return err + } + if !present { + f.caddyfile = "absent" + return nil + } + regions, err := caddy.ManagedRegions(content) + if err != nil { + return err + } + sites, err := caddy.InspectRoutes(content) + if err != nil { + return err + } + f.caddyfile, f.regions = "present", regions + for _, site := range sites { + if site.Owner == "" { + f.manualSites = append(f.manualSites, site) + } + } + return nil +} + +// auditSizesCommand is bounded: --size walks every writable layer. +const auditSizesCommand = `t=; command -v timeout >/dev/null 2>&1 && t='timeout 120'; $t docker ps --all --size --format '{"Names":{{json .Names}},"Size":{{json .Size}}}'` + +func (f *auditFacts) readSizes(ctx context.Context, exec ssh.Executor) error { + out, err := exec.Run(ctx, auditSizesCommand) + if err != nil { + return err + } + for _, line := range doctorLines(out) { + var row struct{ Names, Size string } + if err := json.Unmarshal([]byte(line), &row); err != nil { + return fmt.Errorf("unrecognized docker ps output: %w", err) + } + size, ok := parseDockerSize(strings.SplitN(row.Size, " (", 2)[0]) + if !ok { + return fmt.Errorf("unrecognized container size %q", row.Size) + } + f.sizes[row.Names] = size + } + f.sizesKnown = true + return nil +} + +var dockerSizeRE = regexp.MustCompile(`^([0-9.]+)\s*([A-Za-z]*)$`) + +// parseDockerSize reads docker's human sizes (powers of 1000: go-units). +func parseDockerSize(s string) (int64, bool) { + m := dockerSizeRE.FindStringSubmatch(strings.TrimSpace(s)) + if m == nil { + return 0, false + } + n, err := strconv.ParseFloat(m[1], 64) + if err != nil { + return 0, false + } + mult := map[string]float64{"": 1, "B": 1, "kB": 1e3, "KB": 1e3, "MB": 1e6, "GB": 1e9, "TB": 1e12, "PB": 1e15, + "KiB": 1 << 10, "MiB": 1 << 20, "GiB": 1 << 30, "TiB": 1 << 40}[m[2]] + if mult == 0 { + return 0, false + } + return int64(n * mult), true +} + +func (f *auditFacts) caddyRunning() bool { + return f.containersKnown && f.caddyContainerState() == "running" +} + +// caddyContainerState is the caddy container's state, "" when there is none. +func (f *auditFacts) caddyContainerState() string { + for _, c := range f.containers { + if c.Name == "caddy" { + return c.State + } + } + return "" +} + +func (f *auditFacts) readUpstreams(ctx context.Context, exec ssh.Executor) error { + wanted := map[string]bool{} + add := func(sites []caddy.RouteSite) { + for _, site := range sites { + for _, dial := range site.Upstreams { + if name, _ := domains.UpstreamHost(dial); name != "" { + wanted[name] = true + } + } + } + } + add(f.manualSites) + for _, r := range f.regions { + add(r.Sites) + } + if len(wanted) == 0 { + return nil + } + if !f.caddyRunning() { + return fmt.Errorf("no running caddy container to resolve %d upstream name(s) in", len(wanted)) + } + names := make([]string, 0, len(wanted)) + for n := range wanted { + names = append(names, n) + } + answers, err := domains.ResolveNames(ctx, exec, names) + f.resolves = answers + if err != nil { + return err + } + var missing []string + for _, n := range names { + if answers[n] == "" { + missing = append(missing, n) + } + } + if len(missing) > 0 { + sort.Strings(missing) + return fmt.Errorf("could not resolve %s", strings.Join(missing, ", ")) + } + return nil +} + +// readRoots checks the file roots of managed regions, the only thing besides an +// upstream that can make a block with no app state still serve something. +func (f *auditFacts) readRoots(ctx context.Context, exec ssh.Executor) error { + var firstErr error + for _, r := range f.regions { + for _, root := range caddy.RegionRoots(r.Text) { + if _, seen := f.roots[root]; seen { + continue + } + verdict, err := readRootExists(ctx, exec, root) + if err != nil && firstErr == nil { + firstErr = err + } + f.roots[root] = verdict + } + } + return firstErr +} + +func readRootExists(ctx context.Context, exec ssh.Executor, root string) (string, error) { + if strings.Contains(root, "{") { + return domains.ResolvesUnknown, nil // a placeholder: only Caddy can expand it + } + out, err := exec.Run(ctx, "if [ -e "+ssh.ShellQuote(root)+" ]; then echo yes; else echo no; fi") + if err != nil { + return domains.ResolvesUnknown, err + } + return strings.TrimSpace(out), nil +} + +func (f *auditFacts) readAddrs(ctx context.Context, exec ssh.Executor) error { + out, err := exec.Run(ctx, "hostname -I") + if err != nil { + return err + } + f.addrs = append(strings.Fields(out), exec.Host()) + f.addrsKnown = true + return nil +} + +func (f *auditFacts) readUsage(ctx context.Context, exec ssh.Executor) error { + out, err := exec.Run(ctx, `docker system df --format '{{json .}}'`) + if err != nil { + return err + } + for _, line := range doctorLines(out) { + var row struct{ Type, Size, Reclaimable string } + if err := json.Unmarshal([]byte(line), &row); err != nil { + return fmt.Errorf("unrecognized docker system df output: %w", err) + } + size, ok1 := parseDockerSize(row.Size) + reclaim, ok2 := parseDockerSize(strings.SplitN(row.Reclaimable, " (", 2)[0]) + if !ok1 || !ok2 { + return fmt.Errorf("unrecognized docker system df row %q", line) + } + f.usage[row.Type] = dockerUsage{size, reclaim} + } + f.usageKnown = len(f.usage) > 0 + if !f.usageKnown { + return fmt.Errorf("docker system df returned nothing") + } + return nil +} + +func (f *auditFacts) readDanglingImages(ctx context.Context, exec ssh.Executor) error { + out, err := exec.Run(ctx, "docker images -q -f dangling=true") + if err != nil { + return err + } + f.danglingIDs, f.imagesKnown = doctorLines(out), true + return nil +} + +func (f *auditFacts) readDanglingVolumes(ctx context.Context, exec ssh.Executor) error { + out, err := exec.Run(ctx, "docker volume ls -q -f dangling=true") + if err != nil { + return err + } + f.volumes, f.volsKnown = doctorLines(out), true + return nil +} + +func (f *auditFacts) readDisk(ctx context.Context, exec ssh.Executor) error { + out, err := exec.Run(ctx, "df -B1 -P /") + if err != nil { + return err + } + avail, pct, ok := parseDoctorDisk(out) + if !ok { + return fmt.Errorf("could not parse df output") + } + disk := &auditDisk{Path: "/", AvailableBytes: int64(avail), UsedPercent: pct, Attribution: []auditDiskUse{}} + for _, row := range []struct{ name, key string }{{"images", "Images"}, {"container_layers", "Containers"}, {"volumes", "Local Volumes"}, {"build_cache", "Build Cache"}} { + if u, ok := f.usage[row.key]; ok { + disk.Attribution = append(disk.Attribution, auditDiskUse{row.name, u.size}) + } + } + // Only when it matters: du walks the tree. + if diskUnderPressure(disk) { + if out, err := exec.Run(ctx, `t=; command -v timeout >/dev/null 2>&1 && t='timeout 60'; $t du -sxb /deployments 2>/dev/null`); err == nil { + if fields := strings.Fields(out); len(fields) == 2 { + if n, err := strconv.ParseInt(fields[0], 10, 64); err == nil { + disk.Attribution = append(disk.Attribution, auditDiskUse{"deployments", n}) + } + } + } + } + f.df = disk + return nil +} + +func diskUnderPressure(d *auditDisk) bool { + return d.AvailableBytes < doctorDiskWarnAvailable || d.UsedPercent >= doctorDiskWarnUsedPct +} + +// readRetention reads pins and notices for the apps with versions beyond policy. +func (f *auditFacts) readRetention(ctx context.Context, exec ssh.Executor) error { + if !f.containersKnown { + return fmt.Errorf("container list unavailable") + } + var firstErr error + for app, st := range f.apps { + keep, explicit, keepAll := appliedKeepPolicy(st) + if keepAll || st.DeploymentType == "static" { + continue + } + if len(docker.PrunableVersions(docker.ContainersByApp(f.containers, app), keep, retentionProtected(st, nil, docker.ContainersByApp(f.containers, app))...)) == 0 { + continue + } + pins, err := state.ReadPins(ctx, exec, app) + if err != nil { + f.pinErrs[app] = err.Error() + if firstErr == nil { + firstErr = err + } + continue + } + f.pins[app] = pins + if explicit { + continue + } + data, present, err := state.ReadRemoteFile(ctx, exec, deploy.PruneNoticePath(app)) + if err != nil { + f.noticeErrs[app] = err.Error() + if firstErr == nil { + firstErr = err + } + continue + } + if t, ok := deploy.ParsePruneNotice(data); present && ok { + f.notices[app] = t + } + } + return firstErr +} + +// appliedKeepPolicy reads the retention the running release was deployed with. +// A manifest with no keep_versions, or the 0 an unset field records, is the +// default: an explicit 0 records keep_all_versions. +func appliedKeepPolicy(st *state.AppState) (keep int, explicit, keepAll bool) { + var m struct { + Container struct { + KeepVersions int `json:"keep_versions"` + KeepAll bool `json:"keep_all_versions"` + } `json:"container"` + } + if len(st.AppliedManifest) > 0 { + _ = json.Unmarshal(st.AppliedManifest, &m) + } + switch { + case m.Container.KeepAll: + return 0, true, true + case m.Container.KeepVersions > 0: + return m.Container.KeepVersions, true, false + } + return config.DefaultKeepVersions, false, false +} + +// retentionProtected is what no prune may touch: current, previous, pinned, and +// any version with a running container. +func retentionProtected(st *state.AppState, pins []string, containers []docker.Container) []string { + protected := append([]string{st.CurrentHash, st.PreviousHash}, pins...) + for _, c := range containers { + if v := c.Labels["teploy.version"]; v != "" && c.State == "running" && !slices.Contains(protected, v) { + protected = append(protected, v) + } + } + return protected +} + +// ownedContainers are the containers that make an app's name still in use: its +// own, its previews', and unlabelled ones named after it (older deploys). +func ownedContainers(containers []docker.Container, app string) []docker.Container { + var owned []docker.Container + for _, c := range containers { + label := c.Labels["teploy.app"] + switch { + case label == app, strings.HasPrefix(label, app+"-preview-"): + owned = append(owned, c) + case label == "" && strings.HasPrefix(c.Name, app+"-"): + owned = append(owned, c) + } + } + return owned +} + +// routeFacts is what deciding on one managed route needs: the same judgement for +// the audit and for `route remove`, so the remedy offered is the one that runs. +type routeFacts struct { + app string + region string + sites []caddy.RouteSite + stateKnown bool + st *state.AppState + known bool // containers were listed + containers []docker.Container + resolves map[string]string + roots map[string]string +} + +// removalRefusal says why a route must not be removed, "" when it may be. Only +// what is proven dead can go: an unreadable source refuses like a live app does. +func removalRefusal(r routeFacts) string { + if !r.stateKnown { + return "its app state could not be read" + } + if !r.known { + return "its containers could not be listed" + } + if caddy.RegionHasWebhook(r.region) { + return "it carries the autodeploy webhook route; the next deploy retires it" + } + if r.st != nil && routedByCaddy(r.st) { + if r.st.DeploymentType == "static" { + return "it belongs to a live static app" + } + for _, c := range ownedContainers(r.containers, r.app) { + if c.Labels["teploy.role"] != "accessory" { + return fmt.Sprintf("it belongs to a live app: %s has container %s (%s)", r.app, c.Name, c.State) + } + } + } + if r.st == nil { + for _, c := range ownedContainers(r.containers, r.app) { + return fmt.Sprintf("container %s (%s) still carries this app's name", c.Name, c.State) + } + } + for _, site := range r.sites { + for _, dial := range site.Upstreams { + switch dialVerdict(dial, r.resolves) { + case domains.ResolvesNo: + case domains.ResolvesYes: + return "upstream " + dial + " still resolves" + default: + return "upstream " + dial + " could not be proven dead" + } + } + } + for _, root := range caddy.RegionRoots(r.region) { + switch r.roots[root] { + case "no": + case "yes": + return "it serves files from " + root + ", which exists" + default: + return "whether " + root + " exists could not be read" + } + } + return "" +} + +func routedByCaddy(st *state.AppState) bool { + return st.IngressMode == "" || st.IngressMode == config.IngressCaddy +} + +func dialVerdict(dial string, resolves map[string]string) string { + name, verdict := domains.UpstreamHost(dial) + if name == "" { + return verdict + } + if v := resolves[name]; v != "" { + return v + } + return domains.ResolvesUnknown +} + +func dialsByVerdict(sites []caddy.RouteSite, resolves map[string]string, verdict string) []string { + var dials []string + for _, site := range sites { + for _, dial := range site.Upstreams { + if dialVerdict(dial, resolves) == verdict && !slices.Contains(dials, dial) { + dials = append(dials, dial) + } + } + } + return dials +} + +func siteHosts(sites []caddy.RouteSite) []string { + var hosts []string + for _, site := range sites { + hosts = append(hosts, site.Hosts...) + } + return hosts +} + +func analyzeAudit(f *auditFacts) []auditFinding { + findings := []auditFinding{} + add := func(kind, severity, subject, app, evidence string, size int64, remedy auditRemedy) { + findings = append(findings, auditFinding{ID: kind + ":" + subject, Kind: kind, Severity: severity, Subject: subject, + App: app, Evidence: evidence, SizeBytes: size, Remedy: remedy}) + } + teploy := func(args ...string) []string { return append([]string{"teploy"}, args...) } + + // Managed routes. An app whose state could not be read is not judged. + if f.appsKnown { + for _, r := range f.regions { + if _, bad := f.appErrs[r.App]; bad { + continue + } + st := f.apps[r.App] + facts := routeFacts{app: r.App, region: r.Text, sites: r.Sites, stateKnown: true, st: st, + known: f.containersKnown, containers: f.containers, resolves: f.resolves, roots: f.roots} + refusal := removalRefusal(facts) + removeRemedy := auditRemedy{} + note := "" + if refusal == "" { + removeRemedy.Argv = teploy("route", "remove", "--app", r.App, "--host", f.server) + } else { + note = "; not removable: " + refusal + } + hosts := strings.Join(siteHosts(r.Sites), ", ") + switch { + case len(r.Sites) == 0: + add(findingEmptyRegion, severityInfo, r.App, r.App, "the managed region holds no site block"+note, 0, removeRemedy) + case st == nil: + dead := dialsByVerdict(r.Sites, f.resolves, domains.ResolvesNo) + live := dialsByVerdict(r.Sites, f.resolves, domains.ResolvesYes) + // A live preview has a route and no app state of its own. + if strings.Contains(r.App, "-preview-") && len(live) > 0 && len(dead) == 0 { + continue + } + evidence := fmt.Sprintf("no app %s on this server, but its managed block serves %s", r.App, hosts) + if len(dead) > 0 { + evidence += " and routes to " + strings.Join(dead, ", ") + ", which does not resolve" + } + add(findingStaleRoute, severityWarning, r.App, r.App, evidence+note, 0, removeRemedy) + case !routedByCaddy(st): + add(findingStaleRoute, severityWarning, r.App, r.App, + fmt.Sprintf("%s is deployed with ingress %s, but a managed block still serves %s%s", r.App, st.IngressMode, hosts, note), 0, removeRemedy) + default: + dead := dialsByVerdict(r.Sites, f.resolves, domains.ResolvesNo) + if len(dead) == 0 { + continue + } + severity, remedy, evidence := deadRouteVerdict(f, r, st, dead, hosts) + add(findingDeadUpstream, severity, r.App, r.App, evidence, 0, remedy) + } + } + } + + // Hand-written blocks are never edited: report only. + for _, site := range f.manualSites { + subject := strings.Join(site.Hosts, ",") + if dead := dialsByVerdict([]caddy.RouteSite{site}, f.resolves, domains.ResolvesNo); len(dead) > 0 { + add(findingManualDead, severityWarning, subject, "", fmt.Sprintf("hand-written block for %s routes to %s, which does not resolve", subject, strings.Join(dead, ", ")), 0, auditRemedy{}) + } + if !f.addrsKnown { + continue + } + for _, host := range site.Hosts { + if ip := net.ParseIP(host); ip != nil && !ip.IsLoopback() && !slices.Contains(f.addrs, ip.String()) { + add(findingManualNonLocal, severityInfo, subject, "", fmt.Sprintf("hand-written block is addressed to %s, which is not an address of this server", host), 0, auditRemedy{}) + break + } + } + } + + if f.caddyfile == "present" && f.containersKnown && f.caddyContainerState() != "running" { + how := "no caddy container" + if s := f.caddyContainerState(); s != "" { + how = "the caddy container is " + s + } + severity := severityInfo + routes := len(f.regions) + len(f.manualSites) + if routes > 0 { + severity = severityWarning + } + add(findingCaddyfileNoCaddy, severity, "caddy", "", fmt.Sprintf("/deployments/caddy/Caddyfile exists with %d route block(s), but %s", routes, how), 0, auditRemedy{}) + } + + // State with no containers. + if f.appsKnown && f.containersKnown { + for _, app := range sortedAppNames(f.apps) { + st := f.apps[app] + if st.DeploymentType == "static" || len(ownedContainers(f.containers, app)) > 0 { + continue + } + add(findingOrphanState, severityWarning, app, app, + fmt.Sprintf("%s has deploy state (release %s) but no containers; removing keeps its volumes, accessory data and credentials", app, st.CurrentHash), 0, + auditRemedy{Argv: teploy("remove", "--app", app, "--host", f.server, "-y"), Destructive: true, Confirm: true}) + } + } + + // Versions beyond policy. + if f.appsKnown && f.containersKnown { + for _, app := range sortedAppNames(f.apps) { + st := f.apps[app] + keep, explicit, keepAll := appliedKeepPolicy(st) + if keepAll || st.DeploymentType == "static" { + continue + } + if _, bad := f.pinErrs[app]; bad { + continue + } + owned := docker.ContainersByApp(f.containers, app) + prunable := docker.PrunableVersions(owned, keep, retentionProtected(st, f.pins[app], owned)...) + if len(prunable) == 0 { + continue + } + var size int64 + for _, c := range owned { + if slices.Contains(prunable, c.Labels["teploy.version"]) { + size += f.sizes[c.Name] + } + } + evidence := fmt.Sprintf("%d version(s) beyond keep_versions %d: %s", len(prunable), keep, strings.Join(prunable, ", ")) + severity := severityInfo + if explicit { + evidence += "; the next deploy removes them" + } else { + severity = severityWarning + evidence += "; keep_versions is not set, so " + defaultPruneWarning(f, app) + } + add(findingVersionsBeyond, severity, app, app, evidence, size, + auditRemedy{Argv: teploy("releases", "prune", "--app", app, "--host", f.server, "--keep", strconv.Itoa(keep)), Destructive: true, Confirm: true}) + } + } + + // Build cache and dangling images: rebuildable, so no confirmation. + if f.usageKnown || f.imagesKnown { + cache := f.usage["Build Cache"].reclaimable + if cache >= auditLargeCacheBytes || len(f.danglingIDs) > 0 { + evidence := fmt.Sprintf("%d dangling image(s)", len(f.danglingIDs)) + if f.usageKnown { + evidence = fmt.Sprintf("%s of unused build cache and %s", humanBytes(cache), evidence) + } else { + evidence += " (the build cache size could not be read)" + } + add(findingBuildArtifacts, severityInfo, "docker", "", evidence, cache, + auditRemedy{Argv: teploy("server", "prune", "--host", f.server, "--build-cache", "--dangling")}) + } + } + + // Large containers: never touched, whoever owns them. + if f.sizesKnown && f.containersKnown { + for _, c := range f.containers { + size := f.sizes[c.Name] + managed := c.Labels["teploy.app"] != "" || c.Name == "caddy" || belongsToKnownApp(c.Name, f.apps) + switch { + case managed && size >= auditLargeLayerBytes: + add(findingLargeLayer, severityWarning, c.Name, c.Labels["teploy.app"], + fmt.Sprintf("%s (%s) has written %s to its container layer, which a redeploy discards: move that data to a volume", c.Name, c.State, humanBytes(size)), size, auditRemedy{}) + case !managed && size >= auditLargeUnmanagedBytes: + add(findingUnmanagedLarge, severityWarning, c.Name, "", + fmt.Sprintf("%s (%s) is not managed by teploy and holds %s in its container layer", c.Name, c.State, humanBytes(size)), size, auditRemedy{}) + } + } + } + + // Volumes may hold the only copy of something: there is never a remedy. + if f.volsKnown && len(f.volumes) > 0 { + add(findingDanglingVolumes, severityInfo, "docker", "", + fmt.Sprintf("%d volume(s) attached to no container, %s reclaimable: %s", len(f.volumes), humanBytes(f.usage["Local Volumes"].reclaimable), doctorSample(f.volumes)), + f.usage["Local Volumes"].reclaimable, auditRemedy{}) + } + + if f.scheduler != nil && !f.scheduler.Daemon && f.scheduler.HasJobs() { + severity := severityWarning + if f.scheduler.HasTeployJobs() { + severity = severityCritical + } + add(findingSchedulerMissing, severity, "cron", "", "jobs are scheduled but no cron daemon is running, so none of them run: "+backup.SchedulerRemedy(), 0, auditRemedy{}) + } + + if f.df != nil && diskUnderPressure(f.df) { + severity := severityWarning + if f.df.AvailableBytes < doctorDiskFailAvailable { + severity = severityCritical + } + uses := make([]string, 0, len(f.df.Attribution)) + attribution := append([]auditDiskUse{}, f.df.Attribution...) + sort.Slice(attribution, func(i, j int) bool { return attribution[i].SizeBytes > attribution[j].SizeBytes }) + for _, u := range attribution { + uses = append(uses, u.Name+" "+humanBytes(u.SizeBytes)) + } + var related []string + for _, x := range findings { + switch x.Kind { + case findingVersionsBeyond, findingBuildArtifacts, findingLargeLayer, findingUnmanagedLarge, findingDanglingVolumes: + related = append(related, x.ID) + } + } + sort.Strings(related) + findings = append(findings, auditFinding{ID: findingDiskPressure + ":/", Kind: findingDiskPressure, Severity: severity, Subject: "/", + Evidence: fmt.Sprintf("%s available on / (%d%% used); %s", humanBytes(f.df.AvailableBytes), f.df.UsedPercent, strings.Join(uses, ", ")), Related: related}) + } + + rank := map[string]int{severityCritical: 0, severityWarning: 1, severityInfo: 2} + sort.SliceStable(findings, func(i, j int) bool { + if rank[findings[i].Severity] != rank[findings[j].Severity] { + return rank[findings[i].Severity] < rank[findings[j].Severity] + } + return findings[i].ID < findings[j].ID + }) + return findings +} + +// deadRouteVerdict words a caddy-routed app's dead upstream. Redeploy is offered +// only when it can help: a static app has no containers to bring back, and an +// upstream named after another app (a caddy_extra proxy) stays dead through this +// app's redeploy. +func deadRouteVerdict(f *auditFacts, r caddy.ManagedRegion, st *state.AppState, dead []string, hosts string) (string, auditRemedy, string) { + live := len(dialsByVerdict(r.Sites, f.resolves, domains.ResolvesYes)) > 0 + evidence := fmt.Sprintf("%s routes to %s, which does not resolve in Caddy's network", hosts, strings.Join(dead, ", ")) + ownDead := false + for _, dial := range dead { + if name, _ := domains.UpstreamHost(dial); strings.HasPrefix(name, r.App+"-") { + ownDead = true + } + } + severity := severityCritical + if live || st.DeploymentType == "static" { + severity = severityWarning // part of the site still serves + } + switch { + case st.DeploymentType == "static": + return severity, auditRemedy{}, evidence + fmt.Sprintf("; %s itself serves files, so only those paths fail. A redeploy cannot help: fix the app's caddy_extra or deploy what it proxies to", r.App) + case !ownDead: + return severity, auditRemedy{}, evidence + "; that upstream is not this app's, so a redeploy cannot help: fix the app's caddy_extra or deploy what it proxies to" + } + return severity, auditRemedy{Argv: []string{"teploy", "redeploy", "--app", r.App, "--host", f.server}, Confirm: true}, evidence +} + +func defaultPruneWarning(f *auditFacts, app string) string { + if detail, bad := f.noticeErrs[app]; bad { + return "when automatic pruning starts could not be read (" + detail + ")" + } + if t, ok := f.notices[app]; ok { + return "deploys after " + t.Add(deploy.DefaultPruneNotice).Format("2006-01-02") + " remove them automatically; set keep_versions (0 keeps everything) or pin what you need" + } + return "the next deploy starts a 7-day notice, then deploys remove them automatically; set keep_versions (0 keeps everything) or pin what you need" +} + +func belongsToKnownApp(name string, apps map[string]*state.AppState) bool { + for app := range apps { + if strings.HasPrefix(name, app+"-") { + return true + } + } + return false +} + +func sortedAppNames(apps map[string]*state.AppState) []string { + names := make([]string, 0, len(apps)) + for n := range apps { + names = append(names, n) + } + sort.Strings(names) + return names +} + +func humanBytes(n int64) string { + switch { + case n >= 1e12: + return fmt.Sprintf("%.1f TB", float64(n)/1e12) + case n >= 1e9: + return fmt.Sprintf("%.1f GB", float64(n)/1e9) + case n >= 1e6: + return fmt.Sprintf("%.1f MB", float64(n)/1e6) + } + return fmt.Sprintf("%d B", n) +} diff --git a/internal/cli/server_audit_test.go b/internal/cli/server_audit_test.go new file mode 100644 index 0000000..5663f09 --- /dev/null +++ b/internal/cli/server_audit_test.go @@ -0,0 +1,846 @@ +package cli + +// `teploy server audit` tests. Every fixture models one server through the +// ssh MockExecutor the way domains_test.go does: the responses are synthetic, +// every parse and judgement stage is the real one. The read-only pin at the +// bottom is the audit's safety property: no command it issues may mutate. + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/ssh" +) + +var auditFixtureNow = time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC) + +// auditFixture builds the mock executor for one audited server. Zero values +// mean "the read succeeds and reports nothing" wherever that is a shape the +// real server can produce (no dangling images, no dangling volumes); a read +// that must FAIL is registered explicitly through extra. +type auditFixture struct { + server, host string + now time.Time + + apps []string // /deployments/*/state.json listing + state map[string]string // app -> state JSON; "!" means unreadable + badRC map[string]bool // apps whose state read fails (S rc=1) + + caddyfile string // "" = no Caddyfile on the server + + containers string // docker ps --all --format output + sizes string // docker ps --all --size output + resolves string // getent answers inside caddy, "rc name" lines + roots map[string]string + addrs string // hostname -I + usage string // docker system df rows + danglingIm string + danglingVo string + df string + du string + scheduler string + + files map[string][]byte + extra []ssh.MockCommand // registered first: explicit wins +} + +func auditFrames(fx auditFixture) string { + var b strings.Builder + for _, app := range fx.apps { + st := fx.state[app] + rc := "0" + if fx.badRC[app] { + rc = "1" + } + fmt.Fprintf(&b, "A %s\nS %s %s\nG absent\nL \nM 0\nR 0\n", app, rc, + base64.StdEncoding.EncodeToString([]byte(st))) + } + return b.String() +} + +func (fx auditFixture) exec() *ssh.MockExecutor { + cmds := append([]ssh.MockCommand{}, fx.extra...) + if fx.apps != nil { + cmds = append(cmds, ssh.MockCommand{Match: "for f in /deployments/*/state.json", Output: strings.Join(fx.apps, "\n") + "\n"}) + cmds = append(cmds, ssh.MockCommand{Match: "for app in", Output: auditFrames(fx)}) + } + // Registered before the plain `docker ps --all` listing so the longer + // (size-carrying) command is matched first; an empty output means "the + // read succeeded and reported nothing". + cmds = append(cmds, ssh.MockCommand{Match: "t=; command -v timeout >/dev/null 2>&1 && t='timeout 120'; $t docker ps --all --size", Output: fx.sizes}) + if fx.containers != "" { + cmds = append(cmds, ssh.MockCommand{Match: "docker ps --all ", Output: fx.containers}) + } + if fx.resolves != "" { + cmds = append(cmds, ssh.MockCommand{Match: "printf '%s\\n'", Output: fx.resolves}) + } + for root, verdict := range fx.roots { + cmds = append(cmds, ssh.MockCommand{Match: "if [ -e " + ssh.ShellQuote(root) + " ]", Output: verdict}) + } + if fx.addrs != "" { + cmds = append(cmds, ssh.MockCommand{Match: "hostname -I", Output: fx.addrs}) + } + if fx.usage != "" { + cmds = append(cmds, ssh.MockCommand{Match: "docker system df", Output: fx.usage}) + } + cmds = append(cmds, ssh.MockCommand{Match: "docker images -q -f dangling=true", Output: fx.danglingIm}) + cmds = append(cmds, ssh.MockCommand{Match: "docker volume ls -q -f dangling=true", Output: fx.danglingVo}) + if fx.df != "" { + cmds = append(cmds, ssh.MockCommand{Match: "df -B1 -P /", Output: fx.df}) + } + if fx.du != "" { + cmds = append(cmds, ssh.MockCommand{Match: "t=; command -v timeout >/dev/null 2>&1 && t='timeout 60'; $t du -sxb", Output: fx.du}) + } + if fx.scheduler != "" { + cmds = append(cmds, ssh.MockCommand{Match: "sched_daemon=no; ", Output: fx.scheduler}) + } + m := ssh.NewMockExecutor(fx.host, cmds...) + for p, data := range fx.files { + m.Files[p] = data + } + if fx.caddyfile != "" { + m.Files["/deployments/caddy/Caddyfile"] = []byte(fx.caddyfile) + } + return m +} + +func (fx auditFixture) audit(t *testing.T) serverAuditEnvelope { + t.Helper() + if fx.server == "" { + fx.server = "edge" + } + if fx.host == "" { + fx.host = "192.0.2.10" + } + if fx.now.IsZero() { + fx.now = auditFixtureNow + } + return auditServer(context.Background(), fx.exec(), fx.server, fx.host, fx.now) +} + +// auditStateJSON builds a state.json the canonical parser accepts. +func auditStateJSON(deployment, ingress, current, previous string, manifest map[string]any) string { + m := map[string]any{ + "schema_version": 2, + "deployment_type": deployment, + "ingress_mode": ingress, + "updated_at": "2026-11-01T00:00:00Z", + "current_hash": current, + "generation": 2, + } + if previous != "" { + m["previous_hash"] = previous + } + if manifest != nil { + raw, _ := json.Marshal(manifest) + m["applied_manifest"] = json.RawMessage(raw) + } + b, _ := json.Marshal(m) + return string(b) +} + +// auditContainer renders one docker ps --format row. +func auditContainer(name, image, state, createdAt string, labels map[string]string) string { + c := struct { + ID string `json:"ID"` + Names string `json:"Names"` + Image string `json:"Image"` + State string `json:"State"` + Status string `json:"Status"` + CreatedAt string `json:"CreatedAt"` + Labels map[string]string `json:"Labels"` + }{"id" + name, name, image, state, "Up", createdAt, labels} + b, _ := json.Marshal(c) + return string(b) +} + +func auditSizeRow(name, size string) string { + return fmt.Sprintf(`{"Names":%q,"Size":%q}`, name, size) +} + +func findingsByKind(env serverAuditEnvelope, kind string) []auditFinding { + var out []auditFinding + for _, f := range env.Findings { + if f.Kind == kind { + out = append(out, f) + } + } + return out +} + +func oneFinding(t *testing.T, env serverAuditEnvelope, kind string) auditFinding { + t.Helper() + got := findingsByKind(env, kind) + if len(got) != 1 { + t.Fatalf("kind %s: expected exactly 1 finding, got %d: %+v", kind, len(got), env.Findings) + } + return got[0] +} + +// kitchenSink is a server with one of everything: every finding kind, a pin, +// a prune notice, disk pressure (so the du read fires) and an app whose state +// cannot be read (so a check reads unknown). The expected findings are +// asserted piecewise by the tests below and pinned as the contracts fixture. +func auditKitchenSink() auditFixture { + ghostBlock := "# TEPLOY BEGIN ghost\nghost.example.com {\n\treverse_proxy ghost-web-v1:3000\n}\n# TEPLOY END ghost\n" + hollowBlock := "# TEPLOY BEGIN hollow\n# TEPLOY END hollow\n" + blogBlock := "# TEPLOY BEGIN blog\nblog.example.com {\n\treverse_proxy blog-web-v3:3000\n}\n# TEPLOY END blog\n" + filesBlock := "# TEPLOY BEGIN files\nfiles.example.com {\n\troot * /deployments/files/site\n\tfile_server\n}\n# TEPLOY END files\n" + preview := "# TEPLOY BEGIN blog-preview-p-08e81639\npreview.example.com {\n\treverse_proxy blog-preview-p-08e81639-abc1234:3000\n}\n# TEPLOY END blog-preview-p-08e81639\n" + manual := "hand.example.com {\n\treverse_proxy gone-svc:80\n}\n\nhttp://203.0.113.9 {\n\treverse_proxy 127.0.0.1:9000\n}\n" + + keep := func(n int) map[string]any { + return map[string]any{"container": map[string]any{"keep_versions": n}} + } + manyV := func(v, when string, running bool) string { + state := "exited" + if running { + state = "running" + } + return auditContainer("many-web-"+v, "many:"+v, state, when, + map[string]string{"teploy.app": "many", "teploy.process": "web", "teploy.version": v}) + } + unsetV := func(v, when string, running bool) string { + state := "exited" + if running { + state = "running" + } + return auditContainer("unset-web-"+v, "unset:"+v, state, when, + map[string]string{"teploy.app": "unset", "teploy.process": "web", "teploy.version": v}) + } + containers := []string{ + auditContainer("caddy", "caddy:2", "running", "2026-05-01 00:00:00 +0000 UTC", nil), + auditContainer("blog-web-v2", "blog:v2", "running", "2026-11-04 10:00:00 +0000 UTC", + map[string]string{"teploy.app": "blog", "teploy.process": "web", "teploy.version": "v2"}), + manyV("v4", "2026-11-04 09:00:00 +0000 UTC", true), + manyV("v3", "2026-11-03 09:00:00 +0000 UTC", false), + manyV("v2", "2026-11-02 09:00:00 +0000 UTC", false), + manyV("v1", "2026-11-01 09:00:00 +0000 UTC", false), + unsetV("u7", "2026-11-04 08:00:00 +0000 UTC", true), + unsetV("u6", "2026-11-03 08:00:00 +0000 UTC", false), + unsetV("u5", "2026-11-02 08:00:00 +0000 UTC", false), + unsetV("u4", "2026-11-01 08:00:00 +0000 UTC", false), + unsetV("u3", "2026-10-31 08:00:00 +0000 UTC", false), + unsetV("u2", "2026-10-30 08:00:00 +0000 UTC", false), + unsetV("u1", "2026-10-29 08:00:00 +0000 UTC", false), + auditContainer("reacher-web-1f2e3d", "reacher:1", "running", "2026-08-01 00:00:00 +0000 UTC", + map[string]string{"teploy.app": "reacher", "teploy.process": "web", "teploy.version": "1f2e3d"}), + auditContainer("portainer", "portainer/portainer:2", "running", "2026-05-02 00:00:00 +0000 UTC", nil), + } + sizes := []string{ + auditSizeRow("caddy", "100MB (virtual 200MB)"), + auditSizeRow("blog-web-v2", "50MB (virtual 1GB)"), + auditSizeRow("many-web-v4", "10MB (virtual 1GB)"), + auditSizeRow("many-web-v3", "10MB (virtual 1GB)"), + auditSizeRow("many-web-v2", "10MB (virtual 1GB)"), + auditSizeRow("many-web-v1", "10MB (virtual 1GB)"), + auditSizeRow("unset-web-u7", "10MB (virtual 1GB)"), + auditSizeRow("unset-web-u6", "10MB (virtual 1GB)"), + auditSizeRow("unset-web-u5", "10MB (virtual 1GB)"), + auditSizeRow("unset-web-u4", "10MB (virtual 1GB)"), + auditSizeRow("unset-web-u3", "10MB (virtual 1GB)"), + auditSizeRow("unset-web-u2", "10MB (virtual 1GB)"), + auditSizeRow("unset-web-u1", "10MB (virtual 1GB)"), + auditSizeRow("reacher-web-1f2e3d", "35.7GB (virtual 1.1GB)"), + auditSizeRow("portainer", "2GB (virtual 500MB)"), + } + volumes := make([]string, 21) + for i := range volumes { + volumes[i] = fmt.Sprintf("%031dxxxxxxxdeadbeef", i) + } + return auditFixture{ + apps: []string{"blog", "many", "unset", "orphan", "static-site", "broken"}, + state: map[string]string{ + "blog": auditStateJSON("container", "caddy", "v3", "v2", keep(3)), + "many": auditStateJSON("container", "caddy", "v4", "v3", keep(2)), + "unset": auditStateJSON("container", "caddy", "u7", "u6", map[string]any{"container": map[string]any{"keep_versions": 0}}), + "orphan": auditStateJSON("container", "caddy", "o1", "", nil), + "static-site": auditStateJSON("static", "caddy", "s1", "", nil), + "broken": "", + }, + badRC: map[string]bool{"broken": true}, + caddyfile: ghostBlock + "\n" + hollowBlock + "\n" + blogBlock + "\n" + filesBlock + "\n" + preview + "\n" + manual, + containers: strings.Join(containers, "\n"), + sizes: strings.Join(sizes, "\n"), + resolves: "0 blog-preview-p-08e81639-abc1234\n" + + "2 ghost-web-v1\n" + + "2 blog-web-v3\n" + + "2 gone-svc\n", + roots: map[string]string{"/deployments/files/site": "yes"}, + addrs: "192.0.2.10 172.17.0.1\n", + usage: `{"Type":"Images","Size":"20GB","Reclaimable":"1GB (5%)"}` + "\n" + `{"Type":"Containers","Size":"5GB","Reclaimable":"0B (0%)"}` + "\n" + `{"Type":"Local Volumes","Size":"4GB","Reclaimable":"3.15GB (78%)"}` + "\n" + `{"Type":"Build Cache","Size":"2GB","Reclaimable":"2GB (100%)"}`, + danglingIm: "sha256:1111111111111111111111111111111111111111111111111111111111111111\nsha256:2222222222222222222222222222222222222222222222222222222222222222\nsha256:3333333333333333333333333333333333333333333333333333333333333333\n", + danglingVo: strings.Join(volumes, "\n") + "\n", + df: "Filesystem 1-blocks Used Available Capacity Mounted on\n/dev/vda1 10000000000 9600000000 800000000 96% /\n", + du: "53687091200\t/deployments\n", + scheduler: "daemon=no\nenabled=disabled\ncrontab=yes\ncrontab_entries=2\nteploy_entries=1\n", + files: map[string][]byte{ + "/deployments/many/pinned": []byte("v1\n"), + "/deployments/unset/.prune-notice": []byte("2026-11-01T00:00:00Z\n"), + }, + } +} + +func auditCheckStatus(env serverAuditEnvelope, name string) (string, bool) { + for _, c := range env.Checks { + if c.Name == name { + return c.Status, true + } + } + return "", false +} + +func TestServerAudit_KitchenSinkFindings(t *testing.T) { + env := auditKitchenSink().audit(t) + + if env.State != auditStatePartial { + t.Fatalf("state = %s, want partial (one app state unreadable)", env.State) + } + if st, ok := auditCheckStatus(env, "apps"); !ok || st != auditCheckUnknown { + t.Errorf("apps check = %q %v, want unknown: %+v", st, ok, env.Checks) + } + + var kinds []string + for _, f := range env.Findings { + kinds = append(kinds, f.Kind) + } + got := strings.Join(kinds, ",") + expected := []string{ + "dangling_volumes", + "disk_pressure", + "empty_managed_region", + "large_writable_layer", + "managed_route_dead_upstream", + "manual_route_dead_upstream", + "manual_route_nonlocal_address", + "orphan_state", + "reclaimable_build_artifacts", + "scheduler_missing", + "stale_route", + "unmanaged_container", + "versions_beyond_policy", + } + for _, k := range expected { + if strings.Count(got, k) == 0 { + t.Errorf("expected at least one %s finding; got: %s", k, got) + } + } + if strings.Count(got, "versions_beyond_policy") != 2 { + t.Errorf("expected two versions_beyond_policy findings (explicit + default), got: %s", got) + } + if strings.Count(got, "stale_route") != 2 { + t.Errorf("expected two stale_route findings (ghost + files), got: %s", got) + } + // Findings are ordered by severity then id; a critical must not sort after an info. + rank := map[string]int{severityCritical: 0, severityWarning: 1, severityInfo: 2} + for i := 1; i < len(env.Findings); i++ { + a, b := env.Findings[i-1], env.Findings[i] + if rank[a.Severity] > rank[b.Severity] { + t.Errorf("findings not sorted by severity: %s after %s", b.ID, a.ID) + } + } +} + +func TestServerAudit_StaleRouteAndEmptyRegion(t *testing.T) { + env := auditKitchenSink().audit(t) + + var ghost, files *auditFinding + for _, f := range findingsByKind(env, findingStaleRoute) { + switch f.Subject { + case "ghost": + ghost = &f + case "files": + files = &f + } + } + if ghost == nil || files == nil { + t.Fatalf("missing stale_route findings: %+v", env.Findings) + } + if ghost.App != "ghost" || !strings.Contains(ghost.Evidence, "no app ghost") { + t.Errorf("ghost finding: %+v", ghost) + } + wantArgv := []string{"teploy", "route", "remove", "--app", "ghost", "--host", "edge"} + if fmt.Sprint(ghost.Remedy.Argv) != fmt.Sprint(wantArgv) { + t.Errorf("ghost remedy argv = %v, want %v", ghost.Remedy.Argv, wantArgv) + } + if ghost.Remedy.Destructive || ghost.Remedy.Confirm { + t.Errorf("a proven-dead route needs no confirmation: %+v", ghost.Remedy) + } + + // The files app's block still serves files that exist: reported, remedy null. + if files.Remedy.Argv != nil { + t.Errorf("a route still serving files must be report-only, got argv %v", files.Remedy.Argv) + } + if !strings.Contains(files.Evidence, "not removable") || !strings.Contains(files.Evidence, "/deployments/files/site") { + t.Errorf("files evidence must name the refusal: %q", files.Evidence) + } + + hollow := oneFinding(t, env, findingEmptyRegion) + if hollow.Subject != "hollow" || hollow.Remedy.Argv == nil { + t.Errorf("empty region should be removable: %+v", hollow) + } +} + +func TestServerAudit_ManagedDeadUpstreamOffersRedeploy(t *testing.T) { + env := auditKitchenSink().audit(t) + f := oneFinding(t, env, findingDeadUpstream) + if f.Subject != "blog" { + t.Fatalf("dead upstream subject = %s, want blog", f.Subject) + } + if f.Severity != severityCritical { + t.Errorf("a fully dead route is critical, got %s", f.Severity) + } + if fmt.Sprint(f.Remedy.Argv) != fmt.Sprint([]string{"teploy", "redeploy", "--app", "blog", "--host", "edge"}) { + t.Errorf("remedy argv = %v", f.Remedy.Argv) + } + if !f.Remedy.Confirm || f.Remedy.Destructive { + t.Errorf("redeploy needs confirmation but is not destructive: %+v", f.Remedy) + } + if !strings.Contains(f.Evidence, "blog-web-v3:3000") { + t.Errorf("evidence must name the dead upstream: %q", f.Evidence) + } + + // A dead upstream that is NOT this app's (caddy_extra proxy to another + // app) cannot be fixed by this app's redeploy: report only. + fx := auditKitchenSink() + fx.caddyfile = "# TEPLOY BEGIN blog\nblog.example.com {\n\treverse_proxy other-svc:3000\n}\n# TEPLOY END blog\n" + fx.resolves = "2 other-svc\n" + f = oneFinding(t, fx.audit(t), findingDeadUpstream) + if f.Remedy.Argv != nil { + t.Errorf("someone else's dead upstream must be report-only: %+v", f.Remedy) + } + if !strings.Contains(f.Evidence, "redeploy cannot help") { + t.Errorf("evidence should say a redeploy cannot help: %q", f.Evidence) + } +} + +func TestServerAudit_OrphanState(t *testing.T) { + env := auditKitchenSink().audit(t) + f := oneFinding(t, env, findingOrphanState) + if f.Subject != "orphan" { + t.Fatalf("orphan subject = %s", f.Subject) + } + if fmt.Sprint(f.Remedy.Argv) != fmt.Sprint([]string{"teploy", "remove", "--app", "orphan", "--host", "edge", "-y"}) { + t.Errorf("remedy argv = %v", f.Remedy.Argv) + } + if !f.Remedy.Destructive || !f.Remedy.Confirm { + t.Errorf("removing state is destructive and needs confirmation: %+v", f.Remedy) + } + + // A static app with no containers is normal, not an orphan. + for _, f := range env.Findings { + if f.Kind == findingOrphanState && f.Subject == "static-site" { + t.Errorf("static app reported as orphan: %+v", f) + } + } +} + +func TestServerAudit_VersionsBeyondPolicy(t *testing.T) { + env := auditKitchenSink().audit(t) + var many, unset *auditFinding + for _, f := range findingsByKind(env, findingVersionsBeyond) { + switch f.Subject { + case "many": + many = &f + case "unset": + unset = &f + } + } + if many == nil || unset == nil { + t.Fatalf("missing versions findings: %+v", env.Findings) + } + + // many: keep 2, current v4 + previous v3 protected, v1 pinned → only v2. + if !strings.Contains(many.Evidence, "beyond keep_versions 2: v2") { + t.Errorf("many evidence = %q", many.Evidence) + } + if strings.Contains(many.Evidence, "v1") { + t.Errorf("pinned v1 must not be listed as beyond policy: %q", many.Evidence) + } + if many.Severity != severityInfo || !strings.Contains(many.Evidence, "the next deploy removes them") { + t.Errorf("an explicit keep_versions is housekeeping, not a warning: %+v", many) + } + if fmt.Sprint(many.Remedy.Argv) != fmt.Sprint([]string{"teploy", "releases", "prune", "--app", "many", "--host", "edge", "--keep", "2"}) { + t.Errorf("many remedy argv = %v", many.Remedy.Argv) + } + if !many.Remedy.Destructive || !many.Remedy.Confirm { + t.Errorf("pruning versions is destructive: %+v", many.Remedy) + } + + // unset: keep_versions not set → default 5, warning + the notice date. + if !strings.Contains(unset.Evidence, "beyond keep_versions 5") { + t.Errorf("unset evidence = %q", unset.Evidence) + } + if unset.Severity != severityWarning { + t.Errorf("an unset keep_versions pruning by default is a warning: %+v", unset) + } + if !strings.Contains(unset.Evidence, "2026-11-08") { // notice 2026-11-01 + 7d + t.Errorf("unset evidence should name when automatic pruning starts: %q", unset.Evidence) + } + + // No notice file yet: the warning says the next deploy starts the notice. + fx := auditKitchenSink() + delete(fx.files, "/deployments/unset/.prune-notice") + env = fx.audit(t) + for _, f := range findingsByKind(env, findingVersionsBeyond) { + if f.Subject == "unset" && !strings.Contains(f.Evidence, "7-day notice") { + t.Errorf("unset evidence without a notice file = %q", f.Evidence) + } + } +} + +func TestServerAudit_ManualBlocksAreReportOnly(t *testing.T) { + env := auditKitchenSink().audit(t) + dead := oneFinding(t, env, findingManualDead) + if dead.Remedy.Argv != nil { + t.Errorf("hand-written blocks must never carry a remedy: %+v", dead) + } + if !strings.Contains(dead.Evidence, "gone-svc") || !strings.Contains(dead.Evidence, "hand-written") { + t.Errorf("dead manual evidence = %q", dead.Evidence) + } + nonlocal := oneFinding(t, env, findingManualNonLocal) + if nonlocal.Remedy.Argv != nil { + t.Errorf("hand-written blocks must never carry a remedy: %+v", nonlocal) + } + if !strings.Contains(nonlocal.Evidence, "203.0.113.9") { + t.Errorf("nonlocal evidence = %q", nonlocal.Evidence) + } +} + +func TestServerAudit_HygieneFindings(t *testing.T) { + env := auditKitchenSink().audit(t) + + artifacts := oneFinding(t, env, findingBuildArtifacts) + if fmt.Sprint(artifacts.Remedy.Argv) != fmt.Sprint([]string{"teploy", "server", "prune", "--host", "edge", "--build-cache", "--dangling"}) { + t.Errorf("build artifacts remedy = %v", artifacts.Remedy.Argv) + } + if artifacts.Remedy.Destructive || artifacts.Remedy.Confirm { + t.Errorf("rebuildable cache needs no confirmation: %+v", artifacts.Remedy) + } + + layer := oneFinding(t, env, findingLargeLayer) + if layer.Subject != "reacher-web-1f2e3d" || layer.Remedy.Argv != nil { + t.Errorf("large writable layer is report-only: %+v", layer) + } + if layer.SizeBytes < 35_000_000_000 { + t.Errorf("layer size = %d", layer.SizeBytes) + } + + unmanaged := oneFinding(t, env, findingUnmanagedLarge) + if unmanaged.Subject != "portainer" || unmanaged.Remedy.Argv != nil { + t.Errorf("unmanaged container is report-only: %+v", unmanaged) + } + + volumes := oneFinding(t, env, findingDanglingVolumes) + if volumes.Remedy.Argv != nil { + t.Errorf("volumes may hold the only copy — report only: %+v", volumes) + } + if !strings.Contains(volumes.Evidence, "21 volume(s)") || !strings.Contains(volumes.Evidence, "3.1 GB") { + t.Errorf("volumes evidence = %q", volumes.Evidence) + } + + sched := oneFinding(t, env, findingSchedulerMissing) + if sched.Severity != severityCritical || sched.Remedy.Argv != nil { + t.Errorf("teploy jobs with no cron daemon is critical, report-only: %+v", sched) + } +} + +func TestServerAudit_CaddyfileWithoutCaddy(t *testing.T) { + fx := auditKitchenSink() + fx.containers = strings.ReplaceAll(fx.containers, + auditContainer("caddy", "caddy:2", "running", "2026-05-01 00:00:00 +0000 UTC", nil), + auditContainer("caddy", "caddy:2", "exited", "2026-05-01 00:00:00 +0000 UTC", nil)) + env := fx.audit(t) + f := oneFinding(t, env, findingCaddyfileNoCaddy) + if f.Severity != severityWarning || !strings.Contains(f.Evidence, "the caddy container is exited") { + t.Errorf("caddyfile without caddy: %+v", f) + } + // Without a running caddy nothing can be proven dead: the stale route + // loses its one-click remedy instead of being reported removable. + var ghost *auditFinding + for _, f := range findingsByKind(env, findingStaleRoute) { + if f.Subject == "ghost" { + ghost = &f + } + } + if ghost == nil { + t.Fatalf("no stale_route finding for ghost: %+v", env.Findings) + } + if ghost.Remedy.Argv != nil || !strings.Contains(ghost.Evidence, "not removable") { + t.Errorf("an unprovable route must be report-only: %+v", ghost) + } + if st, ok := auditCheckStatus(env, "upstreams"); !ok || st != auditCheckUnknown { + t.Errorf("upstreams check = %q %v, want unknown without a running caddy", st, ok) + } +} + +func TestServerAudit_DiskPressureAttributesAndLinks(t *testing.T) { + env := auditKitchenSink().audit(t) + f := oneFinding(t, env, findingDiskPressure) + if f.Severity != severityCritical { // 800MB < 2GiB + t.Errorf("800MB free is critical, got %s", f.Severity) + } + if !strings.Contains(f.Evidence, "images 20.0 GB") || !strings.Contains(f.Evidence, "deployments") { + t.Errorf("disk evidence must attribute the space: %q", f.Evidence) + } + var relatedKinds []string + for _, id := range f.Related { + for _, x := range env.Findings { + if x.ID == id { + relatedKinds = append(relatedKinds, x.Kind) + } + } + } + for _, want := range []string{findingVersionsBeyond, findingBuildArtifacts, findingDanglingVolumes, findingLargeLayer} { + found := false + for _, k := range relatedKinds { + if k == want { + found = true + } + } + if !found { + t.Errorf("disk pressure related findings missing %s: %v", want, f.Related) + } + } + if env.Disk == nil || env.Disk.AvailableBytes != 800000000 || env.Disk.UsedPercent != 96 { + t.Errorf("disk = %+v", env.Disk) + } +} + +func TestServerAudit_LivePreviewIsNotAFinding(t *testing.T) { + env := auditKitchenSink().audit(t) + for _, f := range env.Findings { + if strings.Contains(f.ID, "preview") { + t.Errorf("a live preview route is not stale: %+v", f) + } + } +} + +func TestServerAudit_UnreadableYieldsCheckNotFinding(t *testing.T) { + fx := auditKitchenSink() + // The container listing fails: orphan/large-layer/unmanaged findings are + // impossible to judge, and routes cannot be proven dead. + fx.extra = append([]ssh.MockCommand{{Match: "docker ps --all ", Err: errors.New("docker: cannot connect to the docker daemon")}}, fx.extra...) + env := fx.audit(t) + if env.State != auditStatePartial { + t.Fatalf("state = %s, want partial", env.State) + } + if st, ok := auditCheckStatus(env, "containers"); !ok || st != auditCheckUnknown { + t.Fatalf("containers check = %q %v, want unknown: %+v", st, ok, env.Checks) + } + for _, kind := range []string{findingOrphanState, findingLargeLayer, findingUnmanagedLarge, findingCaddyfileNoCaddy} { + if got := findingsByKind(env, kind); len(got) != 0 { + t.Errorf("with containers unreadable there must be no %s finding: %+v", kind, got) + } + } + var ghost *auditFinding + for _, f := range findingsByKind(env, findingStaleRoute) { + if f.Subject == "ghost" { + ghost = &f + } + } + if ghost == nil { + t.Fatalf("no stale_route finding for ghost: %+v", env.Findings) + } + if ghost.Remedy.Argv != nil { + t.Errorf("removal cannot be proven safe without the container list: %+v", ghost.Remedy) + } + // An app whose state could not be read is never judged; the check names it. + if st, ok := auditCheckStatus(env, "apps"); !ok || st != auditCheckUnknown { + t.Errorf("apps check = %q %v, want unknown", st, ok) + } +} + +func TestServerAudit_HealthyServerReportsNothing(t *testing.T) { + fx := auditFixture{ + apps: []string{"blog"}, + state: map[string]string{"blog": auditStateJSON("container", "caddy", "v3", "v2", map[string]any{"container": map[string]any{"keep_versions": 3}})}, + caddyfile: "# TEPLOY BEGIN blog\nblog.example.com {\n\treverse_proxy blog-web-v3:3000\n}\n# TEPLOY END blog\n", + containers: auditContainer("caddy", "caddy:2", "running", "2026-05-01 00:00:00 +0000 UTC", nil) + "\n" + + auditContainer("blog-web-v3", "blog:v3", "running", "2026-11-04 10:00:00 +0000 UTC", + map[string]string{"teploy.app": "blog", "teploy.process": "web", "teploy.version": "v3"}), + sizes: auditSizeRow("caddy", "100MB (virtual 200MB)") + "\n" + auditSizeRow("blog-web-v3", "50MB (virtual 1GB)"), + resolves: "0 blog-web-v3\n", + addrs: "192.0.2.10\n", + usage: `{"Type":"Images","Size":"20GB","Reclaimable":"1GB (5%)"}` + "\n" + `{"Type":"Containers","Size":"5GB","Reclaimable":"0B (0%)"}` + "\n" + `{"Type":"Local Volumes","Size":"4GB","Reclaimable":"0B (0%)"}` + "\n" + `{"Type":"Build Cache","Size":"2GB","Reclaimable":"100MB (5%)"}`, + df: "Filesystem 1-blocks Used Available Capacity Mounted on\n/dev/vda1 100000000000 30000000000 70000000000 30% /\n", + scheduler: "daemon=yes\nenabled=enabled\ncrontab=no\n", + } + env := fx.audit(t) + if env.State != auditStateOK { + t.Fatalf("state = %s, want ok; checks: %+v", env.State, env.Checks) + } + if len(env.Findings) != 0 { + t.Fatalf("a healthy server must report no findings: %+v", env.Findings) + } +} + +func TestServerAudit_JSONEncodesRemedyNull(t *testing.T) { + env := auditKitchenSink().audit(t) + data, err := json.Marshal(env) + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := json.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + findings, _ := doc["findings"].([]any) + if len(findings) == 0 { + t.Fatal("no findings in JSON") + } + nullArgv := 0 + for _, f := range findings { + remedy := f.(map[string]any)["remedy"].(map[string]any) + argv, _ := remedy["argv"].([]any) + if remedy["argv"] == nil { + nullArgv++ + } else if len(argv) == 0 || argv[0] != "teploy" { + t.Errorf("remedy argv must be null or start with teploy: %v", remedy["argv"]) + } + if _, ok := remedy["destructive"].(bool); !ok { + t.Errorf("destructive must always be present: %v", remedy) + } + if _, ok := remedy["confirm"].(bool); !ok { + t.Errorf("confirm must always be present: %v", remedy) + } + } + if nullArgv == 0 { + t.Error("no finding with remedy argv null in the fixture") + } + if _, ok := doc["disk"].(map[string]any); !ok { + t.Error("disk must be an object on a reachable server") + } + if _, ok := doc["checks"].([]any); !ok { + t.Error("checks must be an array") + } +} + +// The read-only pin: run the full kitchen sink (every read fires, including +// the pressure-only du and the retention reads) through a recording wrapper +// and refuse any mutating verb, docker subcommand outside the read set, or +// file upload. +func TestServerAudit_EveryCommandOnlyReads(t *testing.T) { + fx := auditKitchenSink() + base := fx.exec() + rec := &recordingExecutor{Executor: base} + env := auditServer(context.Background(), rec, "edge", "192.0.2.10", auditFixtureNow) + if env.State != auditStatePartial { + t.Fatalf("fixture stopped exercising a read: %+v", env.Checks) + } + sawDU := false + for _, cmd := range rec.commands { + if strings.Contains(cmd, "du -sxb") { + sawDU = true + } + } + if !sawDU { + t.Errorf("the disk-pressure attribution read never ran; recorded: %v", rec.commands) + } + if len(rec.uploads) != 0 { + t.Errorf("the audit uploaded files: %v", rec.uploads) + } + if len(rec.inputs) != 0 { + t.Errorf("the audit streamed stdin to commands: %v", rec.inputs) + } + if len(rec.commands) == 0 { + t.Fatal("no commands recorded") + } + banned := []*regexp.Regexp{ + regexp.MustCompile(`\brm\b`), regexp.MustCompile(`\bmv\b`), regexp.MustCompile(`\bmkdir\b`), + regexp.MustCompile(`\brmdir\b`), regexp.MustCompile(`\bcp\b`), regexp.MustCompile(`\btouch\b`), + regexp.MustCompile(`\btruncate\b`), regexp.MustCompile(`\btee\b`), regexp.MustCompile(`\bchmod\b`), + regexp.MustCompile(`\bchown\b`), regexp.MustCompile(`\bkill\b`), regexp.MustCompile(`\bpkill\b`), + regexp.MustCompile(`\breboot\b`), regexp.MustCompile(`\bshutdown\b`), regexp.MustCompile(`\bsed\b`), + regexp.MustCompile(`\bdd\b`), regexp.MustCompile(`\bmkfs\b`), + regexp.MustCompile(`\bmount\b`), regexp.MustCompile(`\bumount\b`), regexp.MustCompile(`\bcurl\b`), + regexp.MustCompile(`\bwget\b`), regexp.MustCompile(`\bapt\b`), regexp.MustCompile(`\byum\b`), + regexp.MustCompile(`\bdnf\b`), regexp.MustCompile(`\bpip\b`), regexp.MustCompile(`\buseradd\b`), + regexp.MustCompile(`\bpasswd\b`), + regexp.MustCompile(`\bsystemctl\s+(start|stop|restart|enable|disable|mask)\b`), + regexp.MustCompile(`>>`), + } + allowedDocker := map[string]bool{"ps": true, "images": true, "volume": true, "system": true, "exec": true, "inspect": true, "version": true} + for _, cmd := range rec.commands { + for _, re := range banned { + if re.MatchString(cmd) { + t.Errorf("audit ran a mutating command (%s): %s", re, cmd) + } + } + if strings.Contains(cmd, "crontab") && !strings.Contains(cmd, "crontab -l") { + t.Errorf("audit touched crontab beyond -l: %s", cmd) + } + for _, tok := range strings.Fields(cmd) { + if strings.HasPrefix(tok, ">") && tok != ">/dev/null" { + t.Errorf("audit redirects stdout (%s): %s", tok, cmd) + } + } + if i := strings.Index(cmd, "docker "); i >= 0 { + fields := strings.Fields(cmd[i:]) + if len(fields) > 1 && !allowedDocker[fields[1]] { + t.Errorf("audit ran docker %s: %s", fields[1], cmd) + } + if len(fields) > 1 && fields[1] == "exec" && !strings.Contains(cmd, "getent") { + t.Errorf("audit docker exec is not the read-only getent probe: %s", cmd) + } + } + } +} + +// recordingExecutor wraps an Executor and records everything asked of it, so +// a test can pin WHAT a read-only code path may do to a server. +type recordingExecutor struct { + ssh.Executor + commands []string + inputs []string + uploads []string +} + +func (r *recordingExecutor) Run(ctx context.Context, cmd string) (string, error) { + r.commands = append(r.commands, cmd) + return r.Executor.Run(ctx, cmd) +} + +func (r *recordingExecutor) RunStream(ctx context.Context, cmd string, stdout, stderr io.Writer) error { + r.commands = append(r.commands, cmd) + return r.Executor.RunStream(ctx, cmd, stdout, stderr) +} + +func (r *recordingExecutor) RunInput(ctx context.Context, cmd string, stdin io.Reader) error { + r.inputs = append(r.inputs, cmd) + return r.Executor.RunInput(ctx, cmd, stdin) +} + +func (r *recordingExecutor) Upload(ctx context.Context, content io.Reader, remotePath string, mode string) error { + r.uploads = append(r.uploads, remotePath) + return r.Executor.Upload(ctx, content, remotePath, mode) +} + +func TestParseDockerSize(t *testing.T) { + for _, tt := range []struct { + in string + want int64 + ok bool + }{ + {"12B", 12, true}, + {"50MB", 50_000_000, true}, + {"2.5GB", 2_500_000_000, true}, + {"1KiB", 1024, true}, + {"35.7GB", 35_700_000_000, true}, // the "(virtual …)" tail is split off by the caller + {"", 0, false}, + {"lots", 0, false}, + } { + got, ok := parseDockerSize(tt.in) + if ok != tt.ok || got != tt.want { + t.Errorf("parseDockerSize(%q) = %d,%v want %d,%v", tt.in, got, ok, tt.want, tt.ok) + } + } +} diff --git a/internal/domains/domains.go b/internal/domains/domains.go index a8ca8b6..fa3812a 100644 --- a/internal/domains/domains.go +++ b/internal/domains/domains.go @@ -306,10 +306,13 @@ func CheckServer(ctx context.Context, exec ssh.Executor, name string, opts Optio return srv, rows } +// AppListCommand prints the name of every app with state on a server. +const AppListCommand = `for f in /deployments/*/state.json /deployments/*/state; do [ -f "$f" ] && basename "$(dirname "$f")"; done | sort -u` + // readApps lists the deployed apps with their ingress and domain. known=false // means the app list itself could not be read, so ownership cannot be judged. func readApps(ctx context.Context, exec ssh.Executor, srv *Server, issue func(string, error)) (apps []AppInfo, known bool) { - out, err := exec.Run(ctx, `for f in /deployments/*/state.json /deployments/*/state; do [ -f "$f" ] && basename "$(dirname "$f")"; done | sort -u`) + out, err := exec.Run(ctx, AppListCommand) if err != nil { issue("apps", err) return nil, false @@ -834,10 +837,10 @@ func skipFor(row Domain, host string) string { var upstreamHostRE = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`) -// upstreamHost returns the name to resolve for a dial, or why not: an IP literal +// UpstreamHost returns the name to resolve for a dial, or why not: an IP literal // has nothing to resolve; a placeholder or odd token cannot be checked. Names // are validated before they reach a shell, and quoted there. -func upstreamHost(dial string) (name, verdict string) { +func UpstreamHost(dial string) (name, verdict string) { host := dial if h, _, err := net.SplitHostPort(dial); err == nil { host = h @@ -859,7 +862,7 @@ func resolveUpstreams(ctx context.Context, exec ssh.Executor, rows []Domain, srv wanted := map[string]bool{} for i := range rows { for _, u := range rows[i].Route.Upstreams { - if name, _ := upstreamHost(u.Dial); name != "" { + if name, _ := UpstreamHost(u.Dial); name != "" { wanted[name] = true } } @@ -868,32 +871,18 @@ func resolveUpstreams(ctx context.Context, exec ssh.Executor, rows []Domain, srv if len(wanted) > 0 && srv.Caddy.Container == "present" { names := make([]string, 0, len(wanted)) for n := range wanted { - names = append(names, ssh.ShellQuote(n)) + names = append(names, n) } - sort.Strings(names) - cmd := "printf '%s\\n' " + strings.Join(names, " ") + ` | docker exec -i caddy sh -c 'while read -r h; do getent hosts "$h" >/dev/null 2>&1; echo "$? $h"; done'` - out, err := exec.Run(ctx, cmd) - if err != nil { + var err error + if answers, err = ResolveNames(ctx, exec, names); err != nil { issue("upstreams", err) } - for _, line := range strings.Split(out, "\n") { - rc, name, ok := strings.Cut(strings.TrimSpace(line), " ") - if !ok || !wanted[name] { - continue - } - switch rc { - case "0": - answers[name] = ResolvesYes - case "2": - answers[name] = ResolvesNo - } - } } for i := range rows { r := &rows[i].Route yes, no, unknown := 0, 0, 0 for j := range r.Upstreams { - name, verdict := upstreamHost(r.Upstreams[j].Dial) + name, verdict := UpstreamHost(r.Upstreams[j].Dial) if name != "" { if verdict = answers[name]; verdict == "" { verdict = ResolvesUnknown @@ -928,6 +917,36 @@ func resolveUpstreams(ctx context.Context, exec ssh.Executor, rows []Domain, srv } } +// ResolveNames asks the running caddy container, in one docker exec, whether +// each name resolves: yes or no. A name missing from the result is unknown, +// and so is everything when the error is set. Names must pass UpstreamHost. +func ResolveNames(ctx context.Context, exec ssh.Executor, names []string) (map[string]string, error) { + answers := map[string]string{} + if len(names) == 0 { + return answers, nil + } + quoted := make([]string, 0, len(names)) + for _, n := range names { + quoted = append(quoted, ssh.ShellQuote(n)) + } + sort.Strings(quoted) + cmd := "printf '%s\\n' " + strings.Join(quoted, " ") + ` | docker exec -i caddy sh -c 'while read -r h; do getent hosts "$h" >/dev/null 2>&1; echo "$? $h"; done'` + out, err := exec.Run(ctx, cmd) + for _, line := range strings.Split(out, "\n") { + rc, name, ok := strings.Cut(strings.TrimSpace(line), " ") + if !ok { + continue + } + switch rc { + case "0": + answers[name] = ResolvesYes + case "2": + answers[name] = ResolvesNo + } + } + return answers, err +} + // applyTLS fills certificate evidence on rows Caddy would obtain for. Unknown // when the logs cannot back a claim; zero failures only on logs long enough. func applyTLS(rows []Domain, stats map[string]*ObtainStats, status CaddyStatus) { From fd7f0dfd187422f998a766b71cab013294b09580 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:38:44 -0700 Subject: [PATCH 3/6] =?UTF-8?q?feat(route):=20teploy=20route=20remove=20?= =?UTF-8?q?=E2=80=94=20retire=20a=20managed=20block=20proven=20dead?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes one app's managed Caddy block, reloads Caddy, records it as a route.remove event (removed/absent; refused/failed in the detail). An absent block is success. Only a block proven dead goes. Refuses: a live app (state plus running non-accessory containers, or a live static app), any container still carrying the app's name, an upstream that still resolves (asked inside Caddy's network) or cannot be proven dead, files the block still serves, state or containers that cannot be read, and a block carrying the autodeploy webhook route. The commit runs under the app's fence guard and an exact-block compare-and-swap on the region that was judged, so a deploy that rewrites the block in between is refused (ErrRouteCAS) with no partial edit — the staged file is cleaned up and the successor's block stays byte-exact. --- internal/cli/root.go | 1 + internal/cli/route.go | 215 +++++++++++++++++++++++ internal/cli/route_remove_test.go | 281 ++++++++++++++++++++++++++++++ internal/events/events.go | 2 + 4 files changed, 499 insertions(+) create mode 100644 internal/cli/route.go create mode 100644 internal/cli/route_remove_test.go diff --git a/internal/cli/root.go b/internal/cli/root.go index e06be71..c80c595 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -136,6 +136,7 @@ func NewRootCmd(version string) *cobra.Command { root.AddCommand(newValidateCmd(flags)) root.AddCommand(newDoctorCmd(flags, version)) root.AddCommand(newDomainsCmd(flags)) + root.AddCommand(newRouteCmd(flags)) root.AddCommand(newMetaCmd(flags)) root.AddCommand(newPlanCmd(flags)) root.AddCommand(newApplyCmd(flags)) diff --git a/internal/cli/route.go b/internal/cli/route.go new file mode 100644 index 0000000..6bfd4fe --- /dev/null +++ b/internal/cli/route.go @@ -0,0 +1,215 @@ +package cli + +// `teploy route remove`: retire a managed Caddy block that nothing serves. Only +// a route proven dead goes (removalRefusal), only that app's own marked region +// is edited, and the commit is a compare-and-swap on the exact block that was +// judged, so a deploy that rewrites it in between is not undone. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/signal" + "strings" + + "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/caddy" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/domains" + "github.com/useteploy/teploy/internal/events" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +var errRouteRefused = errors.New("route remove refused") + +type routeRemoveResult struct { + MachineInterface int `json:"machine_interface"` + App string `json:"app"` + Server string `json:"server"` + Route string `json:"route"` // removed | absent; refused/failed appear only in event detail + Hosts []string `json:"hosts,omitempty"` + Upstreams []string `json:"upstreams,omitempty"` +} + +func newRouteCmd(flags *Flags) *cobra.Command { + cmd := &cobra.Command{Use: "route", Short: "Manage Caddy routes"} + var app string + remove := &cobra.Command{ + Use: "remove", + Short: "Remove a managed route that nothing serves", + Long: `Removes one app's managed Caddy block, reloads Caddy, and records it in the +app's event history. Hand-written blocks and other apps' blocks are never +touched. An absent block is success. + +Refuses a route belonging to a live app, and any route not proven dead: an +upstream that resolves or cannot be checked, files that still exist, app state +or containers that cannot be read, or a block carrying the autodeploy webhook.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runRouteRemove(flags, app, cmd.OutOrStdout()) + }, + } + remove.Flags().StringVar(&app, "app", "", "the app whose managed block to remove") + _ = remove.MarkFlagRequired("app") + cmd.AddCommand(remove) + return cmd +} + +func runRouteRemove(flags *Flags, app string, out io.Writer) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + serverName, exec, err := connectAppServer(ctx, flags, app) + if err != nil { + return err + } + defer exec.Close() + res, err := executeRouteRemove(ctx, exec, app) + if err != nil { + return err + } + res.Server = serverName + if flags.JSON { + return json.NewEncoder(out).Encode(res) + } + if res.Route == "absent" { + fmt.Fprintf(out, "%s has no managed route on %s\n", app, serverName) + } else { + fmt.Fprintf(out, "Removed %s's route on %s (%s)\n", app, serverName, strings.Join(res.Hosts, ", ")) + } + return nil +} + +// routeLockApp is the app whose lock serializes an edit of this route. A +// preview's route has no directory of its own, so its parent's lock guards it. +func routeLockApp(ctx context.Context, exec ssh.Executor, app string) (string, error) { + exists, err := appDirExists(ctx, exec, app) + if err != nil || exists { + return app, err + } + parent, _, ok := strings.Cut(app, "-preview-") + if !ok || config.ValidateName(parent) != nil { + return "", nil + } + if exists, err = appDirExists(ctx, exec, parent); err != nil || !exists { + return "", err + } + return parent, nil +} + +// appDirExists separates "no directory" from "could not ask". +func appDirExists(ctx context.Context, exec ssh.Executor, app string) (bool, error) { + out, err := exec.Run(ctx, "if [ -d "+ssh.ShellQuote("/deployments/"+app)+" ]; then echo yes; else echo no; fi") + if err != nil { + return false, err + } + return strings.TrimSpace(out) == "yes", nil +} + +func executeRouteRemove(ctx context.Context, exec ssh.Executor, app string) (*routeRemoveResult, error) { + res := &routeRemoveResult{MachineInterface: MachineInterface, App: app, Server: exec.Host(), Route: "absent"} + record := func(outcome string, err error, reason string) { + events.Record(ctx, exec, events.Event{App: app, Kind: events.KindRouteRemove, Outcome: outcome, + Detail: events.ErrDetail(map[string]any{"route": res.Route, "hosts": res.Hosts, "upstreams": res.Upstreams, "reason": reason}, err)}) + } + + lockApp, err := routeLockApp(ctx, exec, app) + if err != nil { + return nil, err + } + var lk *state.Lock + if lockApp != "" { + if lk, err = state.AcquireLockFenced(ctx, exec, lockApp); err != nil { + return nil, fmt.Errorf("acquiring deploy lock: %w", err) + } + defer state.ReleaseLockFenced(exec, lk, lockApp) + } + + cd := caddy.NewClient(exec) + content, present, err := cd.ReadCaddyfile(ctx) + if err != nil { + return nil, fmt.Errorf("reading the Caddyfile: %w", err) + } + var region *caddy.ManagedRegion + if present { + regions, err := caddy.ManagedRegions(content) + if err != nil { + return nil, fmt.Errorf("reading the Caddyfile: %w", err) + } + for i := range regions { + if regions[i].App == app { + region = ®ions[i] + } + } + } + if region == nil { + record(events.Succeeded, nil, "") + return res, nil + } + res.Hosts = siteHosts(region.Sites) + for _, site := range region.Sites { + res.Upstreams = append(res.Upstreams, site.Upstreams...) + } + + facts := routeFacts{app: app, region: region.Text, sites: region.Sites, resolves: map[string]string{}, roots: map[string]string{}} + st, stErr := state.Read(ctx, exec, app) + facts.st, facts.stateKnown = st, stErr == nil + containers, listErr := docker.NewClient(exec).ListAllContainers(ctx) + facts.containers, facts.known = containers, listErr == nil + facts.resolves = resolveRegionDials(ctx, exec, region.Sites, containers) + for _, root := range caddy.RegionRoots(region.Text) { + facts.roots[root], _ = readRootExists(ctx, exec, root) + } + if reason := removalRefusal(facts); reason != "" { + err := fmt.Errorf("%w: %s: %s", errRouteRefused, app, reason) + res.Route = "refused" + record(events.Refused, nil, reason) + return nil, err + } + + generation, _ := caddy.RegionGeneration(region.Text) + guard := "" + if lk != nil { + guard = lk.GuardPrefix() + } + err = cd.WithCommitGuard(guard).WithRouteCAS(app, []string{caddy.ManagedRegionHash(region.Text)}, generation).RemoveRoute(ctx, app) + if err != nil { + res.Route = "failed" + record(events.Failed, err, "") + return nil, fmt.Errorf("removing %s's route: %w", app, err) + } + res.Route = "removed" + record(events.Succeeded, nil, "") + return res, nil +} + +// resolveRegionDials asks the running Caddy about a region's upstreams. No +// answer leaves a name unknown, which refuses the removal. +func resolveRegionDials(ctx context.Context, exec ssh.Executor, sites []caddy.RouteSite, containers []docker.Container) map[string]string { + running := false + for _, c := range containers { + if c.Name == "caddy" && c.State == "running" { + running = true + } + } + var names []string + for _, site := range sites { + for _, dial := range site.Upstreams { + if name, _ := domains.UpstreamHost(dial); name != "" { + names = append(names, name) + } + } + } + if !running || len(names) == 0 { + return map[string]string{} + } + answers, err := domains.ResolveNames(ctx, exec, names) + if err != nil { + return map[string]string{} + } + return answers +} diff --git a/internal/cli/route_remove_test.go b/internal/cli/route_remove_test.go new file mode 100644 index 0000000..406b807 --- /dev/null +++ b/internal/cli/route_remove_test.go @@ -0,0 +1,281 @@ +package cli + +// `teploy route remove` tests: only a block proven dead goes, the commit is a +// compare-and-swap on the exact block that was judged, and every refusal path +// leaves the Caddyfile untouched. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/caddy" + "github.com/useteploy/teploy/internal/ssh" +) + +const routeRemoveCaddyfile = `example.com { + reverse_proxy other:80 +} + +# TEPLOY BEGIN ghost +ghost.example.com { + reverse_proxy ghost-web-v1:3000 +} +# TEPLOY END ghost +` + +// routeRemoveExecutor models a server whose ghost app is entirely gone: a +// directory with a lock, no state, no containers, a dead upstream. Extra +// registrations are consulted first, so a test can override any read. +func routeRemoveExecutor(caddyfile string, extra ...ssh.MockCommand) *ssh.MockExecutor { + cmds := append([]ssh.MockCommand{}, extra...) + cmds = append(cmds, + ssh.MockCommand{Match: "if [ -d '/deployments/ghost' ]", Output: ""}, + ssh.MockCommand{Match: "mkdir /deployments/ghost/.lock", Output: ""}, + ssh.MockCommand{Match: "docker ps --all ", Output: routeRemoveContainers()}, + ssh.MockCommand{Match: "printf '%s\\n'", Output: "2 ghost-web-v1\n"}, + ssh.MockCommand{Match: "mkdir /deployments/caddy/.lock", Output: ""}, + ssh.MockCommand{Match: "docker exec caddy caddy reload", Output: ""}, + ssh.MockCommand{Match: "a=$(docker exec caddy md5sum", Output: "TEPLOY_CADDY_OK"}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq": 1, "ts": "2026-11-05T12:00:00Z"}`}, + ) + m := ssh.NewMockExecutor("192.0.2.10", cmds...) + if caddyfile != "" { + m.Files["/deployments/caddy/Caddyfile"] = []byte(caddyfile) + } + return m +} + +func routeRemoveContainers() string { + return auditContainer("caddy", "caddy:2", "running", "2026-05-01 00:00:00 +0000 UTC", nil) +} + +func recordedEvents(t *testing.T, inputs []string) []map[string]any { + t.Helper() + var out []map[string]any + for _, input := range inputs { + var ev map[string]any + if err := json.Unmarshal([]byte(input), &ev); err != nil { + continue // a non-event stdin payload (the caddy adapt check) + } + if ev["kind"] == "route.remove" { + out = append(out, ev) + } + } + return out +} + +func TestRouteRemove_RemovesProvenDeadBlock(t *testing.T) { + m := routeRemoveExecutor(routeRemoveCaddyfile) + res, err := executeRouteRemove(context.Background(), m, "ghost") + if err != nil { + t.Fatalf("executeRouteRemove: %v", err) + } + if res.Route != "removed" { + t.Errorf("route = %s, want removed", res.Route) + } + if fmt.Sprint(res.Hosts) != fmt.Sprint([]string{"ghost.example.com"}) { + t.Errorf("hosts = %v", res.Hosts) + } + written := string(m.Files["/deployments/caddy/Caddyfile"]) + if strings.Contains(written, "TEPLOY BEGIN ghost") || strings.Contains(written, "ghost.example.com") { + t.Errorf("the ghost block survived: %q", written) + } + if !strings.Contains(written, "example.com {") { + t.Errorf("an unrelated block was dropped: %q", written) + } + // The removal is committed under the exact-block CAS on the judged block. + var casCommit bool + for _, call := range m.Calls { + if strings.Contains(call, "TEPLOY_ROUTE_CAS_MISMATCH") && strings.Contains(call, "mv -fT -- ") { + casCommit = true + } + } + if !casCommit { + t.Errorf("the commit was not guarded by the route CAS; calls: %v", m.Calls) + } + // Recorded in the app's event history. + evs := recordedEvents(t, m.Inputs) + if len(evs) != 1 || evs[0]["outcome"] != "succeeded" { + t.Fatalf("events = %v", evs) + } + if evs[0]["detail"].(map[string]any)["route"] != "removed" { + t.Errorf("event detail = %v", evs[0]["detail"]) + } +} + +func TestRouteRemove_AbsentBlockIsSuccess(t *testing.T) { + m := routeRemoveExecutor("example.com {\n\treverse_proxy other:80\n}\n") + res, err := executeRouteRemove(context.Background(), m, "ghost") + if err != nil { + t.Fatalf("an absent block must succeed: %v", err) + } + if res.Route != "absent" { + t.Errorf("route = %s, want absent", res.Route) + } +} + +func TestRouteRemove_Refusals(t *testing.T) { + tests := []struct { + name string + expect string + edited string // expected Caddyfile after the refusal ("" = unchanged) + extra []ssh.MockCommand + files map[string][]byte + }{ + { + name: "live app", + expect: "belongs to a live app", + extra: []ssh.MockCommand{{Match: "docker ps --all ", Output: routeRemoveContainers() + "\n" + + auditContainer("ghost-web-v1", "ghost:v1", "running", "2026-11-04 10:00:00 +0000 UTC", + map[string]string{"teploy.app": "ghost", "teploy.process": "web", "teploy.version": "v1"})}}, + files: map[string][]byte{"/deployments/ghost/state.json": []byte(auditStateJSON("container", "caddy", "v1", "", nil))}, + }, + { + name: "container still carries the name", + expect: "still carries this app's name", + extra: []ssh.MockCommand{{Match: "docker ps --all ", Output: routeRemoveContainers() + "\n" + + auditContainer("ghost-web-v1", "ghost:v1", "exited", "2026-11-04 10:00:00 +0000 UTC", + map[string]string{"teploy.app": "ghost", "teploy.process": "web", "teploy.version": "v1"})}}, + }, + { + name: "upstream resolves", + expect: "still resolves", + extra: []ssh.MockCommand{{Match: "printf '%s\\n'", Output: "0 ghost-web-v1\n"}}, + }, + { + name: "upstream unknowable", + expect: "could not be proven dead", + extra: []ssh.MockCommand{{Match: "printf '%s\\n'", Err: errors.New("getent: temporary failure")}}, + }, + { + name: "unreadable state", + expect: "state could not be read", + extra: []ssh.MockCommand{{Match: "if [ ! -e '/deployments/ghost/state.json'", Err: errors.New("permission denied")}}, + }, + { + name: "webhook block", + expect: "autodeploy webhook", + edited: "# TEPLOY BEGIN ghost\nghost.example.com {\n\treverse_proxy ghost-web-v1:3000\n\t# TEPLOY WEBHOOK BEGIN ghost\n\t# TEPLOY WEBHOOK END ghost\n}\n# TEPLOY END ghost\n", + }, + { + name: "serving files that exist", + expect: "serves files from", + edited: "# TEPLOY BEGIN ghost\nghost.example.com {\n\troot * /deployments/ghost/site\n\tfile_server\n}\n# TEPLOY END ghost\n", + extra: []ssh.MockCommand{{Match: "if [ -e '/deployments/ghost/site'", Output: "yes"}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + before := routeRemoveCaddyfile + if tt.edited != "" { + before = tt.edited + } + m := routeRemoveExecutor(before, tt.extra...) + for p, data := range tt.files { + m.Files[p] = data + } + _, err := executeRouteRemove(context.Background(), m, "ghost") + if !errors.Is(err, errRouteRefused) { + t.Fatalf("error = %v, want a route remove refusal", err) + } + if !strings.Contains(err.Error(), tt.expect) { + t.Errorf("refusal = %q, want it to name %q", err, tt.expect) + } + if written := string(m.Files["/deployments/caddy/Caddyfile"]); written != before { + t.Errorf("a refusal must not edit the Caddyfile:\n got: %q\nwant: %q", written, before) + } + for _, call := range m.Calls { + if strings.Contains(call, "mv -fT -- ") { + t.Errorf("a refusal must not commit: %s", call) + } + } + evs := recordedEvents(t, m.Inputs) + if len(evs) != 1 || evs[0]["outcome"] != "refused" { + t.Errorf("refusal events = %v", evs) + } + }) + } +} + +// concurrentRewriteExecutor models a deploy rewriting the app's block between +// `route remove` judging it and committing: the second read of the Caddyfile +// (inside the caddy client's mutate) answers different content. +type concurrentRewriteExecutor struct { + *ssh.MockExecutor + rewrite string + catSeen int + catCaddyfile string + eventAppended bool +} + +func (e *concurrentRewriteExecutor) Run(ctx context.Context, cmd string) (string, error) { + if cmd == e.catCaddyfile { + e.catSeen++ + if e.catSeen == 1 { + // A successor deploys a new block for the app between the + // judgement (the framed read) and the commit (this plain cat + // inside the caddy client's mutate). + e.Files["/deployments/caddy/Caddyfile"] = []byte(e.rewrite) + } + } + if strings.HasPrefix(cmd, "python3 -c") { + e.eventAppended = true + } + return e.MockExecutor.Run(ctx, cmd) +} + +// RunStream records the event append too: the ssh package's structured +// capture falls back to RunStream for executors it does not know, so the +// history append arrives here rather than through Run. +func (e *concurrentRewriteExecutor) RunStream(ctx context.Context, cmd string, stdout, stderr io.Writer) error { + if strings.HasPrefix(cmd, "python3 -c") { + e.eventAppended = true + } + return e.MockExecutor.RunStream(ctx, cmd, stdout, stderr) +} + +func TestRouteRemove_ConcurrentRewriteIsRefused(t *testing.T) { + base := routeRemoveExecutor(routeRemoveCaddyfile) + rewritten := `example.com { + reverse_proxy other:80 +} + +# TEPLOY BEGIN ghost +ghost.example.com { + reverse_proxy ghost-web-v9:3000 +} +# TEPLOY END ghost +` + m := &concurrentRewriteExecutor{MockExecutor: base, rewrite: rewritten, catCaddyfile: "cat /deployments/caddy/Caddyfile"} + _, err := executeRouteRemove(context.Background(), m, "ghost") + if err == nil { + t.Fatal("a block that changed under the removal must refuse") + } + var casErr *caddy.ErrRouteCAS + if !errors.As(err, &casErr) { + t.Fatalf("error = %v, want a route CAS refusal", err) + } + after := string(m.Files["/deployments/caddy/Caddyfile"]) + if after != rewritten { + t.Errorf("a refused commit must leave the successor's block exactly as it is: %q", after) + } + if !strings.Contains(after, "TEPLOY BEGIN ghost") { + t.Errorf("the app's block was removed despite the refusal: %q", after) + } + for path := range m.Files { + if strings.Contains(path, ".tmp-") { + t.Errorf("the staged caddyfile was not cleaned up: %s", path) + } + } + // The wrapper's structured capture falls back to RunStream, so the event + // payload is not captured in Inputs; the failure is still recorded — the + // append call itself must have run. + if !m.eventAppended { + t.Error("the refused removal was not recorded in the event history") + } +} diff --git a/internal/events/events.go b/internal/events/events.go index 28851d1..69ecaaf 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -117,6 +117,7 @@ const ( KindBackupOwners = "backup_target.owners_clear" KindBackupAll = "backup.all" KindBackupPolicy = "backup_policy.set" + KindRouteRemove = "route.remove" KindReleasesPrune = "releases.prune" ) @@ -155,6 +156,7 @@ var detailAllow = map[string][]string{ KindBackupOwners: {"fingerprint", "force", "removed", "error"}, KindBackupAll: {"status", "targets_ok", "targets_failed", "omissions", "encrypted", "duration_ms", "heartbeats_ok", "heartbeats_failed", "warnings", "error"}, KindBackupPolicy: {"interval", "encrypted", "volumes", "adopted", "error"}, + KindRouteRemove: {"route", "hosts", "upstreams", "reason", "error"}, KindReleasesPrune: {"keep", "pruned", "protected", "trigger", "error"}, } From 5938424cce3757069ce34b8fa2cec9769c5c50bf Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:38:44 -0700 Subject: [PATCH 4/6] feat(prune): teploy releases prune --keep N and teploy server prune MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit releases prune removes a container app's old versions beyond --keep: containers and images of everything outside the keep window, never the pinned, current, previous or running versions whatever --keep says. Every read happens under the app's lock and each failure refuses the prune (acting without knowing current, previous or the pins could delete them); removals run fenced. Recorded as a releases.prune event with keep/pruned/protected. --keep 0 prunes everything the protections do not name — deliberately the opposite of keep_versions: 0, documented in the command help; the audit never emits it. server prune --build-cache --dangling reclaims docker's unused build cache and dangling images and nothing else — no tagged image, no container, no volume. Recorded as a server.prune event under _fleet. remove: an app whose directory is already gone (an earlier removal finished) now succeeds instead of failing its state deletion — the audit's one-click remove remedy must be idempotent against a second click. The test now models the directory and lock a real remove sees. --- internal/cli/releases.go | 1 + internal/cli/remove.go | 7 +- internal/cli/remove_test.go | 2 + internal/cli/retention.go | 222 ++++++++++++++++++++++++++ internal/cli/retention_test.go | 284 +++++++++++++++++++++++++++++++++ internal/cli/server.go | 1 + internal/events/events.go | 2 + 7 files changed, 517 insertions(+), 2 deletions(-) create mode 100644 internal/cli/retention.go create mode 100644 internal/cli/retention_test.go diff --git a/internal/cli/releases.go b/internal/cli/releases.go index 30563e9..d5de7f1 100644 --- a/internal/cli/releases.go +++ b/internal/cli/releases.go @@ -50,6 +50,7 @@ for teploy.yml-in-a-folder invocations without --app).`, }, } cmd.Flags().StringVar(&appName, "app", "", "app name — read state from the server instead of teploy.yml (requires --host)") + cmd.AddCommand(newReleasesPruneCmd(flags)) return cmd } diff --git a/internal/cli/remove.go b/internal/cli/remove.go index 1b508d3..8ecc99f 100644 --- a/internal/cli/remove.go +++ b/internal/cli/remove.go @@ -378,10 +378,13 @@ func executeRemove(ctx context.Context, exec ssh.Executor, app string, domains [ for _, keep := range keeps { keepArgs += " ! -name " + ssh.ShellQuote(keep) } + // No directory (lk nil) means an earlier removal already finished. delCmd := fmt.Sprintf("find %s -mindepth 1 -maxdepth 1%s -exec rm -rf {} +", ssh.ShellQuote(appDir), keepArgs) - if _, err := exec.Run(ctx, delCmd); err != nil { - return nil, fmt.Errorf("removing deploy state under %s: %w", appDir, err) + if lk != nil { + if _, err := exec.Run(ctx, delCmd); err != nil { + return nil, fmt.Errorf("removing deploy state under %s: %w", appDir, err) + } } if left, err := exec.Run(ctx, "ls "+ssh.ShellQuote(appDir)+" 2>/dev/null"); err == nil { for _, d := range strings.Fields(left) { diff --git a/internal/cli/remove_test.go b/internal/cli/remove_test.go index c729fbe..644dfc7 100644 --- a/internal/cli/remove_test.go +++ b/internal/cli/remove_test.go @@ -110,6 +110,8 @@ func TestExecuteRemovePurgeFallsBackToContainerForForeignOwnedData(t *testing.T) func TestExecuteRemoveDefaultPreservesData(t *testing.T) { cmds := append([]ssh.MockCommand{ removeTestContainers(), + {Match: "[ -d '/deployments/scratch' ]", Output: ""}, + {Match: "mkdir /deployments/scratch/.lock", Output: ""}, {Match: "docker stop -t 10 'scratch-web-aaa'", Output: ""}, {Match: "docker rm 'scratch-web-aaa'", Output: ""}, {Match: "find '/deployments/scratch'", Output: ""}, diff --git a/internal/cli/retention.go b/internal/cli/retention.go new file mode 100644 index 0000000..2f421e7 --- /dev/null +++ b/internal/cli/retention.go @@ -0,0 +1,222 @@ +package cli + +// The two disk remedies `server audit` offers: `releases prune --keep N` and +// `server prune --build-cache --dangling`. Neither touches a volume. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/signal" + "strings" + + "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/docker" + "github.com/useteploy/teploy/internal/events" + "github.com/useteploy/teploy/internal/ssh" + "github.com/useteploy/teploy/internal/state" +) + +type releasesPruneResult struct { + MachineInterface int `json:"machine_interface"` + App string `json:"app"` + Server string `json:"server"` + Keep int `json:"keep"` + Pruned []string `json:"pruned"` + Protected []string `json:"protected"` +} + +func newReleasesPruneCmd(flags *Flags) *cobra.Command { + var app string + var keep int + cmd := &cobra.Command{ + Use: "prune", + Short: "Remove old versions' containers and images beyond --keep", + Long: `Keeps the --keep most recent versions of a container app and removes the +containers and images of the rest. Pinned, current and previous versions and any +version with a running container are never removed, whatever --keep says. Volumes +and accessories are not touched. Takes the app's lock; recorded in its event +history. Nothing beyond --keep is success. + +--keep 0 removes every version the protections above do not name — unlike +keep_versions: 0 in teploy.yml, which keeps everything.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if !cmd.Flags().Changed("keep") { + return fmt.Errorf("--keep is required: how many recent versions to retain") + } + if keep < 0 { + return fmt.Errorf("--keep must be >= 0") + } + return runReleasesPrune(flags, app, keep, cmd.OutOrStdout()) + }, + } + cmd.Flags().StringVar(&app, "app", "", "app name — act on server state instead of teploy.yml (requires --host)") + cmd.Flags().IntVar(&keep, "keep", 0, "how many recent versions to retain") + return cmd +} + +func runReleasesPrune(flags *Flags, app string, keep int, out io.Writer) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + appCfg, exec, err := resolveApp(ctx, flags, app) + if err != nil { + return err + } + defer exec.Close() + res, err := executeReleasesPrune(ctx, exec, appCfg.App, keep) + if err != nil { + return err + } + if flags.JSON { + return json.NewEncoder(out).Encode(res) + } + if len(res.Pruned) == 0 { + fmt.Fprintf(out, "Nothing to prune for %s\n", res.App) + } else { + fmt.Fprintf(out, "Pruned %d version(s) of %s: %s\n", len(res.Pruned), res.App, strings.Join(res.Pruned, ", ")) + } + return nil +} + +func executeReleasesPrune(ctx context.Context, exec ssh.Executor, app string, keep int) (*releasesPruneResult, error) { + res := &releasesPruneResult{MachineInterface: MachineInterface, App: app, Server: exec.Host(), Keep: keep, Pruned: []string{}, Protected: []string{}} + exists, err := appDirExists(ctx, exec, app) + if err != nil { + return nil, err + } + if !exists { + return res, nil // already gone + } + lk, err := state.AcquireLockFenced(ctx, exec, app) + if err != nil { + return nil, fmt.Errorf("acquiring deploy lock: %w", err) + } + defer state.ReleaseLockFenced(exec, lk, app) + + // Every read is under the lock, and each failure stops the prune: acting + // without knowing current, previous or the pins could delete them. + st, err := state.Read(ctx, exec, app) + if err != nil { + return nil, fmt.Errorf("reading state for %s: %w", app, err) + } + if st == nil { + return res, nil + } + if st.DeploymentType == "static" { + return nil, fmt.Errorf("%s is a static app: its releases are pruned on deploy by keep_releases", app) + } + pins, err := state.ReadPins(ctx, exec, app) + if err != nil { + return nil, fmt.Errorf("refusing to prune: %w", err) + } + dk := docker.NewClient(exec) + containers, err := dk.ListContainers(ctx, app) + if err != nil { + return nil, fmt.Errorf("refusing to prune: %w", err) + } + for _, v := range retentionProtected(st, pins, containers) { + if v != "" { + res.Protected = append(res.Protected, v) + } + } + if len(docker.PrunableVersions(containers, keep, res.Protected...)) == 0 { + return res, nil + } + fenced := docker.NewClient(&state.FencedExecutor{Executor: exec, Lock: lk}) + pruned, pruneErr := fenced.PruneVersions(ctx, app, keep, res.Protected...) + res.Pruned = append(res.Pruned, pruned...) + events.Record(ctx, exec, events.Event{App: app, Kind: events.KindReleasesPrune, Outcome: events.Outcome(pruneErr), Release: st.CurrentHash, + Detail: events.ErrDetail(map[string]any{"keep": keep, "pruned": res.Pruned, "protected": res.Protected, "trigger": "command"}, pruneErr)}) + if pruneErr != nil { + return nil, fmt.Errorf("prune incomplete: %w", pruneErr) + } + return res, nil +} + +type serverPruneResult struct { + MachineInterface int `json:"machine_interface"` + Server string `json:"server"` + BuildCache bool `json:"build_cache"` + Dangling bool `json:"dangling"` + Output string `json:"output"` +} + +func newServerPruneCmd(flags *Flags) *cobra.Command { + var buildCache, dangling bool + cmd := &cobra.Command{ + Use: "prune", + Short: "Remove unused build cache and dangling images", + Long: `--build-cache removes docker's unused build cache: the next build rebuilds it. +--dangling removes untagged images no container uses. Nothing else: no tagged +image, no container, no volume. Recorded in the server's event history.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if !buildCache && !dangling { + return fmt.Errorf("say what to prune: --build-cache and/or --dangling") + } + if flags.Host == "" { + return fmt.Errorf("--host is required") + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + host, user, key, err := config.ResolveServer(flags.Host, flags.Host, flags.User, flags.Key) + if err != nil { + return err + } + exec, err := ssh.Connect(ctx, ssh.ConnectConfig{Host: host, User: user, KeyPath: key}) + if err != nil { + return err + } + defer exec.Close() + res, err := executeServerPrune(ctx, exec, flags.Host, buildCache, dangling) + if err != nil { + return err + } + if flags.JSON { + return json.NewEncoder(cmd.OutOrStdout()).Encode(res) + } + fmt.Fprintln(cmd.OutOrStdout(), res.Output) + return nil + }, + } + cmd.Flags().BoolVar(&buildCache, "build-cache", false, "remove unused build cache") + cmd.Flags().BoolVar(&dangling, "dangling", false, "remove dangling images") + return cmd +} + +func executeServerPrune(ctx context.Context, exec ssh.Executor, server string, buildCache, dangling bool) (*serverPruneResult, error) { + res := &serverPruneResult{MachineInterface: MachineInterface, Server: server, BuildCache: buildCache, Dangling: dangling} + var failures []error + var lines []string + run := func(cmd string) { + out, err := exec.Run(ctx, cmd) + if err != nil { + failures = append(failures, fmt.Errorf("%s: %w", cmd, err)) + return + } + for _, line := range doctorLines(out) { + if strings.HasPrefix(line, "Total") { + lines = append(lines, line) + } + } + } + if buildCache { + run("docker builder prune -a -f") + } + if dangling { + run("docker image prune -f") + } + res.Output = strings.Join(lines, "; ") + err := errors.Join(failures...) + events.Record(ctx, exec, events.Event{App: "_fleet", Kind: events.KindServerPrune, Outcome: events.Outcome(err), + Detail: events.ErrDetail(map[string]any{"build_cache": buildCache, "dangling": dangling, "output": res.Output}, err)}) + if err != nil { + return nil, err + } + return res, nil +} diff --git a/internal/cli/retention_test.go b/internal/cli/retention_test.go new file mode 100644 index 0000000..52fe0a5 --- /dev/null +++ b/internal/cli/retention_test.go @@ -0,0 +1,284 @@ +package cli + +// `releases prune --keep N` and `server prune --build-cache --dangling` +// tests: the selection keeps exactly what the plan protects, and acting +// without knowing current, previous or the pins refuses. + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/ssh" +) + +// pruneFixture models a container app with versions v1..v4 on a server. +func pruneFixture(t *testing.T, manifest string, pins string, extra ...ssh.MockCommand) *ssh.MockExecutor { + t.Helper() + cmds := append([]ssh.MockCommand{}, extra...) + cmds = append(cmds, + ssh.MockCommand{Match: "if [ -d '/deployments/blog' ]", Output: "yes"}, + ssh.MockCommand{Match: "mkdir /deployments/blog/.lock", Output: ""}, + ssh.MockCommand{Match: "docker ps --all --filter label=teploy.app='blog'", Output: strings.Join([]string{ + auditContainer("blog-web-v4", "blog:v4", "running", "2026-11-04 09:00:00 +0000 UTC", + map[string]string{"teploy.app": "blog", "teploy.process": "web", "teploy.version": "v4"}), + auditContainer("blog-web-v3", "blog:v3", "exited", "2026-11-03 09:00:00 +0000 UTC", + map[string]string{"teploy.app": "blog", "teploy.process": "web", "teploy.version": "v3"}), + auditContainer("blog-web-v2", "blog:v2", "exited", "2026-11-02 09:00:00 +0000 UTC", + map[string]string{"teploy.app": "blog", "teploy.process": "web", "teploy.version": "v2"}), + auditContainer("blog-web-v1", "blog:v1", "exited", "2026-11-01 09:00:00 +0000 UTC", + map[string]string{"teploy.app": "blog", "teploy.process": "web", "teploy.version": "v1"}), + }, "\n")}, + ssh.MockCommand{Match: "docker rm -f", Output: ""}, + ssh.MockCommand{Match: "docker rmi", Output: ""}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq": 1, "ts": "2026-11-05T12:00:00Z"}`}, + ) + m := ssh.NewMockExecutor("192.0.2.10", cmds...) + m.Files["/deployments/blog/state.json"] = []byte(auditStateJSON("container", "caddy", "v4", "v3", manifestMap(manifest))) + if pins != "" { + m.Files["/deployments/blog/pinned"] = []byte(pins) + } + return m +} + +func manifestMap(s string) map[string]any { + if s == "" { + return nil + } + var m map[string]any + if err := json.Unmarshal([]byte(s), &m); err != nil { + panic(err) + } + return m +} + +func TestReleasesPrune_KeepsExactlyTheExpectedSet(t *testing.T) { + m := pruneFixture(t, `{"container":{"keep_versions":2}}`, "v1\n") + res, err := executeReleasesPrune(context.Background(), m, "blog", 2) + if err != nil { + t.Fatalf("executeReleasesPrune: %v", err) + } + if fmt.Sprint(res.Pruned) != fmt.Sprint([]string{"v2"}) { + t.Errorf("pruned = %v, want [v2] (v4 current, v3 previous, v1 pinned)", res.Pruned) + } + if fmt.Sprint(res.Protected) != fmt.Sprint([]string{"v4", "v3", "v1"}) { + t.Errorf("protected = %v, want [v4 v3 v1]", res.Protected) + } + for _, call := range m.Calls { + if strings.HasPrefix(call, "docker rm -f") || strings.HasPrefix(call, "docker rmi") { + for _, keep := range []string{"v1", "v3", "v4"} { + if strings.Contains(call, keep) { + t.Errorf("protected version %s was removed: %s", keep, call) + } + } + } + } + if !sawCall(m, "docker rm -f 'blog-web-v2'") || !sawCall(m, "docker rmi 'blog:v2'") { + t.Errorf("v2's container and image were not both removed; calls: %v", m.Calls) + } +} + +// keep 0 on the command means "everything the protections do not name" — +// deliberately different from keep_versions: 0 in teploy.yml. +func TestReleasesPrune_KeepZeroPrunesAllUnprotected(t *testing.T) { + m := pruneFixture(t, `{"container":{"keep_versions":2}}`, "v1\n") + res, err := executeReleasesPrune(context.Background(), m, "blog", 0) + if err != nil { + t.Fatalf("executeReleasesPrune: %v", err) + } + if fmt.Sprint(res.Pruned) != fmt.Sprint([]string{"v2"}) { + t.Errorf("pruned = %v, want [v2]: current, previous and pins survive even --keep 0", res.Pruned) + } +} + +func TestReleasesPrune_NothingBeyondKeepIsSuccess(t *testing.T) { + m := pruneFixture(t, `{"container":{"keep_versions":5}}`, "") + res, err := executeReleasesPrune(context.Background(), m, "blog", 5) + if err != nil { + t.Fatalf("executeReleasesPrune: %v", err) + } + if len(res.Pruned) != 0 { + t.Errorf("pruned = %v, want nothing", res.Pruned) + } + for _, call := range m.Calls { + if strings.HasPrefix(call, "docker rm") || strings.HasPrefix(call, "docker rmi") { + t.Errorf("nothing was beyond keep, yet: %s", call) + } + } +} + +func TestReleasesPrune_RunsUnderTheAppLock(t *testing.T) { + m := pruneFixture(t, `{"container":{"keep_versions":2}}`, "v1\n") + if _, err := executeReleasesPrune(context.Background(), m, "blog", 2); err != nil { + t.Fatal(err) + } + if !sawCall(m, "mkdir /deployments/blog/.lock 2>/dev/null") { + t.Errorf("the prune never took the app lock; calls: %v", m.Calls) + } + // Container removals run under the lock's fence guard. + var fenced bool + for _, call := range m.Calls { + if strings.HasPrefix(call, "grep -q ") && strings.Contains(call, "docker rm -f") { + fenced = true + } + } + if !fenced { + t.Errorf("container removals were not fenced by the lock; calls: %v", m.Calls) + } +} + +func TestReleasesPrune_RefusesWhenItCannotKnowTheProtections(t *testing.T) { + tests := []struct { + name string + extra []ssh.MockCommand + want string + }{ + { + name: "unreadable pins", + extra: []ssh.MockCommand{{Match: "if [ ! -e '/deployments/blog/pinned'", Err: errors.New("permission denied")}}, + want: "refusing to prune", + }, + { + name: "unreadable state", + extra: []ssh.MockCommand{{Match: "if [ ! -e '/deployments/blog/state.json'", Err: errors.New("permission denied")}}, + want: "reading state", + }, + { + name: "unreadable containers", + extra: []ssh.MockCommand{{Match: "docker ps --all --filter label=teploy.app='blog'", Err: errors.New("docker: no daemon")}}, + want: "refusing to prune", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := pruneFixture(t, `{"container":{"keep_versions":1}}`, "", tt.extra...) + _, err := executeReleasesPrune(context.Background(), m, "blog", 1) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("error = %v, want it to contain %q", err, tt.want) + } + for _, call := range m.Calls { + if strings.HasPrefix(call, "docker rm") || strings.HasPrefix(call, "docker rmi") { + t.Errorf("a refused prune removed things: %s", call) + } + } + }) + } +} + +func TestReleasesPrune_StaticAppExplainsItself(t *testing.T) { + m := pruneFixture(t, "", "") + m.Files["/deployments/blog/state.json"] = []byte(auditStateJSON("static", "caddy", "s1", "", nil)) + _, err := executeReleasesPrune(context.Background(), m, "blog", 2) + if err == nil || !strings.Contains(err.Error(), "keep_releases") { + t.Fatalf("error = %v, want the static keep_releases explanation", err) + } +} + +func TestReleasesPrune_AlreadyRemovedAppIsSuccess(t *testing.T) { + m := ssh.NewMockExecutor("192.0.2.10", + ssh.MockCommand{Match: "if [ -d '/deployments/blog' ]", Output: "no"}, + ) + res, err := executeReleasesPrune(context.Background(), m, "blog", 2) + if err != nil { + t.Fatalf("an app with no directory must be a no-op success: %v", err) + } + if len(res.Pruned) != 0 { + t.Errorf("pruned = %v", res.Pruned) + } +} + +func sawCall(m *ssh.MockExecutor, prefix string) bool { + for _, call := range m.Calls { + if strings.HasPrefix(call, prefix) { + return true + } + if strings.Contains(call, prefix) { // a fenced/guarded composition + return true + } + } + return false +} + +func TestServerPrune_RunsExactlyTheSelectedPrunes(t *testing.T) { + m := ssh.NewMockExecutor("192.0.2.10", + ssh.MockCommand{Match: "docker builder prune -a -f", Output: "Total reclaimed space: 1.5GB"}, + ssh.MockCommand{Match: "docker image prune -f", Output: "Total reclaimed space: 300MB"}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq": 1, "ts": "2026-11-05T12:00:00Z"}`}, + ) + res, err := executeServerPrune(context.Background(), m, "edge", true, true) + if err != nil { + t.Fatalf("executeServerPrune: %v", err) + } + if !res.BuildCache || !res.Dangling { + t.Errorf("result flags = %+v", res) + } + if res.Output != "Total reclaimed space: 1.5GB; Total reclaimed space: 300MB" { + t.Errorf("output = %q", res.Output) + } + // Only the two reclaim prunes ran: no tagged image, container or volume. + for _, call := range m.Calls { + if strings.Contains(call, "docker volume prune") || strings.Contains(call, "docker system prune") || + strings.Contains(call, "docker container prune") || strings.Contains(call, "-a -f --volumes") { + t.Errorf("server prune went beyond build cache and dangling images: %s", call) + } + } + // Recorded under the fleet pseudo-app. + var fleetEvent bool + for _, input := range m.Inputs { + if strings.Contains(input, `"app":"_fleet"`) && strings.Contains(input, `"server.prune"`) { + fleetEvent = true + } + } + if !fleetEvent { + t.Errorf("no server.prune event under _fleet; inputs: %v", m.Inputs) + } +} + +func TestServerPrune_OnlyTheSelectedFlagRuns(t *testing.T) { + m := ssh.NewMockExecutor("192.0.2.10", + ssh.MockCommand{Match: "docker image prune -f", Output: "Total reclaimed space: 300MB"}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq": 1, "ts": "2026-11-05T12:00:00Z"}`}, + ) + if _, err := executeServerPrune(context.Background(), m, "edge", false, true); err != nil { + t.Fatal(err) + } + for _, call := range m.Calls { + if strings.Contains(call, "builder prune") { + t.Errorf("--build-cache was not asked for: %s", call) + } + } +} + +func TestServerPrune_FailureIsReported(t *testing.T) { + m := ssh.NewMockExecutor("192.0.2.10", + ssh.MockCommand{Match: "docker builder prune -a -f", Err: errors.New("docker: no space left on device")}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq": 1, "ts": "2026-11-05T12:00:00Z"}`}, + ) + _, err := executeServerPrune(context.Background(), m, "edge", true, false) + if err == nil { + t.Fatal("a failed prune must surface its error") + } +} + +func TestServerPruneFlagValidation(t *testing.T) { + var out bytes.Buffer + cmd := newServerPruneCmd(&Flags{}) + cmd.SetOut(&out) + cmd.SetErr(&out) + cmd.SetArgs([]string{}) + if err := cmd.Execute(); err == nil || !strings.Contains(err.Error(), "say what to prune") { + t.Errorf("no flags: %v", err) + } + + cmd = newServerPruneCmd(&Flags{}) + cmd.SetOut(&out) + cmd.SetErr(&out) + cmd.SetArgs([]string{"--build-cache"}) + if err := cmd.Execute(); err == nil || !strings.Contains(err.Error(), "--host is required") { + t.Errorf("missing --host: %v", err) + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 6037a64..9930a9b 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -30,6 +30,7 @@ func newServerCmd(flags *Flags) *cobra.Command { cmd.AddCommand(newServerBackupTargetCmd(flags)) cmd.AddCommand(newServerBackupPolicyCmd(flags)) cmd.AddCommand(newServerAuditCmd(flags)) + cmd.AddCommand(newServerPruneCmd(flags)) return cmd } diff --git a/internal/events/events.go b/internal/events/events.go index 69ecaaf..d4d514b 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -119,6 +119,7 @@ const ( KindBackupPolicy = "backup_policy.set" KindRouteRemove = "route.remove" KindReleasesPrune = "releases.prune" + KindServerPrune = "server.prune" ) // detailAllow is the per-kind allowlist of detail keys. Env and secret @@ -158,6 +159,7 @@ var detailAllow = map[string][]string{ KindBackupPolicy: {"interval", "encrypted", "volumes", "adopted", "error"}, KindRouteRemove: {"route", "hosts", "upstreams", "reason", "error"}, KindReleasesPrune: {"keep", "pruned", "protected", "trigger", "error"}, + KindServerPrune: {"build_cache", "dangling", "output", "error"}, } // maxErrorDetail bounds the "error" detail so a command's full stderr From 74002247b4486b433ed8d96b0711d89846984903 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:38:44 -0700 Subject: [PATCH 5/6] contracts: server-audit-envelope, corpus rev 19 (server-audit-v1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New artifact server-audit-envelope: the `teploy server audit --json` envelope — findings with stable kind:subject ids, severity, evidence and remedy {argv (null where nothing the CLI does is safe), destructive, confirm}, per-source checks where unknown means the findings depending on it are missing rather than clean, and disk attribution. Schema derived from the real serverAuditEnvelope DTO; fixture GENERATED from the real auditServer over a mock executor (one of each finding including null-remedy classes, one unknown check, partial state). version-handshake advertises server-audit-v1 (mi2.json regenerated; the capability registry golden updated to force the addition). MANIFEST bumped to rev 19 with the artifact row. No MI bump. --- contracts/MANIFEST.md | 2 + .../server-audit-envelope/valid/findings.json | 352 ++++++++++++++++++ .../fixtures/version-handshake/valid/mi2.json | 1 + .../schema/server-audit-envelope.schema.json | 83 +++++ internal/cli/contracts_golden_test.go | 10 + internal/cli/machineinterface.go | 9 +- internal/cli/machineinterface_test.go | 2 +- 7 files changed, 457 insertions(+), 2 deletions(-) create mode 100644 contracts/fixtures/server-audit-envelope/valid/findings.json create mode 100644 contracts/schema/server-audit-envelope.schema.json diff --git a/contracts/MANIFEST.md b/contracts/MANIFEST.md index 5f6720b..f520cbc 100644 --- a/contracts/MANIFEST.md +++ b/contracts/MANIFEST.md @@ -10,6 +10,7 @@ Neutron/Nucleus dependency and a public mirror. | Corpus rev | Emitting CLI | Machine Interface | Notes | |---|---|---|---| +| 19 | main (Dash auto-config Slice 6: cleanup / audit) | 2 | Additive. New artifact server-audit-envelope (`teploy server audit --json`, read-only: findings with stable `id` = kind:subject, severity, evidence, and remedy {argv (null when nothing the CLI can do safely), destructive, confirm}, per-source `checks` where unknown means findings depending on it are missing — never clean — and disk attribution), fixture GENERATED from the real `auditServer` over a mock executor (one of each finding incl. null-remedy, one unknown check). New remedy commands `route remove --app X` (exact-block CAS), `releases prune --keep N` (pins/current/previous/running never pruned) and `server prune --build-cache --dangling`; events `route.remove`, `releases.prune`, `server.prune` (the last under `_fleet`). keep_versions becomes optional: unset → 5 pruned only after a 7-day notice recorded server-side (`deploy/retention.go`), `init` writes 3, explicit 0 keeps everything and records `keep_all_versions` in the applied manifest (unset and N record what they always did, so existing digests hold). version-handshake advertises `server-audit-v1` (regenerated mi2.json). No MI bump. | | 18 | main (Dash auto-config Slice 4: fleet backup engine) | 2 | Additive. New artifacts backup-status-envelope (`backup status --json`: every DB-like container and app volume with coverage state, last success/failure, newest object, last verify, plus target and policy summaries, the aws CLI's `aws_cli_problem` and the target's `incoming_lifecycle`; no secrets), backup-run-envelope (`backup run --all --json`) and backup-policy-envelope (`server backup-policy show/set --json`), fixtures GENERATED from the real DTOs. backup-list rows gain optional `encrypted`, `previous_prefix`, and `database` only with `--databases` or `--adopted` (a caller restoring by id alone would replace the primary); backup-create gains `encrypted` and `objects`; backup-restore gains `pre_backup_id`; backup-target gains `previous_prefixes`. Event kinds `backup.all`, `backup_policy.set` and `backup_target.owners_clear` (a marker removal or refusal) under the pseudo-app `_fleet`; adopted containers log under `_adopted`. version-handshake advertises `backup-fleet-v1` (regenerated mi2.json). No MI bump. | | 17 | main (dash auto-config slice 5: domain health) | 2 | Additive. New artifact domains-check-envelope (`teploy domains check [--server] [--app] [--log-window] --json`: per (server, app, host) DNS via the fixed public resolvers, Caddy certificate failures from its logs — `failures` is null, never 0, on short or unavailable logs — and the route with upstream resolution; per-server state so an unreachable server is explicit), fixture GENERATED from the real `sweepDomains` over a mock executor. version-handshake advertises `domains-check-v1` (regenerated mi2.json). release-record's `caddy` gains optional `tls_http_only` (absent on older records; every caddy release since records true or false). No MI bump. | | 16 | main (server-authority spec P3 / plan D4, git sources) | 2 | Additive. New artifacts source-show-envelope (`config source show --json`: repository/branch/poll, public credential facts — type, deploy-key public half + fingerprint, token username, pinned host-key fingerprints — and the per-ref admission state), source-key-envelope (`config source key --json` for ssh/token and `host-key --json`) and trigger-poll-envelope (`trigger poll --json`: one result object per app — ref, head, admitted, reason vocabulary incl. force-push/retry-limit/not-due, deployed_release), all fixtures GENERATED from the real DTOs. version-handshake advertises `git-source-v1` (git: code source, per-app credentials outside /deployments, build_secrets) and `trigger-poll-v1` (the one admitter + reset/retry/install-timer) (regenerated mi2.json). No MI bump. | @@ -60,6 +61,7 @@ Neutron/Nucleus dependency and a public mirror. | backup-run-envelope | yes (rev 18) | valid x2 (partial, skipped; real backupRunEnvelope) | teploy-cli | | backup-policy-envelope | yes (rev 18) | valid x2 (configured, unconfigured; real backupPolicyDTO) | teploy-cli | | domains-check-envelope | yes (rev 17) | valid (one server with each finding, one unreachable; real `sweepDomains`) | teploy-cli | +| server-audit-envelope | yes (rev 19) | valid (one of each finding incl. null-remedy, one unknown check; real `auditServer` over a mock executor) | teploy-cli | | releases-envelope | yes (rev 14) | valid x2 (container app, static app; real releasesDTO) | teploy-cli | | rollback-result | yes (rev 14) | valid (real rollbackResult) | teploy-cli | | redeploy-result | yes (rev 14) | valid x2 (teploy-home, folder-home; real redeployDTO) | teploy-cli | diff --git a/contracts/fixtures/server-audit-envelope/valid/findings.json b/contracts/fixtures/server-audit-envelope/valid/findings.json new file mode 100644 index 0000000..de4683b --- /dev/null +++ b/contracts/fixtures/server-audit-envelope/valid/findings.json @@ -0,0 +1,352 @@ +{ + "machine_interface": 2, + "server": "edge", + "host": "192.0.2.10", + "audited_at": "2026-11-05T12:00:00Z", + "state": "partial", + "checks": [ + { + "name": "apps", + "status": "unknown", + "detail": "could not read the state of broken" + }, + { + "name": "caddyfile", + "status": "ok" + }, + { + "name": "containers", + "status": "ok" + }, + { + "name": "container_sizes", + "status": "ok" + }, + { + "name": "upstreams", + "status": "ok" + }, + { + "name": "static_roots", + "status": "ok" + }, + { + "name": "addresses", + "status": "ok" + }, + { + "name": "docker_usage", + "status": "ok" + }, + { + "name": "dangling_images", + "status": "ok" + }, + { + "name": "dangling_volumes", + "status": "ok" + }, + { + "name": "disk", + "status": "ok" + }, + { + "name": "scheduler", + "status": "ok" + }, + { + "name": "retention", + "status": "ok" + } + ], + "disk": { + "path": "/", + "available_bytes": 800000000, + "used_percent": 96, + "attribution": [ + { + "name": "images", + "size_bytes": 20000000000 + }, + { + "name": "container_layers", + "size_bytes": 5000000000 + }, + { + "name": "volumes", + "size_bytes": 4000000000 + }, + { + "name": "build_cache", + "size_bytes": 2000000000 + }, + { + "name": "deployments", + "size_bytes": 53687091200 + } + ] + }, + "findings": [ + { + "id": "disk_pressure:/", + "kind": "disk_pressure", + "severity": "critical", + "subject": "/", + "evidence": "800.0 MB available on / (96% used); deployments 53.7 GB, images 20.0 GB, container_layers 5.0 GB, volumes 4.0 GB, build_cache 2.0 GB", + "related": [ + "dangling_volumes:docker", + "large_writable_layer:reacher-web-1f2e3d", + "reclaimable_build_artifacts:docker", + "unmanaged_container:portainer", + "versions_beyond_policy:many", + "versions_beyond_policy:unset" + ], + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "managed_route_dead_upstream:blog", + "kind": "managed_route_dead_upstream", + "severity": "critical", + "subject": "blog", + "app": "blog", + "evidence": "blog.example.com routes to blog-web-v3:3000, which does not resolve in Caddy's network", + "remedy": { + "argv": [ + "teploy", + "redeploy", + "--app", + "blog", + "--host", + "edge" + ], + "destructive": false, + "confirm": true + } + }, + { + "id": "scheduler_missing:cron", + "kind": "scheduler_missing", + "severity": "critical", + "subject": "cron", + "evidence": "jobs are scheduled but no cron daemon is running, so none of them run: run `teploy setup` against the server (it installs and enables cron), or on the server: apt-get install cron \u0026\u0026 systemctl enable --now cron", + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "large_writable_layer:reacher-web-1f2e3d", + "kind": "large_writable_layer", + "severity": "warning", + "subject": "reacher-web-1f2e3d", + "app": "reacher", + "evidence": "reacher-web-1f2e3d (running) has written 35.7 GB to its container layer, which a redeploy discards: move that data to a volume", + "size_bytes": 35700000000, + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "manual_route_dead_upstream:hand.example.com", + "kind": "manual_route_dead_upstream", + "severity": "warning", + "subject": "hand.example.com", + "evidence": "hand-written block for hand.example.com routes to gone-svc:80, which does not resolve", + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "orphan_state:orphan", + "kind": "orphan_state", + "severity": "warning", + "subject": "orphan", + "app": "orphan", + "evidence": "orphan has deploy state (release o1) but no containers; removing keeps its volumes, accessory data and credentials", + "remedy": { + "argv": [ + "teploy", + "remove", + "--app", + "orphan", + "--host", + "edge", + "-y" + ], + "destructive": true, + "confirm": true + } + }, + { + "id": "stale_route:files", + "kind": "stale_route", + "severity": "warning", + "subject": "files", + "app": "files", + "evidence": "no app files on this server, but its managed block serves files.example.com; not removable: it serves files from /deployments/files/site, which exists", + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "stale_route:ghost", + "kind": "stale_route", + "severity": "warning", + "subject": "ghost", + "app": "ghost", + "evidence": "no app ghost on this server, but its managed block serves ghost.example.com and routes to ghost-web-v1:3000, which does not resolve", + "remedy": { + "argv": [ + "teploy", + "route", + "remove", + "--app", + "ghost", + "--host", + "edge" + ], + "destructive": false, + "confirm": false + } + }, + { + "id": "unmanaged_container:portainer", + "kind": "unmanaged_container", + "severity": "warning", + "subject": "portainer", + "evidence": "portainer (running) is not managed by teploy and holds 2.0 GB in its container layer", + "size_bytes": 2000000000, + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "versions_beyond_policy:unset", + "kind": "versions_beyond_policy", + "severity": "warning", + "subject": "unset", + "app": "unset", + "evidence": "2 version(s) beyond keep_versions 5: u2, u1; keep_versions is not set, so deploys after 2026-11-08 remove them automatically; set keep_versions (0 keeps everything) or pin what you need", + "size_bytes": 20000000, + "remedy": { + "argv": [ + "teploy", + "releases", + "prune", + "--app", + "unset", + "--host", + "edge", + "--keep", + "5" + ], + "destructive": true, + "confirm": true + } + }, + { + "id": "dangling_volumes:docker", + "kind": "dangling_volumes", + "severity": "info", + "subject": "docker", + "evidence": "21 volume(s) attached to no container, 3.1 GB reclaimable: 0000000000000000000000000000000xxxxxxxdeadbeef, 0000000000000000000000000000001xxxxxxxdeadbeef, 0000000000000000000000000000002xxxxxxxdeadbeef, 0000000000000000000000000000003xxxxxxxdeadbeef, 0000000000000000000000000000004xxxxxxxdeadbeef (+16 more)", + "size_bytes": 3150000000, + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "empty_managed_region:hollow", + "kind": "empty_managed_region", + "severity": "info", + "subject": "hollow", + "app": "hollow", + "evidence": "the managed region holds no site block", + "remedy": { + "argv": [ + "teploy", + "route", + "remove", + "--app", + "hollow", + "--host", + "edge" + ], + "destructive": false, + "confirm": false + } + }, + { + "id": "manual_route_nonlocal_address:203.0.113.9", + "kind": "manual_route_nonlocal_address", + "severity": "info", + "subject": "203.0.113.9", + "evidence": "hand-written block is addressed to 203.0.113.9, which is not an address of this server", + "remedy": { + "argv": null, + "destructive": false, + "confirm": false + } + }, + { + "id": "reclaimable_build_artifacts:docker", + "kind": "reclaimable_build_artifacts", + "severity": "info", + "subject": "docker", + "evidence": "2.0 GB of unused build cache and 3 dangling image(s)", + "size_bytes": 2000000000, + "remedy": { + "argv": [ + "teploy", + "server", + "prune", + "--host", + "edge", + "--build-cache", + "--dangling" + ], + "destructive": false, + "confirm": false + } + }, + { + "id": "versions_beyond_policy:many", + "kind": "versions_beyond_policy", + "severity": "info", + "subject": "many", + "app": "many", + "evidence": "1 version(s) beyond keep_versions 2: v2; the next deploy removes them", + "size_bytes": 10000000, + "remedy": { + "argv": [ + "teploy", + "releases", + "prune", + "--app", + "many", + "--host", + "edge", + "--keep", + "2" + ], + "destructive": true, + "confirm": true + } + } + ] +} diff --git a/contracts/fixtures/version-handshake/valid/mi2.json b/contracts/fixtures/version-handshake/valid/mi2.json index e08815f..fe7538b 100644 --- a/contracts/fixtures/version-handshake/valid/mi2.json +++ b/contracts/fixtures/version-handshake/valid/mi2.json @@ -32,6 +32,7 @@ "repair-debt", "rollback-json-v1", "secret-stdin-v1", + "server-audit-v1", "server-rename", "server-status-machine", "server-update", diff --git a/contracts/schema/server-audit-envelope.schema.json b/contracts/schema/server-audit-envelope.schema.json new file mode 100644 index 0000000..e2356a7 --- /dev/null +++ b/contracts/schema/server-audit-envelope.schema.json @@ -0,0 +1,83 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/server-audit-envelope.schema.json", + "title": "`teploy server audit --json` envelope: what is left over on a server, with evidence and the command that fixes it where one can (rev 19). The audit itself is read-only; a source that could not be read is an unknown check, never a clean bill.", + "type": "object", + "required": ["machine_interface", "server", "host", "audited_at", "state", "checks", "disk", "findings"], + "properties": { + "machine_interface": {"type": "integer", "minimum": 1}, + "server": {"type": "string", "minLength": 1}, + "host": {"type": "string"}, + "audited_at": {"type": "string", "format": "date-time"}, + "state": {"enum": ["ok", "partial", "unreachable"]}, + "error": {"type": "string"}, + "checks": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "status"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "status": {"enum": ["ok", "unknown"]}, + "detail": {"type": "string"} + } + } + }, + "disk": { + "oneOf": [ + {"type": "null"}, + { + "type": "object", + "required": ["path", "available_bytes", "used_percent", "attribution"], + "properties": { + "path": {"type": "string"}, + "available_bytes": {"type": "integer", "minimum": 0}, + "used_percent": {"type": "integer", "minimum": 0, "maximum": 100}, + "attribution": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "size_bytes"], + "properties": { + "name": {"type": "string"}, + "size_bytes": {"type": "integer", "minimum": 0} + } + } + } + } + } + ] + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "kind", "severity", "subject", "evidence", "remedy"], + "properties": { + "id": {"type": "string", "minLength": 1, "description": "kind:subject — stable across runs so a finding can be dismissed"}, + "kind": {"enum": ["stale_route", "empty_managed_region", "managed_route_dead_upstream", "orphan_state", "manual_route_dead_upstream", "manual_route_nonlocal_address", "versions_beyond_policy", "reclaimable_build_artifacts", "large_writable_layer", "unmanaged_container", "dangling_volumes", "caddyfile_without_caddy", "scheduler_missing", "disk_pressure"]}, + "severity": {"enum": ["critical", "warning", "info"]}, + "subject": {"type": "string", "minLength": 1}, + "app": {"type": "string", "description": "set when the finding belongs to one app: the name a destructive confirmation asks for"}, + "evidence": {"type": "string", "minLength": 1}, + "size_bytes": {"type": "integer", "minimum": 0}, + "related": {"type": "array", "items": {"type": "string"}, "description": "finding ids whose remedies would free space on a disk_pressure"}, + "remedy": { + "type": "object", + "required": ["destructive", "confirm"], + "properties": { + "argv": { + "oneOf": [ + {"type": "null", "description": "nothing the CLI can do safely: report only"}, + {"type": "array", "items": {"type": "string"}, "minItems": 1, "description": "the command that fixes it; argv[0] is teploy"} + ] + }, + "destructive": {"type": "boolean"}, + "confirm": {"type": "boolean", "description": "ask a person before running (a destructive remedy always confirms)"} + } + } + } + } + } + } +} diff --git a/internal/cli/contracts_golden_test.go b/internal/cli/contracts_golden_test.go index a6f8cb2..65b7a65 100644 --- a/internal/cli/contracts_golden_test.go +++ b/internal/cli/contracts_golden_test.go @@ -614,6 +614,16 @@ func TestContractsGitSourceGolden(t *testing.T) { // TestContractsBackupFleetGolden pins the fleet engine's envelopes (rev 18, // backup-fleet-v1). Dash's backups page builds on backup-status-envelope. +// TestContractsServerAuditGolden pins `teploy server audit --json` +// (rev 19, server-audit-v1): one server carrying one of each finding class — +// including report-only findings whose remedy argv is null — plus one unknown +// check (an app whose state could not be read), so a consumer can never +// mistake unreadable for clean. Generated from the real `auditServer` over a +// mock executor. +func TestContractsServerAuditGolden(t *testing.T) { + writeFixture(t, "server-audit-envelope/valid/findings.json", auditKitchenSink().audit(t)) +} + func TestContractsBackupFleetGolden(t *testing.T) { at := time.Date(2026, 11, 4, 3, 17, 4, 0, time.UTC) failedAt := at.Add(-time.Hour) diff --git a/internal/cli/machineinterface.go b/internal/cli/machineinterface.go index f2f0ff2..3aa1576 100644 --- a/internal/cli/machineinterface.go +++ b/internal/cli/machineinterface.go @@ -193,6 +193,12 @@ const ( // failures and the route per (server, app, host); read-only, runs from the // caller over SSH with nothing new on the server. CapDomainsCheckV1 = "domains-check-v1" + // CapServerAuditV1: `teploy server audit --json` (read-only + // findings with evidence and a remedy argv or null) and its remedies + // `route remove --app X`, `releases prune --keep N` and + // `server prune --build-cache --dangling`. keep_versions unset now means 5 + // after a notice period. server-audit-envelope pins the JSON. + CapServerAuditV1 = "server-audit-v1" ) // MachineCapabilities returns every capability token this build @@ -237,7 +243,8 @@ func MachineCapabilities() []string { CapRollbackJsonV1, CapRedeployV1, CapEnvConfigGuardV1, - CapDomainsCheckV1} + CapDomainsCheckV1, + CapServerAuditV1} sort.Strings(tokens) return tokens } diff --git a/internal/cli/machineinterface_test.go b/internal/cli/machineinterface_test.go index 935c8cb..3bba1f8 100644 --- a/internal/cli/machineinterface_test.go +++ b/internal/cli/machineinterface_test.go @@ -106,7 +106,7 @@ func TestCapabilityTokenRegistry(t *testing.T) { "trigger-convergence-v1", } want = append(want, "actor-flag-v1", "events-v1", "config-home-v1", "undeployed-list-v1", "lock-wait-v1", "config-merge-v1", "config-unprivileged-v1", "deploy-home-v1", "backup-target-v1", "backup-json-v1", "backup-fleet-v1") - want = append(want, "secret-stdin-v1", "releases-json-v1", "rollback-json-v1", "redeploy-v1", "env-config-guard-v1", "git-source-v1", "trigger-poll-v1") + want = append(want, "secret-stdin-v1", "releases-json-v1", "rollback-json-v1", "redeploy-v1", "env-config-guard-v1", "git-source-v1", "trigger-poll-v1", "server-audit-v1") sort.Strings(want) got := MachineCapabilities() if len(got) != len(want) { From f6e12dea6626bbce532f91863a482506fd56ebac Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:38:44 -0700 Subject: [PATCH 6/6] docs: fleet cleanup/audit (Slice 6), v0.1.53 unreleased, audit opens README: a Fleet cleanup & audit section (server audit, route remove, releases prune, server prune, the finding/remedy table, the never-reported-clean rule) and the keep_versions config comment now states the D3 semantics. CHANGELOG unreleased entry. AUDIT_OPEN records what the audit deliberately leaves report-only (hand-written blocks, writable layers, dangling volumes, unmanaged containers, live/webhook/ file-serving routes, foreign upstreams), the deferred surfaces (backup output sizes stay in the backup engine, attribution beyond docker + /deployments), and the decisions worth reviewing. --- AUDIT_OPEN.md | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++ CHANGELOG.md | 40 ++++++++++++++++++++++++++++++++ README.md | 39 ++++++++++++++++++++++++++++++- 3 files changed, 141 insertions(+), 1 deletion(-) diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 15dffac..a275b7d 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -1,5 +1,68 @@ # Open audit items +## 2026-11-05 — Dash auto-config Slice 6 CLI: cleanup / audit (P5) + +Plan: `../_internal/DASH_AUTOCONFIG_PLAN_2026-11-03.md` (section 9). Branch +`slice6-cleanup`. `teploy server audit` (read-only findings + remedy argv), +`route remove` (exact-block CAS), `releases prune --keep N`, +`server prune --build-cache --dangling`, keep_versions D3 (unset → 5 after a +recorded 7-day notice, `init` writes 3, 0 keeps everything), contracts rev 19. + +Deliberately report-only (design principle 4: hand-written config and other +people's containers are never edited by teploy): +- Hand-written Caddy blocks: dead upstream, non-local address. Evidence only. +- Large container writable layers (managed ≥ 5 GB) and large unmanaged + containers (≥ 1 GB): a layer's data belongs to whoever wrote it; the + evidence names the move-to-volume fix. +- Dangling volumes: may hold the only copy; there is never a remedy argv. +- Unmanaged containers (portainer et al.): not teploy's to remove. +- A managed block that still serves files (a `root` that exists), carries the + autodeploy webhook, or whose app is live: `route remove` refuses; the audit + reports the finding with the refusal as its evidence and a null remedy. +- An upstream that is not this app's (a `caddy_extra` proxy to another app's + dead container): this app's redeploy cannot fix it, so no redeploy remedy. + +Deferred surface: +- **Backup output sizes in the audit** (plan lesson: "backup jobs whose + outputs are implausibly small"): covered at write time by the engine's + minimum-size refusal (Slice 2, `internal/backup/engine.go`) and + `backup status`'s per-target last success/failure; the audit does not + re-walk buckets, which would need target credentials it must not use. +- **`server audit` on a server with no caddyfile at all** reports no route + findings by construction; a Caddyfile with no Caddy container is flagged + (`caddyfile_without_caddy`, compute-2's case). +- **Disk attribution beyond docker + /deployments**: `du` of other trees + (logs, /var) is not attempted; the pressure finding attributes what teploy + can name and links the reclaimable findings. +- **Audit does not pin the notice file's mtime against clock skew**: the + notice is a RFC 3339 server-side timestamp written by the deploy that + starts it, same trust domain as state.json. + +Decisions worth reviewing: +- `route remove` protects running-container versions of an app's name even + without state (`ownedContainers`, unlabelled `-*` names included): + an older deploy's container keeps the name alive, so the route is not + proven dead until it is gone. +- `releases prune --keep 0` (and the audit never emits it) prunes everything + the protections do not name — deliberately the opposite reading of + `keep_versions: 0`, documented in the command's help, because "retain none + beyond the protected set" is a coherent manual operation. +- Automatic pruning under the default keep_versions runs only inside a + deploy of that app (never from the audit, never fleet-wide), so the notice + is always written by an operation that already holds the app lock. +- `retentionProtected` includes any version with a running container, beyond + the plan's pins/current/previous: `docker rm -f` of a running stranger is + not something a cleanup command should do even inside the keep window. + +Verified: unit/hermetic only (mock SSH executors, the same discipline as +`domains check`); every audit finding kind, all `route remove` refusal +paths, the CAS refusal on a concurrent rewrite, prune selections (pins/ +current/previous/running survive, `--keep 0`, unreadable pins refuse), +the notice-period state machine, and the read-only pin (no mutating verb, +docker subcommand or upload in any command `server audit` issues). Not yet +exercised live: the four fleet servers' first real audit run and the Dash +cleanup page (Dash side, next slice). + ## 2026-11-05 — Dash auto-config Slice 4 CLI: fleet backup engine (G1, G3, G4, G5) Plan: `../_internal/DASH_AUTOCONFIG_PLAN_2026-11-03.md` (sections 2.1-2.3, 5). Branch diff --git a/CHANGELOG.md b/CHANGELOG.md index a7205f0..f335b8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,46 @@ All notable changes to teploy are documented here. Format follows [Keep a Change ## [Unreleased] +### Added + +- `teploy server audit --json`: what is left over on a server, each + finding with its evidence and — only where a CLI command fixes it safely — + the remedy argv. Stale and empty managed routes, managed routes with dead + upstreams, app state with no containers, versions beyond `keep_versions`, + reclaimable build cache and dangling images, large container writable + layers, large containers teploy does not manage, dangling volumes, a + Caddyfile with no Caddy container, scheduled jobs with no cron daemon, and + disk pressure with attribution. Read-only: every command the audit runs on + the server only reads (a test pins that). A source that could not be read + is an `unknown` check and yields no finding — never a clean bill. +- `teploy route remove --app X`: removes a managed Caddy block proven dead + (no app state, no containers carrying the name, no upstream that still + resolves, no files it still serves, nothing unreadable), under the app's + lock and an exact-block compare-and-swap, so a concurrent rewrite is + refused rather than undone. Recorded as a `route.remove` event. +- `teploy releases prune --app X --keep N`: removes old versions' containers + and images beyond `--keep`. Pinned, current, previous and running versions + are never removed whatever `--keep` says; an unreadable pin set, state or + container list refuses the prune. Recorded as a `releases.prune` event. +- `teploy server prune --host S --build-cache --dangling`: reclaims docker's + unused build cache and dangling images — nothing else (no tagged image, no + container, no volume). Recorded as a `server.prune` event under `_fleet`. +- Contracts: `server-audit-envelope` (schema + generated fixture, corpus rev + 19); version-handshake advertises `server-audit-v1`. + +### Changed + +- `keep_versions` is now optional (decision D3): unset means 5, pruned by + deploys only after a 7-day notice recorded on the server (the deploy + output and `server audit` warn first); `teploy init` writes 3; `0` keeps + everything. Applied manifests record unset and N as before, so existing + digests hold; an explicit 0 records `keep_all_versions`. +- `teploy remove` on an app whose directory is already gone (an earlier + removal finished) succeeds instead of failing its state deletion. +- `teploy doctor` disk remediations point at `teploy server audit`, which + attributes the space; one named a `releases prune` command that did not + exist. + ## [0.1.52] - 2026-11-05 ### Added diff --git a/README.md b/README.md index 905173c..4ce7a77 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,9 @@ drain_seconds: 10 # request drain: keep the old container serving in # the drain policy — size it to your longest normal request. memory: 2g # cgroup RAM cap (docker units: 512m, 2g). Unset = unlimited. cpu: "1.5" # CPU cap in cores. Unset = unlimited. -keep_versions: 3 # auto-prune older versions after deploy (0 = keep all, default) +keep_versions: 3 # versions retained after a deploy. Unset = 5, pruned only after a + # 7-day notice (teploy init writes 3); 0 keeps everything. + # The current, previous and pinned versions are never pruned. backup_before_hooks: true # back up running databases before a deploy whose pre_deploy hook will run (default off) # `teploy pin ` protects a version from this prune @@ -354,6 +356,7 @@ teploy plan # preview what a deploy would change ( teploy plan --out plan.json # write a bound plan record (config digest + target identity + generation) teploy apply plan.json # execute a reviewed plan; refuses naming what drifted since it was made teploy scale # multi-server deploy + LB update +teploy server audit # read-only: what is left over and what fixes it teploy version / update # version info and self-update ``` @@ -595,6 +598,40 @@ serves. Read-only, run from your machine over SSH, nothing new on the server: `ingress: external` and an unfronted `ingress: host` domain are labels: they carry a `skip` reason. A server that cannot be reached is reported as such. +### Fleet cleanup & audit + +```bash +teploy server audit # what is left over, with evidence and the fix +teploy server audit --json # contracts/schema/server-audit-envelope.schema.json +teploy route remove --app X --host S # remove a managed Caddy block proven dead (CAS) +teploy releases prune --app X --host S --keep N # remove versions beyond N (pins/current/previous survive) +teploy server prune --host S --build-cache --dangling # reclaim unused build cache and dangling images +``` + +`server audit` changes nothing: every command it runs on the server only +reads (a test pins that). Each finding says what it is, the evidence for it, +and — only where a CLI command can fix it safely — the remedy: + +| Finding | Remedy | +|---|---| +| stale/empty managed route | `teploy route remove --app X` | +| managed route, dead upstream | `teploy redeploy` (confirm) | +| app state, no containers | `teploy remove --app X -y` (keeps volumes; confirm) | +| versions beyond `keep_versions` | `teploy releases prune --keep N` (confirm) | +| build cache / dangling images | `teploy server prune --build-cache --dangling` | +| hand-written blocks, big writable layers, dangling volumes, unmanaged containers, disk pressure, a Caddyfile with no Caddy, scheduled jobs with no cron daemon | report only | + +A source that could not be read shows as `unknown` in `checks[]` and the +findings depending on it are missing — a server is never reported clean +because it could not be inspected. With `--json` the exit code is 0 whenever +the envelope was produced; read `state` (`ok` / `partial` / `unreachable`). + +`route remove` only takes a block proven dead — no app state, no containers +carrying the name, no upstream that still resolves (asked inside Caddy's +network), no files it still serves, nothing unreadable — and commits under +an exact-block compare-and-swap, so a deploy that rewrites the block in +between is never undone. + ### Accessories ``` teploy accessory list # list running accessories