From 779b6f498f57f3fee430bcaa4b486ec5bf036b94 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Wed, 4 Nov 2026 22:34:35 -0700 Subject: [PATCH 01/17] feat(backup): verified object store, age encryption, per-database series, GFS, prefix safety Every artifact and sidecar now goes through one store path (G1, G3, G5): - put: age-encrypt to the owner key and the server identity, prove the ciphertext decrypts, upload to .incoming/, check size, recorded SHA-256 and (to 256 MiB) the bytes, then copy server-side to the final key. A final key only ever holds a verified object; sidecars precede the primary that commits a backup; a failure to encrypt never uploads plaintext. - fetch: checks the recorded checksum and free space, decrypts on the server with its own identity. Legacy objects keep restoring. - One object per database for postgres and mysql/mariadb, in a series of its own; hostile names are shell-quoted and key-encoded, and a name the tools cannot address is reported, never skipped silently. - Adopted containers: the _adopted namespace, addressed by docker name. - Retention: restic forget semantics (keep_last, daily, weekly, monthly, and max_age_days as keep-within). The newest good backup is never pruned, below-floor objects never count, every sidecar goes with its backup, refused deletes do not stop the rest. - Prefixes: listing no longer breaks under a prefix; previous prefixes are searched and only pruned when owned; .owners markers refuse a prune in a namespace another server wrote to. - Backups build in /var/tmp with a space preflight that measures the backup and its ciphertext, and have deadlines. - DR bundles: members encrypted, the age key wrapped to the owner only. Co-Authored-By: Claude Opus 5.5 --- .../backup-list-envelope/valid/app.json | 9 +- .../schema/backup-list-envelope.schema.json | 151 +++- go.mod | 7 +- go.sum | 4 + internal/backup/accessory.go | 563 ++++++++++--- internal/backup/backup.go | 189 ++++- internal/backup/backup_test.go | 63 +- internal/backup/bundle.go | 61 +- internal/backup/bundle_campaign_test.go | 2 +- internal/backup/bundle_confighome_test.go | 2 +- internal/backup/bundle_restore.go | 21 +- internal/backup/bundle_restore_test.go | 6 +- internal/backup/bundle_secretmeta_test.go | 4 +- internal/backup/bundle_test.go | 6 +- internal/backup/cron.go | 12 +- internal/backup/engine.go | 22 +- internal/backup/engine_test.go | 12 +- internal/backup/list.go | 99 ++- internal/backup/postgres.go | 43 +- internal/backup/postgres_restore_test.go | 54 +- internal/backup/restore.go | 123 ++- internal/backup/restore_staging_test.go | 21 +- internal/backup/retention.go | 423 ++++++++-- internal/backup/retention_test.go | 14 +- internal/backup/scheduler_test.go | 2 +- internal/backup/store.go | 726 +++++++++++++++++ internal/backup/store_test.go | 754 ++++++++++++++++++ internal/backup/store_test_helpers_test.go | 23 + internal/backup/target.go | 17 +- internal/backup/verify.go | 213 +++-- internal/backup/verify_test.go | 13 +- internal/ssh/result.go | 7 + 32 files changed, 3127 insertions(+), 539 deletions(-) create mode 100644 internal/backup/store.go create mode 100644 internal/backup/store_test.go create mode 100644 internal/backup/store_test_helpers_test.go diff --git a/contracts/fixtures/backup-list-envelope/valid/app.json b/contracts/fixtures/backup-list-envelope/valid/app.json index 7009663..445fcd8 100644 --- a/contracts/fixtures/backup-list-envelope/valid/app.json +++ b/contracts/fixtures/backup-list-envelope/valid/app.json @@ -14,7 +14,8 @@ "size_bytes": 1048576, "created_at": "2026-10-20T03:17:04Z", "key": "s3://teploy-backups/blog/volumes/20261020-031704-1f2e3d4c5b6a7988.tar.gz", - "restorable": true + "restorable": true, + "encrypted": false }, { "id": "20261020-031704-1f2e3d4c5b6a7988", @@ -22,7 +23,8 @@ "size_bytes": 4096, "created_at": "2026-10-20T03:17:04Z", "key": "s3://teploy-backups/blog/teploy/20261020-031704-1f2e3d4c5b6a7988.tar.gz", - "restorable": true + "restorable": true, + "encrypted": false }, { "id": "20261019-030001", @@ -31,7 +33,8 @@ "size_bytes": 2097152, "created_at": "2026-10-19T03:00:01Z", "key": "s3://teploy-backups/blog/accessories/db/20261019-030001.sql.gz", - "restorable": true + "restorable": true, + "encrypted": false } ], "schedules": [ diff --git a/contracts/schema/backup-list-envelope.schema.json b/contracts/schema/backup-list-envelope.schema.json index 9ea3813..e6efc29 100644 --- a/contracts/schema/backup-list-envelope.schema.json +++ b/contracts/schema/backup-list-envelope.schema.json @@ -3,36 +3,110 @@ "$id": "https://teploy.github.io/contracts/schema/backup-list-envelope.schema.json", "title": "`teploy backup list --json`: parsed backups across the volumes, accessories and teploy prefixes plus schedules discovered from teploy cron markers (backup-json-v1, rev 14).", "type": "object", - "required": ["machine_interface", "host", "app", "backups", "schedules"], + "required": [ + "machine_interface", + "host", + "app", + "backups", + "schedules" + ], "properties": { - "machine_interface": {"type": "integer", "minimum": 1}, - "host": {"type": "string", "minLength": 1}, - "app": {"type": "string", "minLength": 1}, + "machine_interface": { + "type": "integer", + "minimum": 1 + }, + "host": { + "type": "string", + "minLength": 1 + }, + "app": { + "type": "string", + "minLength": 1 + }, "target": { "type": "object", "description": "The stored server target in force (absent when the legacy flags overrode it).", - "required": ["bucket", "region", "has_credentials"], + "required": [ + "bucket", + "region", + "has_credentials" + ], "properties": { - "bucket": {"type": "string", "minLength": 1}, - "region": {"type": "string", "minLength": 1}, - "endpoint": {"type": "string"}, - "prefix": {"type": "string"}, - "has_credentials": {"type": "boolean"} + "bucket": { + "type": "string", + "minLength": 1 + }, + "region": { + "type": "string", + "minLength": 1 + }, + "endpoint": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "has_credentials": { + "type": "boolean" + } } }, "backups": { "type": "array", "items": { "type": "object", - "required": ["id", "kind", "size_bytes", "created_at", "key", "restorable"], + "required": [ + "id", + "kind", + "size_bytes", + "created_at", + "key", + "restorable" + ], "properties": { - "id": {"type": "string", "description": "Backup identity: 20060102-150405, optionally suffixed -0123456789abcdef."}, - "kind": {"enum": ["volumes", "accessory", "teploy"]}, - "accessory": {"type": "string", "description": "Accessory name; present only when kind is accessory."}, - "size_bytes": {"type": "integer", "minimum": 0}, - "created_at": {"type": "string", "description": "RFC3339 UTC derived from the backup id's timestamp."}, - "key": {"type": "string", "pattern": "^s3://"}, - "restorable": {"type": "boolean", "description": "Whether the id satisfies the restore grammar (teploy rows are listed but not restorable through `backup restore`)."} + "id": { + "type": "string", + "description": "Backup identity: 20060102-150405, optionally suffixed -0123456789abcdef." + }, + "kind": { + "enum": [ + "volumes", + "accessory", + "teploy" + ] + }, + "accessory": { + "type": "string", + "description": "Accessory name; present only when kind is accessory." + }, + "size_bytes": { + "type": "integer", + "minimum": 0 + }, + "created_at": { + "type": "string", + "description": "RFC3339 UTC derived from the backup id's timestamp." + }, + "key": { + "type": "string", + "pattern": "^s3://" + }, + "restorable": { + "type": "boolean", + "description": "Whether the id satisfies the restore grammar (teploy rows are listed but not restorable through `backup restore`)." + }, + "database": { + "type": "string", + "description": "Present on one database's own object (rev 17)." + }, + "encrypted": { + "type": "boolean", + "description": "age-encrypted (.age, rev 17)." + }, + "previous_prefix": { + "type": "boolean", + "description": "Found under an old prefix of the bucket (rev 17)." + } } } }, @@ -40,14 +114,41 @@ "type": "array", "items": { "type": "object", - "required": ["kind", "cron", "form"], + "required": [ + "kind", + "cron", + "form" + ], "properties": { - "kind": {"enum": ["volumes", "accessory"]}, - "accessory": {"type": "string"}, - "cron": {"type": "string", "description": "The five cron schedule fields."}, - "keep_last": {"type": "integer", "minimum": 0}, - "max_age_days": {"type": "integer", "minimum": 0}, - "form": {"enum": ["run", "legacy", "custom"], "description": "run: `backup run` form; legacy: pre-D3 teploy-generated (bucket/credentials embedded); custom: hand-written command under a teploy marker, left alone by rewrites."} + "kind": { + "enum": [ + "volumes", + "accessory" + ] + }, + "accessory": { + "type": "string" + }, + "cron": { + "type": "string", + "description": "The five cron schedule fields." + }, + "keep_last": { + "type": "integer", + "minimum": 0 + }, + "max_age_days": { + "type": "integer", + "minimum": 0 + }, + "form": { + "enum": [ + "run", + "legacy", + "custom" + ], + "description": "run: `backup run` form; legacy: pre-D3 teploy-generated (bucket/credentials embedded); custom: hand-written command under a teploy marker, left alone by rewrites." + } } } } diff --git a/go.mod b/go.mod index 8530dad..74d12eb 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,14 @@ module github.com/useteploy/teploy go 1.26.0 require ( + filippo.io/age v1.2.1 github.com/BurntSushi/toml v1.6.0 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.9 golang.org/x/crypto v0.57.0 + golang.org/x/sys v0.48.0 golang.org/x/term v0.46.0 gopkg.in/yaml.v3 v3.0.1 ) -require ( - github.com/inconshreveable/mousetrap v1.1.0 // indirect - golang.org/x/sys v0.48.0 // indirect -) +require github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index 6aaf702..d9b8ff7 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805 h1:u2qwJeEvnypw+OCPUHmoZE3IqwfuN5kgDfo5MLzpNM0= +c2sp.org/CCTV/age v0.0.0-20240306222714-3ec4d716e805/go.mod h1:FomMrUJ2Lxt5jCLmZkG3FHa72zUprnhd3v/Z18Snm4w= +filippo.io/age v1.2.1 h1:X0TZjehAZylOIj4DubWYU1vWQxv9bJpo+Uu2/LGhi1o= +filippo.io/age v1.2.1/go.mod h1:JL9ew2lTN+Pyft4RiNGguFfOpewKwSHm5ayKD/A4004= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= diff --git a/internal/backup/accessory.go b/internal/backup/accessory.go index b772935..9a39152 100644 --- a/internal/backup/accessory.go +++ b/internal/backup/accessory.go @@ -4,8 +4,11 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" + "os" "path" + "sort" "strconv" "strings" "time" @@ -13,14 +16,91 @@ import ( "github.com/useteploy/teploy/internal/ssh" ) +// BackupOptions tune an accessory backup. +type BackupOptions struct { + // AllDatabases backs up every database on the server, each as its own + // object: what a policy-managed run does, so a database created tomorrow is + // covered with no action. + AllDatabases bool + // Databases names exactly the extra databases to back up (an adopted + // entry's databases: [...]). A listed database that is missing is a failure. + Databases []string + // SingleDatabase backs up only this database (the safety backup before a + // `restore --database`). + SingleDatabase string + // IgnoreSpace skips the free-space preflight. + IgnoreSpace bool +} + +// PrimaryDatabase names the database a default restore of this accessory would +// replace ("" for an engine without named databases). +func (c *Client) PrimaryDatabase(ctx context.Context, app, name, image string, env map[string]string) string { + det, err := c.ResolveEngine(ctx, app, name, image, env) + if err != nil { + return "" + } + switch det.Engine { + case EnginePostgres: + db, _ := postgresDBAndUser(app, env) + return db + case EngineMySQL, EngineMariaDB: + return mysqlDB(app, env) + } + return "" +} + +// BackupDeadline bounds one backup of one target, so a wedged dump or upload +// cannot hold a run (and its lock) forever. +const BackupDeadline = 6 * time.Hour + +func (c *Client) backupDeadline() time.Duration { + if c.deadline > 0 { + return c.deadline + } + return BackupDeadline +} + +// backupTmpDir is where a backup builds its artifacts: disk, not the tmpfs /tmp +// is on many hosts, which a database dump would fill along with the RAM. +func backupTmpDir() string { + if d := strings.TrimSpace(os.Getenv("TEPLOY_BACKUP_TMPDIR")); d != "" { + return strings.TrimRight(d, "/") + } + return "/var/tmp" +} + +func (c *Client) newBackupWorkspace(ctx context.Context) (string, func(), error) { + runOut, err := c.exec.Run(ctx, "umask 077; mktemp -d "+ssh.ShellQuote(backupTmpDir()+"/teploy-backup.XXXXXXXX")) + if err != nil { + return "", nil, fmt.Errorf("creating backup workspace: %w", err) + } + workDir := strings.TrimSpace(runOut) + return workDir, func() { + rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) + defer cancel() + c.exec.Run(rctx, "rm -rf "+ssh.ShellQuote(workDir)) + }, nil +} + // AccessoryBackup performs a database-aware backup for an accessory. +func (c *Client) AccessoryBackup(ctx context.Context, app, name, image string, env map[string]string, s3 S3Config) (BackupResult, error) { + return c.AccessoryBackupWith(ctx, app, name, image, env, s3, BackupOptions{}) +} + +// AccessoryBackupWith is AccessoryBackup with options. // // All intermediate artifacts (dumps, snapshots, scratch tars) are built // inside one private 0700 mktemp workspace and removed on every exit — the // old predictable /tmp/-… paths were shared across accessories of the // same app and exposed database dumps to other local users (audit F34). -func (c *Client) AccessoryBackup(ctx context.Context, app, name, image string, env map[string]string, s3 S3Config) (res BackupResult, err error) { +// +// Objects are stored sidecars first, primary last (see store.go), and one +// database failing does not stop the others: the result names it and the +// returned error makes the backup a failure, so nothing is pruned. +func (c *Client) AccessoryBackupWith(ctx context.Context, app, name, image string, env map[string]string, s3 S3Config, opts BackupOptions) (res BackupResult, err error) { start := time.Now() + ctx, cancel := context.WithTimeout(ctx, c.backupDeadline()) + defer cancel() defer func() { res.DurationMs = time.Since(start).Milliseconds() }() if err := c.ensureAWSCLI(ctx); err != nil { return res, err @@ -30,87 +110,294 @@ func (c *Client) AccessoryBackup(ctx context.Context, app, name, image string, e return res, err } res.Engine = det.Engine + res.Encrypted = s3.Encrypted() + if err := c.prepareEncryption(ctx, s3); err != nil { + return res, err + } + adopted := app == AdoptedApp + if adopted && !det.IsDB() || adopted && det.Engine == EngineNucleus { + return res, fmt.Errorf("adopted container %s: engine %q has no adopted backup method (adopted containers need a database engine; set engine: in the policy entry if detection is wrong)", name, det.Engine) + } timestamp, err := newBackupID(time.Now()) if err != nil { return res, err } res.ID = timestamp - containerName := app + "-" + name - - runOut, err := c.exec.Run(ctx, "umask 077; mktemp -d /tmp/teploy-backup.XXXXXXXX") - if err != nil { - return res, fmt.Errorf("creating backup workspace: %w", err) - } - workDir := strings.TrimSpace(runOut) - cleanup := func() { - c.exec.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(workDir)) - } + containerName := ContainerName(app, name) - // Consistency classification + the dump command come from the shared - // planner (also used by the DR bundle path) so the two snapshot paths - // can never drift apart. - plan, err := planAccessoryDump(app, name, det, env, workDir) + workDir, cleanup, err := c.newBackupWorkspace(ctx) if err != nil { - cleanup() return res, err } - res.Consistency = plan.consistency - s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s%s", s3.Base(), app, name, timestamp, plan.ext) - res.Key = s3Key - if err := plan.stage(ctx, c.exec); err != nil { - cleanup() - return res, fmt.Errorf("staging the mysql credential file for %s (no dump was run): %w", name, err) - } - // A server with more databases than this backup covers is still backed - // up: a partial backup and a loud warning beat none. The coverage reaches - // the backup.run event, doctor and `backup run`'s alert script; Dash - // alerting on it arrives in Slice 4. + defer cleanup() + + // What this backup covers. A server with more databases than it covers is + // still backed up: a partial backup and a loud warning beat none. var pgT pgTarget - var coverage pgCoverage if det.Engine == EnginePostgres { db, user := postgresDBAndUser(app, env) + if adopted { + db = "" + } pgT = newPGTarget(containerName, db, user, env) } - coverage, res.UncoveredDatabases, res.Warnings = c.backupCoverage(ctx, app, name, det, pgT, env) + coverage, cerr := c.coverageWith(ctx, app, name, det, pgT, env, opts) + res.UncoveredDatabases = coverage.Uncovered + res.Warnings = coverage.warningsFor(app, name, det, env) + multi := opts.AllDatabases || len(opts.Databases) > 0 || opts.SingleDatabase != "" || adopted + if cerr != nil { + if multi && (det.Engine == EnginePostgres || det.Engine == EngineMySQL || det.Engine == EngineMariaDB) { + return res, fmt.Errorf("listing the databases of %s, so what to back up is unknown: %w", containerName, cerr) + } + res.Warnings = append(res.Warnings, cerr.Error()) + } for _, w := range res.Warnings { fmt.Fprintf(c.out, "Warning: %s\n", w) } fmt.Fprintf(c.out, "Backing up %s (%s, detected from %s)...\n", containerName, det.Engine, det.Source) + + failed := map[string]string{} + for _, missing := range coverage.Missing { + failed[missing] = "listed for backup but not on the server" + } + + // Sidecars and extra databases first: the primary object commits the backup. + if det.Engine == EnginePostgres { + if w := c.backupPostgresGlobals(ctx, pgT, timestamp, workDir, s3, app, name); w != "" { + res.Warnings = append(res.Warnings, w) + fmt.Fprintf(c.out, "Warning: %s\n", w) + } + } + for i, db := range coverage.Extra { + obj, derr := c.backupOneDatabase(ctx, app, name, det, env, db, workDir, i, timestamp, s3, opts) + if derr != nil { + failed[db] = derr.Error() + fmt.Fprintf(c.out, "Backing up database %s FAILED: %v\n", db, derr) + continue + } + res.Objects = append(res.Objects, obj) + res.SizeBytes += obj.Size + fmt.Fprintf(c.out, "Backed up database %s\n", db) + } + + primary := true + switch { + case opts.SingleDatabase != "" && (!coverage.PrimaryIsSingle || coverage.PrimaryMissing): + primary = false + case (det.Engine == EnginePostgres || det.Engine == EngineMySQL || det.Engine == EngineMariaDB) && (adopted || (multi && coverage.PrimaryMissing)): + // Nothing named this container's primary database: every database has + // its own series. Pruning sees that as one series per database. + primary = false + } + if primary { + obj, consistency, perr := c.backupPrimary(ctx, app, name, det, env, pgT, workDir, timestamp, s3, opts) + res.Consistency = consistency + if perr != nil { + return res, joinDatabaseFailures(failed, perr) + } + res.Key = obj.Key + res.SizeBytes += obj.Size + res.Objects = append(res.Objects, obj) + } else if plan, perr := planAccessoryDump(app, name, det, env, workDir); perr == nil { + res.Consistency = plan.consistency + } + if res.Key == "" && len(res.Objects) > 0 { + res.Key = res.Objects[0].Key + } + if len(res.Objects) == 0 && len(failed) == 0 { + // An empty server (no schema yet) has nothing to store. Not a failure, + // but never a silent success either. + res.Warnings = append(res.Warnings, containerName+" holds no database yet, so nothing was stored") + return res, nil + } + res.FailedDatabases = failed + c.markNamespaceOwner(ctx, s3, app) + if len(failed) > 0 { + return res, joinDatabaseFailures(failed, nil) + } + fmt.Fprintf(c.out, "Backup complete: %s\n", res.Key) + return res, nil +} + +// joinDatabaseFailures renders per-database failures into one error. +func joinDatabaseFailures(failed map[string]string, primary error) error { + if len(failed) == 0 { + return primary + } + names := make([]string, 0, len(failed)) + for db := range failed { + names = append(names, db) + } + sort.Strings(names) + parts := make([]string, 0, len(names)) + for _, db := range names { + parts = append(parts, db+": "+failed[db]) + } + msg := "database(s) not backed up: " + strings.Join(parts, "; ") + if primary != nil { + return fmt.Errorf("%w; %s", primary, msg) + } + return errors.New(msg) +} + +// backupPrimary dumps and stores the accessory's primary object: the backup's +// commit point. It is also the only object a single-database engine has. +func (c *Client) backupPrimary(ctx context.Context, app, name string, det EngineDetection, env map[string]string, pgT pgTarget, workDir, id string, s3 S3Config, opts BackupOptions) (BackupObject, string, error) { + var obj BackupObject + plan, err := planAccessoryDump(app, name, det, env, workDir) + if err != nil { + return obj, "", err + } + if err := c.preflightDumpSpace(ctx, app, name, det, env, pgT, workDir, opts); err != nil { + return obj, plan.consistency, err + } + if err := plan.stage(ctx, c.exec); err != nil { + return obj, plan.consistency, fmt.Errorf("staging the mysql credential file for %s (no dump was run): %w", name, err) + } if _, err := c.exec.Run(ctx, plan.cmd); err != nil { // Never leave partial backups around: they're disk-fillers at best, // restore-bait at worst. - cleanup() - return res, fmt.Errorf("dumping %s: %w", name, err) + return obj, plan.consistency, fmt.Errorf("dumping %s: %w", name, err) } size, err := c.checkArtifactSize(ctx, plan.artifactPath, plan.minBytes) - res.SizeBytes = size if err != nil { - cleanup() - return res, fmt.Errorf("dumping %s: %w", name, err) + return obj, plan.consistency, fmt.Errorf("dumping %s: %w", name, err) } - - fmt.Fprintf(c.out, "Uploading to %s...\n", s3Key) - if _, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(plan.artifactPath), ssh.ShellQuote(s3Key)))); err != nil { - cleanup() - return res, fmt.Errorf("uploading to S3: %w", err) + if det.Engine == EnginePostgres { + c.storeDatabaseMeta(ctx, pgT, pgT.db, s3, seriesPrefix(s3, app, name, "")+"meta/"+id+".json", workDir) + } + key := seriesPrefix(s3, app, name, "") + id + plan.ext + fmt.Fprintf(c.out, "Uploading to %s...\n", key) + stored, err := c.put(ctx, s3, plan.artifactPath, key) + if err != nil { + return obj, plan.consistency, fmt.Errorf("storing %s: %w", name, err) } + _ = size + return BackupObject{Key: stored.Key, Size: stored.Size, Encrypted: stored.Encrypted}, plan.consistency, nil +} +// backupOneDatabase dumps and stores one more database as its own object. +// Series objects sit below the accessory's backups, so primary listings see one +// backup per id; `backup restore --database` restores one. +func (c *Client) backupOneDatabase(ctx context.Context, app, name string, det EngineDetection, env map[string]string, db, workDir string, n int, id string, s3 S3Config, opts BackupOptions) (BackupObject, error) { + obj := BackupObject{Database: db} + sub := fmt.Sprintf("%s/db-%d", workDir, n) + if _, err := c.exec.Run(ctx, "mkdir -p "+ssh.ShellQuote(sub)); err != nil { + return obj, err + } + defer c.exec.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(sub)) + envDB := envForDatabase(env, det.Engine, db) + plan, err := planAccessoryDump(app, name, det, envDB, sub) + if err != nil { + return obj, err + } + var pgT pgTarget if det.Engine == EnginePostgres { - c.backupPostgresGlobals(ctx, app, name, timestamp, env, workDir, s3) - c.recordPostgresSize(ctx, pgT, s3, app, name, timestamp, workDir) - for _, extra := range coverage.Extra { - if err := c.backupPostgresExtra(ctx, pgT, extra, s3, app, name, timestamp, workDir); err != nil { - cleanup() - return res, fmt.Errorf("backing up database %s (backup_databases): %w", extra, err) - } - fmt.Fprintf(c.out, "Backed up database %s\n", extra) - } + _, user := postgresDBAndUser(app, env) + pgT = newPGTarget(ContainerName(app, name), db, user, env) } + if err := c.preflightDumpSpace(ctx, app, name, det, envDB, pgT, sub, opts); err != nil { + return obj, err + } + if err := plan.stage(ctx, c.exec); err != nil { + return obj, fmt.Errorf("staging the credential file (no dump was run): %w", err) + } + if _, err := c.exec.Run(ctx, plan.cmd); err != nil { + return obj, err + } + if _, err := c.checkArtifactSize(ctx, plan.artifactPath, plan.minBytes); err != nil { + return obj, err + } + series := seriesPrefix(s3, app, name, db) + if det.Engine == EnginePostgres { + c.storeDatabaseMeta(ctx, pgT, db, s3, series+"meta/"+id+".json", sub) + } + stored, err := c.put(ctx, s3, plan.artifactPath, series+id+plan.ext) + if err != nil { + return obj, err + } + obj.Key, obj.Size, obj.Encrypted = stored.Key, stored.Size, stored.Encrypted + return obj, nil +} - cleanup() - fmt.Fprintf(c.out, "Backup complete: %s\n", s3Key) - return res, nil +// envForDatabase is the accessory's env with its database variables naming db, +// so the shared planner dumps that database with the same flags and checks. +func envForDatabase(env map[string]string, engine, db string) map[string]string { + out := make(map[string]string, len(env)+2) + for k, v := range env { + out[k] = v + } + switch engine { + case EnginePostgres: + out["POSTGRES_DB"] = db + delete(out, "POSTGRESQL_DATABASE") + case EngineMySQL, EngineMariaDB: + out["MYSQL_DATABASE"] = db + delete(out, "MARIADB_DATABASE") + } + return out +} + +// dumpReserveBytes is free space a dump must leave: a database on the same +// filesystem as the workspace panics when the disk fills. +const dumpReserveBytes = 256 << 20 + +// preflightDumpSpace refuses a dump that would fill the workspace's disk: on a +// small host the same disk holds the live database. It measures what the backup +// will occupy, plaintext and ciphertext at once, not the size of the live data +// alone. A size it cannot measure is not a refusal. +func (c *Client) preflightDumpSpace(ctx context.Context, app, name string, det EngineDetection, env map[string]string, pgT pgTarget, workDir string, opts BackupOptions) error { + if opts.IgnoreSpace { + return nil + } + q := ssh.ShellQuote + container := ContainerName(app, name) + var live int64 + var known bool + readInt := func(out string, err error) (int64, bool) { + if err != nil { + return 0, false + } + f := strings.Fields(out) + if len(f) == 0 { + return 0, false + } + n, perr := strconv.ParseInt(f[0], 10, 64) + return n, perr == nil + } + var need int64 + switch det.Engine { + case EnginePostgres: + live, known = readInt(c.exec.Run(ctx, pgT.psql(pgT.db, "SELECT pg_database_size(current_database())"))) + // A compressed custom-format archive, plus its ciphertext. + need = live + case EngineMySQL, EngineMariaDB: + sql := "SELECT COALESCE(SUM(data_length + index_length), 0) FROM information_schema.TABLES WHERE table_schema = '" + strings.ReplaceAll(mysqlDB(app, env), "'", "''") + "'" + live, known = readInt(c.exec.Run(ctx, mysqlEnvClient(det.Engine, container, sql, "-N", "-B"))) + // The plain SQL and its gzip side by side, then the gzip and its ciphertext. + need = live + live/2 + case EngineMongo: + live, known = readInt(c.exec.Run(ctx, "docker exec "+q(container)+" du -sk /data/db")) + live *= 1024 + need = live + case EngineRedis, EngineValkey, EngineDragonfly: + live, known = readInt(c.exec.Run(ctx, "docker exec "+q(container)+" du -sk /data")) + live *= 1024 + need = live + default: + live, known = readInt(c.exec.Run(ctx, "du -sk "+q(fmt.Sprintf("%s/%s/accessories/%s", deploymentsDir, app, name)))) + live *= 1024 + need = live + } + if !known { + return nil + } + free, freeKnown := c.freeBytes(ctx, "", workDir) + if freeKnown && free < need+dumpReserveBytes { + return fmt.Errorf("not enough free space in %s for this dump: %d bytes free, about %d needed plus a %d byte reserve (live size %d); nothing was dumped. Set TEPLOY_BACKUP_TMPDIR to a larger disk or pass --ignore-space-check", workDir, free, need, int64(dumpReserveBytes), live) + } + return nil } // postgresGlobalsKey is where a postgres backup's cluster globals (roles, @@ -118,23 +405,22 @@ func (c *Client) AccessoryBackup(ctx context.Context, app, name, image string, e // sits one directory below the accessory's backups, so listings, latest // resolution and retention counting never see it as a backup of its own. func postgresGlobalsKey(s3 S3Config, app, name, id string) string { - return fmt.Sprintf("s3://%s/%s/accessories/%s/globals/%s.sql.gz", s3.Base(), app, name, id) + return seriesPrefix(s3, app, name, "") + "globals/" + id + ".sql.gz" } // backupPostgresGlobals stores the cluster's roles beside the dump so a // restore into a fresh cluster (and verify-backup's scratch) can recreate -// object ownership. Best effort: the database dump is the backup. -func (c *Client) backupPostgresGlobals(ctx context.Context, app, name, id string, env map[string]string, workDir string, s3 S3Config) { - _, user := postgresDBAndUser(app, env) +// object ownership. Best effort: the database dump is the backup. A failure +// comes back as a warning, so it is reported and not only printed. +func (c *Client) backupPostgresGlobals(ctx context.Context, t pgTarget, id, workDir string, s3 S3Config, app, name string) string { gz := workDir + "/globals.sql.gz" - if err := c.dumpPostgresGlobals(ctx, newPGTarget(app+"-"+name, "", user, env), workDir, gz); err != nil { - fmt.Fprintf(c.out, "Warning: the cluster roles (pg_dumpall --globals-only) were not saved: %v\n", err) - return + if err := c.dumpPostgresGlobals(ctx, t, workDir, gz); err != nil { + return fmt.Sprintf("the cluster roles (pg_dumpall --globals-only) were not saved: %v", err) } - key := postgresGlobalsKey(s3, app, name, id) - if _, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(gz), ssh.ShellQuote(key)))); err != nil { - fmt.Fprintf(c.out, "Warning: uploading the cluster roles failed: %v\n", err) + if _, err := c.put(ctx, s3, gz, postgresGlobalsKey(s3, app, name, id)); err != nil { + return fmt.Sprintf("uploading the cluster roles failed: %v", err) } + return "" } // dumpPostgresGlobals writes pg_dumpall --globals-only, gzipped, to gz. A @@ -157,50 +443,105 @@ func (c *Client) applyPostgresGlobals(ctx context.Context, super pgTarget, gz st c.exec.Run(ctx, "gunzip -c "+q(gz)+" > "+q(sql)+" && "+super.tool("psql", true)+" -X -q -U "+q(super.user)+" -d postgres < "+q(sql)+" >/dev/null 2>&1; true") } -// backupCoverage says what a backup of this accessory leaves out, per engine. +// backupCoverage says what a default backup of this accessory leaves out, per +// engine (the DR bundle's question; it stores one object per accessory). +func (c *Client) backupCoverage(ctx context.Context, app, name string, det EngineDetection, t pgTarget, env map[string]string) (pgCoverage, []string, []string) { + cov, err := c.coverageWith(ctx, app, name, det, t, env, BackupOptions{}) + w := cov.warningsFor(app, name, det, env) + if err != nil { + w = append(w, err.Error()) + } + return cov, cov.Uncovered, w +} + +// coverageWith works out which databases a backup with these options stores. // Mongo dumps the whole server, so its killed-restore throwaways are dropped // first: they would otherwise land in this backup silently. -func (c *Client) backupCoverage(ctx context.Context, app, name string, det EngineDetection, t pgTarget, env map[string]string) (pgCoverage, []string, []string) { - container := app + "-" + name +func (c *Client) coverageWith(ctx context.Context, app, name string, det EngineDetection, t pgTarget, env map[string]string, opts BackupOptions) (pgCoverage, error) { + container := ContainerName(app, name) + listed := func() []string { + switch { + case opts.SingleDatabase != "": + return []string{opts.SingleDatabase} + case opts.AllDatabases: + return []string{"all"} + case len(opts.Databases) > 0: + return opts.Databases + } + labels, _ := c.inspectEngineHints(ctx, container) + return listedDatabases(labels) + } switch det.Engine { case EnginePostgres: - labels, _ := c.inspectEngineHints(ctx, container) - cov, err := c.pgReadCoverage(ctx, t, listedDatabases(labels)) + cov, err := c.pgReadCoverage(ctx, t, listed()) if err != nil { - return cov, nil, []string{fmt.Sprintf("could not list the cluster's databases, so coverage is unknown: %v", err)} + return cov, fmt.Errorf("could not list the cluster's databases, so coverage is unknown: %w", err) } - return cov, cov.Uncovered, cov.warnings(t.db) + cov.applySingle(t.db, opts) + return cov, nil case EngineMySQL, EngineMariaDB: db := mysqlDB(app, env) - uncovered, left, err := c.mysqlReadCoverage(ctx, det.Engine, container, db) - if err != nil { - return pgCoverage{}, nil, []string{fmt.Sprintf("could not list the server's schemas, so coverage is unknown: %v", err)} + if app == AdoptedApp { + db = "" } - var w []string - if len(uncovered) > 0 { - w = append(w, fmt.Sprintf("this backup covers only %s: the server also holds %s, not backed up", db, strings.Join(uncovered, ", "))) - } - if len(left) > 0 { - w = append(w, fmt.Sprintf("a killed restore left the dry-run database %s: `teploy backup restore --accessory --release` removes it", strings.Join(left, ", "))) + cov, err := c.mysqlCoverage(ctx, det.Engine, container, db, listed()) + if err != nil { + return cov, fmt.Errorf("could not list the server's schemas, so coverage is unknown: %w", err) } - return pgCoverage{}, uncovered, w + cov.applySingle(db, opts) + return cov, nil case EngineMongo: dropped, err := c.dropMongoCheckDatabases(ctx, container) if err != nil { - return pgCoverage{}, nil, []string{fmt.Sprintf("could not clear dry-run databases, so this backup may include them: %v", err)} + return pgCoverage{}, fmt.Errorf("could not clear dry-run databases, so this backup may include them: %w", err) } - if len(dropped) > 0 { - return pgCoverage{}, nil, []string{fmt.Sprintf("dropped the dry-run databases a killed restore left: %s", strings.Join(dropped, ", "))} + return pgCoverage{DroppedThrowaway: dropped}, nil + } + return pgCoverage{}, nil +} + +// applySingle narrows a coverage to the one database a safety backup is for. +func (cov *pgCoverage) applySingle(primary string, opts BackupOptions) { + if opts.SingleDatabase == "" { + return + } + // A database that does not exist has nothing to protect: restoring it after + // it was dropped is the point, not a failure of its safety backup. + cov.Uncovered, cov.Missing = nil, nil + cov.PrimaryIsSingle = opts.SingleDatabase == primary + if cov.PrimaryIsSingle { + cov.Extra = nil + } +} + +// warningsFor renders the coverage problems for messages and events, by engine. +func (cov pgCoverage) warningsFor(app, name string, det EngineDetection, env map[string]string) []string { + switch det.Engine { + case EnginePostgres: + db, _ := postgresDBAndUser(app, env) + return cov.warnings(db) + case EngineMySQL, EngineMariaDB: + var w []string + if len(cov.Uncovered) > 0 { + w = append(w, fmt.Sprintf("this backup covers only %s: the server also holds %s, not backed up", strings.Join(append([]string{mysqlDB(app, env)}, cov.Extra...), ", "), strings.Join(cov.Uncovered, ", "))) + } + if len(cov.Throwaway) > 0 { + w = append(w, fmt.Sprintf("a killed restore left the dry-run database %s: `teploy backup restore --accessory --release` removes it", strings.Join(cov.Throwaway, ", "))) + } + return w + case EngineMongo: + if len(cov.DroppedThrowaway) > 0 { + return []string{fmt.Sprintf("dropped the dry-run databases a killed restore left: %s", strings.Join(cov.DroppedThrowaway, ", "))} } } - return pgCoverage{}, nil, nil + return nil } -// recordPostgresSize stores the database's logical size beside the dump: a +// storeDatabaseMeta stores a database's logical size beside its dump: a // restore's free-space check needs what the BACKUP will occupy, which // neither the compressed artifact nor the live database says. Best effort. -func (c *Client) recordPostgresSize(ctx context.Context, t pgTarget, s3 S3Config, app, name, id, workDir string) { - out, err := c.exec.Run(ctx, t.psql(t.db, "SELECT pg_database_size(current_database())")) +func (c *Client) storeDatabaseMeta(ctx context.Context, t pgTarget, db string, s3 S3Config, key, workDir string) { + out, err := c.exec.Run(ctx, t.psql(db, "SELECT pg_database_size(current_database())")) if err != nil { return } @@ -213,28 +554,11 @@ func (c *Client) recordPostgresSize(ctx context.Context, t pgTarget, s3 S3Config if err := c.exec.Upload(ctx, bytes.NewReader(body), local, "0600"); err != nil { return } - c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(local), ssh.ShellQuote(pgSidecarKey(s3, app, name, "meta", id, ".json"))))) -} - -// backupPostgresExtra dumps one more database of the cluster. Extras sit -// below the accessory's backups, so retention counting and listings see one -// backup per id; `backup restore --database` restores one. -func (c *Client) backupPostgresExtra(ctx context.Context, t pgTarget, db string, s3 S3Config, app, name, id, workDir string) error { - q := ssh.ShellQuote - local := workDir + "/extra.pgdump" - cmd := fmt.Sprintf("%s -Fc -U %s %s > %s && [ \"$(head -c 5 %s)\" = PGDMP ]", t.tool("pg_dump", false), q(t.user), q(db), q(local), q(local)) - if _, err := c.exec.Run(ctx, cmd); err != nil { - return err - } - if _, err := c.checkArtifactSize(ctx, local, minPostgresDump); err != nil { - return err - } - _, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 cp %s %s", q(local), q(pgExtraKey(s3, app, name, db, id))))) - return err + c.put(ctx, s3, local, key) } func pgExtraKey(s3 S3Config, app, name, db, id string) string { - return fmt.Sprintf("s3://%s/%s/accessories/%s/databases/%s/%s.pgdump", s3.Base(), app, name, db, id) + return seriesPrefix(s3, app, name, db) + id + ".pgdump" } // checkArtifactSize measures a freshly written artifact and refuses one @@ -257,31 +581,6 @@ func (c *Client) checkArtifactSize(ctx context.Context, path string, minBytes in return size, nil } -// resolveAccessoryKey finds the stored object for backup date among the -// extensions an engine has used over time (postgres: custom-format -// .pgdump since G7, plain .sql.gz before), by listing the date's prefix. -func (c *Client) resolveAccessoryKey(ctx context.Context, s3 S3Config, prefix, date string, exts ...string) (key, ext string, err error) { - out, lsErr := c.exec.Run(ctx, s3.AWS("s3 ls "+ssh.ShellQuote(prefix+date+"."))) - names := map[string]bool{} - if lsErr == nil { - for _, line := range strings.Split(out, "\n") { - if f := strings.Fields(line); len(f) > 0 { - names[f[len(f)-1]] = true - } - } - } - for _, e := range exts { - if names[date+e] { - return prefix + date + e, e, nil - } - } - detail := "" - if lsErr != nil { - detail = fmt.Sprintf(" (listing: %v)", lsErr) - } - return "", "", fmt.Errorf("no backup %s found under %s (looked for %s)%s", date, prefix, strings.Join(exts, ", "), detail) -} - // pipefail joins producer | consumer so that a failure of EITHER side fails // the whole command. POSIX sh reports only the consumer's status (and // Debian's dash has no `set -o pipefail`), so the producer records its own @@ -360,7 +659,7 @@ const ( // raw-tar branch: generic is chosen only when detection found no engine // (or the teploy.engine=generic label asserts it). func planAccessoryDump(app, name string, det EngineDetection, env map[string]string, workDir string) (accessoryDumpPlan, error) { - containerName := app + "-" + name + containerName := ContainerName(app, name) qContainer := ssh.ShellQuote(containerName) // Dumps are redirected into files with `>`, never piped: a shell @@ -402,7 +701,7 @@ func planAccessoryDump(app, name string, det EngineDetection, env map[string]str base.method, base.consistency, base.ext, base.minBytes = client, consistencyEngineDump, ".sql.gz", minMySQLDump base.cmd = command + " > " + qTmp + " && { tail -c 512 " + qTmp + " | grep -q 'Dump completed' || { echo '" + client + " output has no completion trailer (truncated or failed dump)' >&2; exit 1; }; }" + - " && gzip -c " + qTmp + " > " + ssh.ShellQuote(dumpPath) + " && gzip -c " + qTmp + " > " + ssh.ShellQuote(dumpPath) + " && rm -f " + qTmp return base, nil case EngineMongo: @@ -671,7 +970,7 @@ func redisRestoreScript(container, dumpPath, rdbPath, oldRdb string) string { // stop/copy (A40): "not proven yes" is not "proven no", and an AOF-enabled // redis/valkey would ignore a dump.rdb restore on restart. func (c *Client) redisAOFPreflight(ctx context.Context, engine, app, name string) error { - containerName := app + "-" + name + containerName := ContainerName(app, name) aofOut, aofErr := c.exec.Run(ctx, fmt.Sprintf("docker exec %s %s --raw config get appendonly", ssh.ShellQuote(containerName), redisCLI(engine))) if aofErr != nil { return fmt.Errorf("cannot establish the %s persistence mode for %s: %w", engine, containerName, aofErr) diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 7ae8b13..b3a23c8 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -187,6 +187,17 @@ type S3Config struct { // credentials should back this config (set when it was resolved from // the server's stored target rather than flags). StoredTarget string + // OwnerRecipient, when set, encrypts every object written with this config + // to this age public key and the server's own identity (G1). + OwnerRecipient string + // ExtraIdentity is a server-side identity file fetch also tries (a DR + // restore on a host that is not the one that wrote the backup). + ExtraIdentity string + // PreviousPrefixes are old key prefixes of this bucket, searched by list, + // latest and restore. Only owned ones are ever pruned (G3). + PreviousPrefixes []PreviousPrefix + isPrevious bool + previousOwned bool } // Base is the bucket-root-relative key prefix every backup key is built @@ -239,6 +250,14 @@ type Client struct { // loadDeadline overrides the deadlines on dump loads and their cleanup // (0 = scaled to the dump; tests shorten it). loadDeadline time.Duration + // serverRecip and serverFP memoize the server's age recipient and + // fingerprint for this client's lifetime. + serverRecip, serverFP string + // engineOverride forces a container's engine: an adopted entry's engine:, + // which has no label to live on. + engineOverride map[string]string + // deadline overrides BackupDeadline (tests shorten it). + deadline time.Duration } // NewClient creates a new backup client. @@ -264,19 +283,46 @@ type BackupResult struct { // does not contain; Warnings are non-fatal coverage problems. UncoveredDatabases []string Warnings []string + // Encrypted says the objects are age-encrypted (G1). + Encrypted bool + // Objects are every primary and per-database object this backup stored. + Objects []BackupObject + // FailedDatabases are databases of this server whose dump failed this run, + // with the reason. Non-empty always comes with a returned error. + FailedDatabases map[string]string } -// BackupVolumes archives the app's volumes (and .env) to S3, reporting the -// created backup's identity and size. The .env rides inside the archive as -// a top-level member which restore puts back beside the volumes directory; -// the archive is built inside a private 0700 mktemp workspace (audit F34). -func (c *Client) BackupVolumes(ctx context.Context, app string, s3 S3Config) (res BackupResult, err error) { +// BackupObject is one stored backup object. +type BackupObject struct { + Database string `json:"database,omitempty"` + Key string `json:"key"` + Size int64 `json:"size_bytes"` + Encrypted bool `json:"encrypted"` +} + +// BackupVolumes archives the app's volumes (and, only when the archive will be +// encrypted, .env) to S3, reporting the created backup's identity and size. The +// .env rides inside the archive as a top-level member which restore puts back +// beside the volumes directory. It holds resolved secrets, so an unencrypted +// archive leaves it out and says so. The archive is built inside a private 0700 +// mktemp workspace (audit F34). +func (c *Client) BackupVolumes(ctx context.Context, app string, s3 S3Config) (BackupResult, error) { + return c.BackupVolumesWith(ctx, app, s3, BackupOptions{}) +} + +// BackupVolumesWith is BackupVolumes with options. +func (c *Client) BackupVolumesWith(ctx context.Context, app string, s3 S3Config, opts BackupOptions) (res BackupResult, err error) { start := time.Now() - res = BackupResult{Engine: "volumes", Consistency: consistencyCrash} + ctx, cancel := context.WithTimeout(ctx, c.backupDeadline()) + defer cancel() + res = BackupResult{Engine: "volumes", Consistency: consistencyCrash, Encrypted: s3.Encrypted()} defer func() { res.DurationMs = time.Since(start).Milliseconds() }() if err := c.ensureAWSCLI(ctx); err != nil { return res, err } + if err := c.prepareEncryption(ctx, s3); err != nil { + return res, err + } timestamp, err := newBackupID(time.Now()) if err != nil { @@ -285,70 +331,97 @@ func (c *Client) BackupVolumes(ctx context.Context, app string, s3 S3Config) (re res.ID = timestamp volumesDir := fmt.Sprintf("%s/%s/volumes", deploymentsDir, app) appDir := fmt.Sprintf("%s/%s", deploymentsDir, app) + q := ssh.ShellQuote - runOut, err := c.exec.Run(ctx, "umask 077; mktemp -d /tmp/teploy-backup.XXXXXXXX") + workDir, cleanup, err := c.newBackupWorkspace(ctx) if err != nil { - return res, fmt.Errorf("creating backup workspace: %w", err) - } - workDir := strings.TrimSpace(runOut) - cleanup := func() { - c.exec.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(workDir)) + return res, err } + defer cleanup() archivePath := workDir + "/backup.tar.gz" s3Key := fmt.Sprintf("s3://%s/%s/volumes/%s.tar.gz", s3.Base(), app, timestamp) res.Key = s3Key + if !opts.IgnoreSpace { + // A live tree gives no better estimate than its own size for the tar; + // encrypting holds a second copy until the plaintext is removed. + if out, derr := c.exec.Run(ctx, "du -sk "+q(volumesDir)); derr == nil { + if f := strings.Fields(out); len(f) > 0 { + if kb, perr := strconv.ParseInt(f[0], 10, 64); perr == nil { + need := kb * 1024 + if s3.Encrypted() { + need += need / 2 + } + if free, known := c.freeBytes(ctx, "", workDir); known && free < need+dumpReserveBytes { + return res, fmt.Errorf("not enough free space in %s for this volume archive: %d bytes free, about %d needed plus a %d byte reserve; nothing was archived. Set TEPLOY_BACKUP_TMPDIR to a larger disk or pass --ignore-space-check", workDir, free, need, int64(dumpReserveBytes)) + } + } + } + } + } + + // The .env holds resolved secrets: it only travels encrypted. + if !s3.Encrypted() { + if out, _ := c.exec.Run(ctx, "if [ -f "+q(appDir+"/.env")+" ]; then echo yes; fi"); strings.TrimSpace(out) == "yes" { + w := fmt.Sprintf("%s/.env was left out of this backup: it holds secrets and no recovery key is configured to encrypt it (teploy backup keygen; teploy server backup-policy set --recipient). Restoring this backup keeps the server's current .env", app) + res.Warnings = append(res.Warnings, w) + fmt.Fprintf(c.out, "Warning: %s\n", w) + } + } + // An explicit if/else, not `tar ... || tar ...`: the old fallback // re-ran the tar without .env whenever the first tar failed for ANY - // reason, masking real archive failures as successes. + // reason, masking real archive failures as successes. GNU tar exits 1 with + // "file changed" when a live app writes mid-read (the artifact is labelled + // crash-consistent); real failures exit 2. fmt.Fprintf(c.out, "Archiving volumes for %s...\n", app) - cmd := fmt.Sprintf( - "if [ -f %s ]; then tar -czf %s -C %s . -C %s .env; else tar -czf %s -C %s .; fi", - ssh.ShellQuote(appDir+"/.env"), - ssh.ShellQuote(archivePath), ssh.ShellQuote(volumesDir), ssh.ShellQuote(appDir), - ssh.ShellQuote(archivePath), ssh.ShellQuote(volumesDir), - ) + cmd := fmt.Sprintf("tar -czf %s -C %s . || [ $? -eq 1 ]", q(archivePath), q(volumesDir)) + if s3.Encrypted() { + cmd = fmt.Sprintf( + "if [ -f %s ]; then tar -czf %s -C %s . -C %s .env || [ $? -eq 1 ]; else tar -czf %s -C %s . || [ $? -eq 1 ]; fi", + q(appDir+"/.env"), + q(archivePath), q(volumesDir), q(appDir), + q(archivePath), q(volumesDir), + ) + } if _, err := c.exec.Run(ctx, cmd); err != nil { - cleanup() return res, fmt.Errorf("creating archive: %w", err) } - if size, err := c.remoteSize(ctx, archivePath); err == nil { - res.SizeBytes = size - } - - // Upload to S3. - fmt.Fprintf(c.out, "Uploading to %s...\n", s3Key) - uploadCmd := s3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(archivePath), ssh.ShellQuote(s3Key))) - if _, err := c.exec.Run(ctx, uploadCmd); err != nil { - cleanup() - return res, fmt.Errorf("uploading to S3: %w", err) + if _, err := c.checkArtifactSize(ctx, archivePath, minTarArchive); err != nil { + return res, fmt.Errorf("creating archive: %w", err) } // The app's server data (Teploy config repository and event history) // travels as a sibling object of the same timestamp, never inside the - // volume archive, whose root is the volumes directory. + // volume archive, whose root is the volumes directory. Stored first: the + // volumes object commits the backup. teployDir := confighome.Dir(app) if present, err := c.remoteExists(ctx, teployDir); err != nil { - cleanup() return res, fmt.Errorf("checking %s: %w", teployDir, err) } else if present { teployArchive := workDir + "/teploy.tar.gz" teployKey := fmt.Sprintf("s3://%s/%s/teploy/%s.tar.gz", s3.Base(), app, timestamp) - res.TeployKey = teployKey - tarCmd := fmt.Sprintf("tar -czf %s --exclude=./.write.lock --exclude=./events.lock .", ssh.ShellQuote(teployArchive)) - if _, err := c.exec.Run(ctx, fmt.Sprintf("cd %s && if [ -d config.git ] && command -v flock >/dev/null 2>&1; then flock .write.lock %s; else %s; fi", ssh.ShellQuote(teployDir), tarCmd, tarCmd)); err != nil { - cleanup() + tarCmd := fmt.Sprintf("tar -czf %s --exclude=./.write.lock --exclude=./events.lock .", q(teployArchive)) + if _, err := c.exec.Run(ctx, fmt.Sprintf("cd %s && if [ -d config.git ] && command -v flock >/dev/null 2>&1; then flock .write.lock %s; else %s; fi", q(teployDir), tarCmd, tarCmd)); err != nil { return res, fmt.Errorf("archiving %s: %w", teployDir, err) } fmt.Fprintf(c.out, "Uploading Teploy config and history to %s...\n", teployKey) - if _, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(teployArchive), ssh.ShellQuote(teployKey)))); err != nil { - cleanup() - return res, fmt.Errorf("uploading Teploy config to S3: %w", err) + stored, err := c.put(ctx, s3, teployArchive, teployKey) + if err != nil { + return res, fmt.Errorf("storing Teploy config: %w", err) } + res.TeployKey = stored.Key } - cleanup() - fmt.Fprintf(c.out, "Backup complete: %s\n", s3Key) + fmt.Fprintf(c.out, "Uploading to %s...\n", s3Key) + stored, err := c.put(ctx, s3, archivePath, s3Key) + if err != nil { + return res, fmt.Errorf("storing volume archive: %w", err) + } + res.Key, res.SizeBytes = stored.Key, stored.Size + res.Objects = append(res.Objects, BackupObject{Key: stored.Key, Size: stored.Size, Encrypted: stored.Encrypted}) + c.markNamespaceOwner(ctx, s3, app) + fmt.Fprintf(c.out, "Backup complete: %s\n", res.Key) return res, nil } @@ -362,6 +435,12 @@ func (c *Client) remoteSize(ctx context.Context, path string) (int64, error) { return strconv.ParseInt(strings.TrimSpace(out), 10, 64) } +// FindVolumeBackup finds a volume backup by id across the current and previous +// prefixes, so a restore with nothing to restore stops nothing. +func (c *Client) FindVolumeBackup(ctx context.Context, app, date string, s3 S3Config) (string, string, S3Config, error) { + return c.findObject(ctx, s3, app+"/volumes/", date, ".tar.gz") +} + // RestoreVolumes downloads and extracts a volume backup from S3. If the // archive carries the app-level .env (see BackupVolumes), it is restored // beside the volumes directory in the app directory — the pre-restore file @@ -400,7 +479,10 @@ func (c *Client) RestoreVolumesKeep(ctx context.Context, app, date string, s3 S3 volumesDir := fmt.Sprintf("%s/%s/volumes", deploymentsDir, app) appDir := fmt.Sprintf("%s/%s", deploymentsDir, app) envPath := appDir + "/.env" - s3Key := fmt.Sprintf("s3://%s/%s/volumes/%s.tar.gz", s3.Base(), app, date) + s3Key, _, loc, err := c.findObject(ctx, s3, app+"/volumes/", date, ".tar.gz") + if err != nil { + return "", err + } runOut, err := c.exec.Run(ctx, "umask 077; mktemp -d /tmp/teploy-volume-restore.XXXXXXXX") if err != nil { @@ -415,7 +497,7 @@ func (c *Client) RestoreVolumesKeep(ctx context.Context, app, date string, s3 S3 stagedEnv := runDir + "/new.env" fmt.Fprintf(c.out, "Downloading %s...\n", s3Key) - if _, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(s3Key), ssh.ShellQuote(archivePath)))); err != nil { + if err := c.fetch(ctx, loc, s3Key, archivePath); err != nil { cleanupRun() return "", fmt.Errorf("downloading from S3: %w", err) } @@ -704,9 +786,28 @@ func (c *Client) SetSchedule(ctx context.Context, schedule, command, marker stri return nil } +// AWSCLIRemedy is the exact way to give a server the aws CLI. `teploy setup` +// runs AWSCLIInstallScript; Ubuntu 24.04+ has no apt package for it. +const AWSCLIRemedy = "run `teploy setup` against the server, or on the server: apt-get install -y awscli || { curl -fsSL \"https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip\" -o /tmp/awscliv2.zip && python3 -m zipfile -e /tmp/awscliv2.zip /tmp/awscliv2 && chmod -R 755 /tmp/awscliv2/aws && /tmp/awscliv2/aws/install; }" + +// AWSCLIInstallScript installs the aws CLI: the distribution package when it +// has one, else AWS's own bundle. python3 unpacks it (unzip may be absent), which +// drops the file modes, so they are restored for every user before installing: +// with u+x alone the installed aws runs for root and "Permission denied"s for cron. +const AWSCLIInstallScript = `set -e +export DEBIAN_FRONTEND=noninteractive +if apt-get install -y -qq awscli >/dev/null 2>&1 && command -v aws >/dev/null 2>&1; then exit 0; fi +d=$(mktemp -d /tmp/teploy-awscli.XXXXXXXX) +trap 'rm -rf "$d"' EXIT +curl -fsSL --max-time 300 "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "$d/a.zip" +python3 -m zipfile -e "$d/a.zip" "$d/x" +chmod -R 755 "$d/x/aws" +"$d/x/aws/install" --update +command -v aws >/dev/null 2>&1` + func (c *Client) ensureAWSCLI(ctx context.Context) error { if _, err := c.exec.Run(ctx, "which aws"); err != nil { - return fmt.Errorf("aws CLI not found on server — install with: apt install awscli") + return fmt.Errorf("aws CLI not found on server — %s", AWSCLIRemedy) } return nil } diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index 5a1aa0b..5899e8c 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -17,21 +17,28 @@ import ( ) func TestBackupVolumes(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, ssh.MockCommand{Match: "if [ -f", Output: ""}, + ssh.MockCommand{Match: "tar -czf", Output: ""}, ssh.MockCommand{Match: "aws s3 cp", Output: "upload: done\n"}, ssh.MockCommand{Match: "rm -f", Output: ""}, ssh.MockCommand{Match: "if [ -e '/deployments/.teploy/apps/myapp' ]", Output: "absent\n"}, + ssh.MockCommand{Match: "command -v age", Output: "ok\n"}, + ssh.MockCommand{Match: "if [ -r", Output: "yes\n"}, + ssh.MockCommand{Match: "age-keygen -y", Output: testServerRecipient + "\n"}, + ssh.MockCommand{Match: "age -r", Output: ""}, + ssh.MockCommand{Match: "age -d", Output: ""}, ) var buf bytes.Buffer client := NewClient(mock, &buf) _, err := client.BackupVolumes(context.Background(), "myapp", S3Config{ - Bucket: "my-bucket", - Region: "us-east-1", + Bucket: "my-bucket", + Region: "us-east-1", + OwnerRecipient: testOwnerRecipient, }) if err != nil { t.Fatalf("BackupVolumes: %v", err) @@ -53,6 +60,9 @@ func TestBackupVolumes(t *testing.T) { if strings.Contains(call, "tar -czf") { tarCmd = call } + if strings.Contains(call, "aws s3 cp '/tmp/teploy-backup.abc123/backup.tar.gz'") { + t.Errorf("the plaintext archive must never be uploaded when encrypting: %s", call) + } } foundS3 := false for _, call := range mock.Calls { @@ -77,7 +87,7 @@ func TestBackupVolumes(t *testing.T) { } func TestAccessoryBackup_Postgres(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, @@ -123,7 +133,7 @@ func TestAccessoryBackup_Postgres(t *testing.T) { // pipeline still reported "Backup complete" — see the exit-code-masking // test below for that half of the bug. func TestAccessoryBackup_Postgres_CustomDBName(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, @@ -169,7 +179,7 @@ func TestAccessoryBackup_Postgres_CustomDBName(t *testing.T) { // 20-byte gzip of nothing. The fixed command redirects pg_dump's own // output with `>` (not a pipe) so its actual exit code propagates. func TestAccessoryBackup_Postgres_DumpFailureIsNotSwallowed(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, @@ -194,7 +204,7 @@ func TestAccessoryBackup_Postgres_DumpFailureIsNotSwallowed(t *testing.T) { } func TestAccessoryBackup_MySQL(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, @@ -228,7 +238,7 @@ func TestAccessoryBackup_MySQL(t *testing.T) { // `sleep 2`; a slow snapshot copied the previous dump.rdb instead. The dump // command must capture LASTSAVE before bgsave and poll until it changes. func TestAccessoryBackup_RedisWaitsForBgsave(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, @@ -279,7 +289,7 @@ func TestAccessoryBackup_RedisWaitsForBgsave(t *testing.T) { // MYSQL_PASSWORD) and reach the container as MYSQL_PWD env — never argv, // which `ps` inside the container would show. func TestAccessoryBackup_MySQL_UsesRootPasswordEnv(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, @@ -328,7 +338,7 @@ func TestAccessoryBackup_MySQL_PasswordFallbackAndAbsence(t *testing.T) { var buf bytes.Buffer // MYSQL_PASSWORD is the fallback when MYSQL_ROOT_PASSWORD is unset. - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, @@ -359,7 +369,7 @@ func TestAccessoryBackup_MySQL_PasswordFallbackAndAbsence(t *testing.T) { } // No password configured: keep the bare command (passwordless root). - mock = ssh.NewMockExecutor("1.2.3.4", + mock = newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, @@ -376,7 +386,7 @@ func TestAccessoryBackup_MySQL_PasswordFallbackAndAbsence(t *testing.T) { for _, call := range mock.Calls { // The read-only coverage query reads whatever the container's env // holds, in the container; the dump is what must not expect one. - if strings.Contains(call, "MYSQL_PWD") && !strings.Contains(call, "information_schema.SCHEMATA") { + if strings.Contains(call, "MYSQL_PWD") && !strings.Contains(call, "information_schema.") { t.Errorf("no password configured, so no MYSQL_PWD expected: %s", call) } } @@ -386,7 +396,7 @@ func TestAccessoryBackup_MySQL_PasswordFallbackAndAbsence(t *testing.T) { // shell exposure at all — there is no quoting surface left to get right // because the password never enters a command string. func TestAccessoryBackup_MySQL_QuotesHostilePassword(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, @@ -417,11 +427,11 @@ func TestAccessoryBackup_MySQL_QuotesHostilePassword(t *testing.T) { } func TestAccessoryRestore_MySQL_UsesRootPasswordEnv(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 99 20260101-000000.sql.gz\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: "download: done\n"}, - ssh.MockCommand{Match: "mktemp -d '/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, + ssh.MockCommand{Match: "mktemp -d '/var/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, ssh.MockCommand{Match: "umask 077; gunzip -c", Output: ""}, ssh.MockCommand{Match: "docker exec", Output: ""}, ssh.MockCommand{Match: "rm -f", Output: ""}, @@ -462,11 +472,11 @@ func TestAccessoryRestore_MySQL_UsesRootPasswordEnv(t *testing.T) { // deliberately kept for inspection on failure, but the mysql credential // must be removed by name from what stays behind. func TestAccessoryRestore_MySQL_FailureKeepsSQLNotSecret(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: "download: done\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 99 20260101-000000.sql.gz\n"}, - ssh.MockCommand{Match: "mktemp -d '/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, + ssh.MockCommand{Match: "mktemp -d '/var/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, // The restore command fails so the failure path runs. ssh.MockCommand{Match: "umask 077; gunzip -c", Err: errors.New("exit status 1: gzip: invalid")}, ssh.MockCommand{Match: "rm -f", Output: ""}, @@ -496,7 +506,7 @@ func TestAccessoryRestore_MySQL_FailureKeepsSQLNotSecret(t *testing.T) { } func TestAccessoryBackup_Generic(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, @@ -532,7 +542,7 @@ func TestAccessoryBackup_Generic(t *testing.T) { // truncated the job at the %. The installed line must escape the command's // % signs while leaving the marker comment (and its grep -vF dedup) intact. func TestSetSchedule_EscapesPercentForCron(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "raw=$(crontab -l", Output: ""}, ) @@ -569,7 +579,7 @@ func TestSetSchedule_EscapesPercentForCron(t *testing.T) { } func TestListBackups(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "2026-03-11 00:00:00 1024 20260311-000000.tar.gz\n2026-03-10 12:00:00 2048 20260310-120000.tar.gz\n"}, ) @@ -589,7 +599,7 @@ func TestListBackups(t *testing.T) { } func TestEnsureAWSCLI_Missing(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Err: errNotFound}, ) @@ -753,7 +763,7 @@ esac t.Fatal(err) } - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "raw=$(crontab -l", Err: errors.New("unused")}, ) var buf bytes.Buffer @@ -823,7 +833,7 @@ func TestValidateSchedule_RejectsSignedValues(t *testing.T) { // marker — direct callers cannot bypass validation or split one job into // several crontab lines. func TestSetSchedule_EnforcesGrammarAtSink(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4") + mock := newStoreMock("1.2.3.4") client := NewClient(mock, io.Discard) if err := client.SetSchedule(context.Background(), "99 * * * *", "cmd", "tag"); err == nil { t.Error("an out-of-range schedule must be rejected at the sink") @@ -855,7 +865,7 @@ func TestRedisRestore_AOFRequiresProvenNo(t *testing.T) { {"yes is refused", "appendonly yes", nil}, } { t.Run(tc.name, func(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 99 20260101-000000.rdb.gz\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: "ok"}, @@ -885,7 +895,7 @@ func TestRedisRestore_AOFRequiresProvenNo(t *testing.T) { // compensation invoked from BOTH failure branches — a failed install and a // failed final docker start. func TestRedisRestore_SnapshotsAfterStopAndCompensatesStartFailure(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 99 20260101-000000.rdb.gz\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: "ok"}, @@ -929,11 +939,12 @@ func TestRedisRestore_SnapshotsAfterStopAndCompensatesStartFailure(t *testing.T) // sibling object of the same timestamp, taken under the config writer's // flock, never inside the volume archive. func TestBackupVolumesIncludesTeployServerData(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, ssh.MockCommand{Match: "if [ -f", Output: ""}, + ssh.MockCommand{Match: "tar -czf", Output: ""}, ssh.MockCommand{Match: "aws s3 cp", Output: "upload: done\n"}, ssh.MockCommand{Match: "rm -f", Output: ""}, ssh.MockCommand{Match: "if [ -e '/deployments/.teploy/apps/myapp' ]", Output: "present\n"}, diff --git a/internal/backup/bundle.go b/internal/backup/bundle.go index 05970a3..1f007df 100644 --- a/internal/backup/bundle.go +++ b/internal/backup/bundle.go @@ -5,6 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "io" + "path" "sort" "strings" "time" @@ -111,6 +113,10 @@ type SecretsRecord struct { // can only be decrypted with the ORIGINAL host's age key — Recovery // states that requirement in operator language. AgeKeyIncluded bool `json:"age_key_included"` + // AgeKeyWrapped says the bundled key is the member age-key.age, encrypted to + // the owner's recovery key only: restoring it needs `dr restore --identity`. + // A bundle with AgeKeyIncluded and not this is from before G1 (plain member). + AgeKeyWrapped bool `json:"age_key_wrapped,omitempty"` // Included lists the secret-material members inside the bundle // (populated only in "encrypted" mode). Included []string `json:"included,omitempty"` @@ -256,18 +262,41 @@ func (s S3BundleStore) key(app, id, member string) string { return fmt.Sprintf("s3://%s/%s/dr/%s/%s", s.S3.Base(), app, id, member) } +// Upload stores a member through the same verified path as every other backup +// object, encrypted when a recovery key is configured (key.age). The manifest +// stays plaintext (it is the completeness marker and a list of what is inside; +// its members' names are visible in the bucket anyway) and a member that is +// already age ciphertext is not wrapped twice. func (s S3BundleStore) Upload(ctx context.Context, exec ssh.Executor, app, id, member, serverPath string) error { - _, err := exec.Run(ctx, s.S3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(serverPath), ssh.ShellQuote(s.key(app, id, member))))) - if err != nil { + cfg := s.S3 + if member == "manifest.json" || strings.HasSuffix(member, ageSuffix) { + cfg.OwnerRecipient = "" + } + if _, err := NewClient(exec, io.Discard).put(ctx, cfg, serverPath, s.key(app, id, member)); err != nil { return fmt.Errorf("uploading %s: %w", s.key(app, id, member), err) } return nil } +// Download fetches a member, decrypting it when it was stored encrypted: +// bundles from before G1, and bundles made with no recovery key, have the plain +// member name. func (s S3BundleStore) Download(ctx context.Context, exec ssh.Executor, app, id, member, destPath string) error { - _, err := exec.Run(ctx, s.S3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(s.key(app, id, member)), ssh.ShellQuote(destPath)))) + key := s.key(app, id, member) + client := NewClient(exec, io.Discard) + if out, err := exec.Run(ctx, s.S3.AWS("s3 ls "+ssh.ShellQuote(key+ageSuffix))); err == nil && !strings.HasSuffix(member, ageSuffix) { + for _, line := range strings.Split(out, "\n") { + if f := strings.Fields(line); len(f) > 0 && f[len(f)-1] == path.Base(member)+ageSuffix { + if err := client.fetch(ctx, s.S3, key+ageSuffix, destPath); err != nil { + return fmt.Errorf("downloading %s: %w", key+ageSuffix, err) + } + return nil + } + } + } + _, err := exec.Run(ctx, s.S3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(key), ssh.ShellQuote(destPath)))) if err != nil { - return fmt.Errorf("downloading %s: %w", s.key(app, id, member), err) + return fmt.Errorf("downloading %s: %w", key, err) } return nil } @@ -402,6 +431,10 @@ type BundleOptions struct { // IncludeAgeKey also carries /deployments/.age-key so a fresh host can // decrypt the ciphertexts. Even more sensitive; never default. IncludeAgeKey bool + // OwnerRecipient is the recovery key an included age key is wrapped to. + // IncludeAgeKey refuses without one: the key is the keys to every secret on + // the server, and a bucket must not hold it readable. + OwnerRecipient string // StopApp stops the app's web containers before snapshotting volumes // (and restarts them after): the copies are then QUIESCED rather than @@ -614,6 +647,14 @@ func (c *Client) CreateBundle(ctx context.Context, opts BundleOptions, store Bun // The identity travels from wherever it lives on this target (D2 // moved it out of /deployments; ResolveIdentity is shared with the // secret store so the two can never disagree). + if opts.OwnerRecipient == "" { + cleanup() + return nil, fmt.Errorf("--include-age-key needs a recovery key to wrap the server's private key to: nothing was created. Generate one with `teploy backup keygen` and store it with `teploy server backup-policy set --recipient `") + } + if err := ValidateRecipient(opts.OwnerRecipient); err != nil { + cleanup() + return nil, err + } keyPath, err := secret.ResolveIdentity(ctx, c.exec) if err != nil { cleanup() @@ -623,13 +664,17 @@ func (c *Client) CreateBundle(ctx context.Context, opts BundleOptions, store Bun cleanup() return nil, fmt.Errorf("--include-age-key: no age identity on the target (expected %s or %s)", secret.IdentityPath, secret.LegacyIdentityPath) } - if err := c.wsCopy(ctx, ws, "age-key", keyPath); err != nil { + // Straight from the identity file to ciphertext: no plaintext copy of the + // key ever sits in the bundle workspace. + wrapped := ws + "/age-key" + ageSuffix + if _, err := c.exec.Run(ctx, "umask 077; age -r "+ssh.ShellQuote(opts.OwnerRecipient)+" -o "+ssh.ShellQuote(wrapped)+" "+ssh.ShellQuote(keyPath)); err != nil { cleanup() - return nil, err + return nil, fmt.Errorf("wrapping the age key to the recovery key: %w", err) } m.Secrets.AgeKeyIncluded = true + m.Secrets.AgeKeyWrapped = true if !opts.IncludeSecrets { - m.Secrets.Recovery += " — the bundle carries the age key itself (--include-age-key); use it to re-key a fresh target's secret store" + m.Secrets.Recovery += " — the bundle carries the age key itself (--include-age-key), wrapped to the recovery key; `teploy dr restore --identity ` unwraps it to re-key a fresh target's secret store" } } @@ -829,7 +874,7 @@ func (c *Client) CreateBundle(ctx context.Context, opts BundleOptions, store Bun } } if opts.IncludeAgeKey { - if err := uploadMember("age-key", ws+"/age-key"); err != nil { + if err := uploadMember("age-key"+ageSuffix, ws+"/age-key"+ageSuffix); err != nil { cleanup() return nil, err } diff --git a/internal/backup/bundle_campaign_test.go b/internal/backup/bundle_campaign_test.go index a4da80f..5dc39c7 100644 --- a/internal/backup/bundle_campaign_test.go +++ b/internal/backup/bundle_campaign_test.go @@ -31,7 +31,7 @@ func TestBundleManifestRejectsUnsafeMembersBeforeStaging(t *testing.T) { } func TestS3BundleListRecognizesPrefixRows(t *testing.T) { - mock := ssh.NewMockExecutor("host", ssh.MockCommand{Match: "aws", Output: " PRE 20260923-101500-0123456789abcdef/\n2026-09-23 10:15:00 12 manifest.json\n"}) + mock := newStoreMock("host", ssh.MockCommand{Match: "aws", Output: " PRE 20260923-101500-0123456789abcdef/\n2026-09-23 10:15:00 12 manifest.json\n"}) ids, err := (S3BundleStore{S3: S3Config{Bucket: "dummy"}}).ListIDs(context.Background(), mock, "myapp") if err != nil { t.Fatal(err) diff --git a/internal/backup/bundle_confighome_test.go b/internal/backup/bundle_confighome_test.go index ad13c2a..c4c26b9 100644 --- a/internal/backup/bundle_confighome_test.go +++ b/internal/backup/bundle_confighome_test.go @@ -15,7 +15,7 @@ const ( ) func homeMock(targetRepo string) *ssh.MockExecutor { - mock := ssh.NewMockExecutor("1.2.3.4") + mock := newStoreMock("1.2.3.4") if targetRepo != "" { repo := "/deployments/.teploy/apps/blog/config.git" mock.Files[repo+"/HEAD"] = []byte("ref: refs/heads/teploy\n") diff --git a/internal/backup/bundle_restore.go b/internal/backup/bundle_restore.go index ea3147f..167c638 100644 --- a/internal/backup/bundle_restore.go +++ b/internal/backup/bundle_restore.go @@ -97,9 +97,12 @@ type CutoverReceipt struct { // BundleRestoreOptions configures an isolated restore or cutover. type BundleRestoreOptions struct { - App string - ID string - Config config.AppConfig // the restore-time local teploy.yml + // Identity is a server-side age identity file: the recovery key, to unwrap a + // wrapped age key on a host that is not the one that made the bundle. + Identity string + App string + ID string + Config config.AppConfig // the restore-time local teploy.yml // Home resolves a config-home mismatch at cutover between the bundle // and an existing target app: "bundle" installs the bundle's home and // config, "current" keeps the target's. Empty refuses a mismatch. @@ -164,7 +167,17 @@ func (c *Client) RestoreBundleIsolated(ctx context.Context, opts BundleRestoreOp } } if manifest.Secrets.AgeKeyIncluded { - if err := store.Download(ctx, c.exec, opts.App, manifest.ID, "age-key", staging+"/age-key"); err != nil { + if manifest.Secrets.AgeKeyWrapped { + if opts.Identity == "" { + return nil, fmt.Errorf("this bundle's age key is wrapped to the owner's recovery key: pass --identity (nothing was changed)") + } + if err := store.Download(ctx, c.exec, opts.App, manifest.ID, "age-key"+ageSuffix, staging+"/age-key"+ageSuffix); err != nil { + return nil, err + } + if _, err := c.exec.Run(ctx, "umask 077; age -d -i "+ssh.ShellQuote(opts.Identity)+" -o "+ssh.ShellQuote(staging+"/age-key")+" "+ssh.ShellQuote(staging+"/age-key"+ageSuffix)); err != nil { + return nil, fmt.Errorf("unwrapping the bundle's age key with the given identity: %w", err) + } + } else if err := store.Download(ctx, c.exec, opts.App, manifest.ID, "age-key", staging+"/age-key"); err != nil { return nil, err } } diff --git a/internal/backup/bundle_restore_test.go b/internal/backup/bundle_restore_test.go index efb74a5..c1e9672 100644 --- a/internal/backup/bundle_restore_test.go +++ b/internal/backup/bundle_restore_test.go @@ -41,7 +41,7 @@ func drRestoreMock(manifest *BundleManifest, extra ...ssh.MockCommand) *ssh.Mock staging := DRStagingPath("myapp", manifest.ID) manifestJSON, _ := json.Marshal(manifest) // extras are PREPENDED so injected failures outrank the base successes. - mock := ssh.NewMockExecutor("dr-host", append(extra, []ssh.MockCommand{ + mock := newStoreMock("dr-host", append(extra, []ssh.MockCommand{ {Match: "rm -rf '" + staging, Output: ""}, {Match: "umask 077; mkdir -p '" + staging, Output: ""}, {Match: "mkdir -p '" + staging, Output: ""}, @@ -231,7 +231,7 @@ func drCutoverMock(receipt *RestoreReceipt, manifest *BundleManifest, extra ...s receiptJSON, _ := json.Marshal(receipt) manifestJSON, _ := json.Marshal(manifest) // extras are PREPENDED so injected failures outrank the base successes. - mock := ssh.NewMockExecutor("dr-host", append(extra, []ssh.MockCommand{ + mock := newStoreMock("dr-host", append(extra, []ssh.MockCommand{ {Match: "mkdir /deployments/myapp/.lock", Output: ""}, // Accessory admission (R2-06) runs before any cutover effect. {Match: "teploy_volume_actor", Output: ""}, @@ -281,7 +281,7 @@ func drOKReceipt() *RestoreReceipt { func TestCutover_RequiresValidatedStaging(t *testing.T) { manifest := drTestManifest() - noReceipt := ssh.NewMockExecutor("dr-host") + noReceipt := newStoreMock("dr-host") _, err := NewClient(noReceipt, &bytes.Buffer{}).CutoverBundle(context.Background(), BundleRestoreOptions{App: "myapp", ID: manifest.ID, Config: drTestConfig()}) if !errors.Is(err, ErrValidationRequired) { diff --git a/internal/backup/bundle_secretmeta_test.go b/internal/backup/bundle_secretmeta_test.go index 590c1a9..164c0ce 100644 --- a/internal/backup/bundle_secretmeta_test.go +++ b/internal/backup/bundle_secretmeta_test.go @@ -26,7 +26,7 @@ func assertNoOldSecrets(t *testing.T, where string, data []byte) { // A DR bundle is a copy that travels: records written before redaction must // be exported without their secret env values, other fields intact. func TestCollectReleaseRecords_RedactsSecretEnv(t *testing.T) { - mock := ssh.NewMockExecutor("dr-host", + mock := newStoreMock("dr-host", ssh.MockCommand{Match: "find '/deployments/myapp/meta'", Output: "/deployments/myapp/meta/abc123.json\n"}, ) mock.Files["/deployments/myapp/meta/abc123.json"] = []byte(oldShapeRecord) @@ -50,7 +50,7 @@ func TestCollectReleaseRecords_RedactsSecretEnv(t *testing.T) { // Restoring a bundle made before redaction must not put secret values back // at rest on the target. func TestInstallReleaseRecords_RedactsOldBundles(t *testing.T) { - mock := ssh.NewMockExecutor("dr-host") + mock := newStoreMock("dr-host") err := NewClient(mock, &bytes.Buffer{}).installReleaseRecords(context.Background(), "/deployments/myapp", []json.RawMessage{json.RawMessage(oldShapeRecord)}) if err != nil { t.Fatal(err) diff --git a/internal/backup/bundle_test.go b/internal/backup/bundle_test.go index 2c9c696..16cc53f 100644 --- a/internal/backup/bundle_test.go +++ b/internal/backup/bundle_test.go @@ -16,7 +16,7 @@ import ( const drTestState = `{"schema_version":2,"deployment_type":"container","ingress_mode":"caddy","domain":"myapp.example.com","updated_at":"2026-09-20T10:00:00Z","image_ref":"nginx:1.27","generation":4,"current_port":3000,"current_hash":"abc123"}` func drBaseMock(extra ...ssh.MockCommand) *ssh.MockExecutor { - mock := ssh.NewMockExecutor("dr-host", append([]ssh.MockCommand{ + mock := newStoreMock("dr-host", append([]ssh.MockCommand{ {Match: "umask 077; mktemp -d /tmp/teploy-dr.XXXXXXXX", Output: "/tmp/drws1\n"}, {Match: "mkdir -p '/tmp/drws1", Output: ""}, {Match: "mkdir -p '/var/drstore/myapp/dr/", Output: ""}, @@ -135,7 +135,7 @@ func TestCreateBundle_ManifestAndConsistency(t *testing.T) { // nothing to recover — the error must say so and name the data-only // command, not mint a bundle that lies about scope. func TestCreateBundle_NoStateRefuses(t *testing.T) { - mock := ssh.NewMockExecutor("dr-host", + mock := newStoreMock("dr-host", ssh.MockCommand{Match: "umask 077; mktemp -d /tmp/teploy-dr.XXXXXXXX", Output: "/tmp/drws1\n"}, ) client := NewClient(mock, &bytes.Buffer{}) @@ -338,7 +338,7 @@ func TestDeriveAppRun_PicksNewestRecord(t *testing.T) { // only directories that CONTAIN manifest.json. func TestDirStore_ListIDs(t *testing.T) { root := "/var/drstore" - mock := ssh.NewMockExecutor("h", + mock := newStoreMock("h", ssh.MockCommand{ Match: "find '/var/drstore/myapp/dr'", Output: root + "/myapp/dr/20260923-101500-0123456789abcdef\n", diff --git a/internal/backup/cron.go b/internal/backup/cron.go index 320d594..0dd31f1 100644 --- a/internal/backup/cron.go +++ b/internal/backup/cron.go @@ -33,7 +33,13 @@ type CronMarker struct { Command string } -var cronMarkerRE = regexp.MustCompile(`\[(teploy-backup:[A-Za-z0-9._-]+|teploy-accessory-backup:[A-Za-z0-9._-]+:[A-Za-z0-9._-]+)\]\s*$`) +var cronMarkerRE = regexp.MustCompile(`\[(teploy-backup-all|teploy-backup:[A-Za-z0-9._-]+|teploy-accessory-backup:[A-Za-z0-9._-]+:[A-Za-z0-9._-]+)\]\s*$`) + +// FleetCronMarker tags the one policy cron line, `teploy backup run --all`. +const FleetCronMarker = "teploy-backup-all" + +// KindFleet is the CronMarker kind of the policy line. +const KindFleet = "fleet" var legacyKeepLastRE = regexp.MustCompile(`head -n -([0-9]+)`) @@ -50,6 +56,10 @@ func FindCronMarker(line string) *CronMarker { return nil } mk := &CronMarker{Tag: tag, Cron: strings.Join(fields[:5], " "), Command: strings.Join(fields[5:], " ")} + if tag == FleetCronMarker { + mk.Kind, mk.Form = KindFleet, CronFormRun + return mk + } if rest, ok := strings.CutPrefix(tag, "teploy-accessory-backup:"); ok { app, acc, found := strings.Cut(rest, ":") if !found { diff --git a/internal/backup/engine.go b/internal/backup/engine.go index fd0cde0..c74f3dc 100644 --- a/internal/backup/engine.go +++ b/internal/backup/engine.go @@ -229,10 +229,30 @@ func engineFromImageName(image string) string { // labels and entrypoint are read best-effort (an absent container — the // teploy.yml fallback — still classifies from image and env). func (c *Client) ResolveEngine(ctx context.Context, app, name, image string, env map[string]string) (EngineDetection, error) { - labels, argv := c.inspectEngineHints(ctx, app+"-"+name) + container := ContainerName(app, name) + if e := c.engineOverride[container]; e != "" { + return EngineDetection{Engine: e, Source: "policy"}, nil + } + labels, argv := c.inspectEngineHints(ctx, container) return DetectEngine(image, env, labels, argv) } +// SetEngineOverride forces a container's engine (an adopted policy entry's +// engine:, which has no label to live on). +func (c *Client) SetEngineOverride(container, engine string) error { + if engine == "" { + return nil + } + if !knownEngines[engine] { + return fmt.Errorf("unknown engine %q (postgres, mysql, mariadb, redis, valkey, dragonfly, mongo)", engine) + } + if c.engineOverride == nil { + c.engineOverride = map[string]string{} + } + c.engineOverride[container] = engine + return nil +} + // inspectEngineHints reads the container's labels and Entrypoint+Cmd. func (c *Client) inspectEngineHints(ctx context.Context, container string) (map[string]string, []string) { out, err := c.exec.Run(ctx, fmt.Sprintf("docker inspect -f '{{json .Config.Labels}}\n{{json .Config.Entrypoint}}\n{{json .Config.Cmd}}' %s", ssh.ShellQuote(container))) diff --git a/internal/backup/engine_test.go b/internal/backup/engine_test.go index c1fffef..d0f23ca 100644 --- a/internal/backup/engine_test.go +++ b/internal/backup/engine_test.go @@ -72,7 +72,7 @@ func TestDetectEngine(t *testing.T) { } func TestResolveEngineReadsContainerHints(t *testing.T) { - mock := ssh.NewMockExecutor("h", + mock := newStoreMock("h", ssh.MockCommand{Match: "docker inspect -f '{{json .Config.Labels}}", Output: `{"teploy.engine":"valkey"}` + "\nnull\n" + `["valkey-server"]` + "\n"}, ) det, err := NewClient(mock, io.Discard).ResolveEngine(context.Background(), "app", "cache", "acme/kv:1", nil) @@ -80,7 +80,7 @@ func TestResolveEngineReadsContainerHints(t *testing.T) { t.Fatalf("ResolveEngine = %+v, %v", det, err) } // An absent container still classifies from image and env. - det, err = NewClient(ssh.NewMockExecutor("h"), io.Discard).ResolveEngine(context.Background(), "app", "db", "pgvector/pgvector:pg17", map[string]string{"PG_MAJOR": "17"}) + det, err = NewClient(newStoreMock("h"), io.Discard).ResolveEngine(context.Background(), "app", "db", "pgvector/pgvector:pg17", map[string]string{"PG_MAJOR": "17"}) if err != nil || det.Engine != EnginePostgres { t.Fatalf("ResolveEngine without container = %+v, %v", det, err) } @@ -148,7 +148,7 @@ func TestPlanAccessoryDump_EngineCommands(t *testing.T) { // A detected pgvector accessory is dumped with pg_dump even though its // image is not called "postgres" — the U3 false green. func TestAccessoryBackup_PgvectorIsDumpedNotTarred(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, ssh.MockCommand{Match: "docker exec 'myapp-db' pg_dump -Fc", Output: ""}, @@ -161,7 +161,7 @@ func TestAccessoryBackup_PgvectorIsDumpedNotTarred(t *testing.T) { if err != nil { t.Fatal(err) } - if res.Engine != EnginePostgres || res.Consistency != consistencyEngineDump || res.SizeBytes != 2123 || !strings.HasSuffix(res.Key, ".pgdump") { + if res.Engine != EnginePostgres || res.Consistency != consistencyEngineDump || res.SizeBytes != 4096 || !strings.HasSuffix(res.Key, ".pgdump") { t.Fatalf("unexpected result %+v", res) } for _, c := range mock.Calls { @@ -174,7 +174,7 @@ func TestAccessoryBackup_PgvectorIsDumpedNotTarred(t *testing.T) { // The deploy-ovh incident: a dump that "succeeds" with an implausibly small // artifact (gzip of an empty stream) must fail and never be uploaded. func TestAccessoryBackup_RefusesImplausiblySmallArtifact(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, ssh.MockCommand{Match: "docker exec", Output: ""}, @@ -241,7 +241,7 @@ func TestDragonflyLoadScriptOrder(t *testing.T) { // verify-backup classifies a timescale accessory as postgres (restored and // queried), never as a volume archive that passes on files>0. func TestVerifyBackup_TimescaleIsVerifiedAsPostgres(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "umask 077; mktemp -d", Output: "/var/tmp/teploy-verify.test\n"}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "docker inspect -f '{{.Config.Image}}'", Output: "timescale/timescaledb:latest-pg17\n"}, diff --git a/internal/backup/list.go b/internal/backup/list.go index 9423064..96c5f5b 100644 --- a/internal/backup/list.go +++ b/internal/backup/list.go @@ -5,11 +5,8 @@ import ( "fmt" "regexp" "sort" - "strconv" "strings" "time" - - "github.com/useteploy/teploy/internal/ssh" ) // Listing (D3): `backup list --json` reports parsed rows across the @@ -24,54 +21,73 @@ const ( // ListedBackup is one parsed object under the app's S3 layout. type ListedBackup struct { - ID string `json:"id"` - Kind string `json:"kind"` - Accessory string `json:"accessory,omitempty"` + ID string `json:"id"` + Kind string `json:"kind"` + Accessory string `json:"accessory,omitempty"` + // Database names one database's own object (`backup restore --database`); + // empty for an accessory's primary object. + Database string `json:"database,omitempty"` SizeBytes int64 `json:"size_bytes"` CreatedAt string `json:"created_at"` Key string `json:"key"` Restorable bool `json:"restorable"` + // Encrypted says the object is age-encrypted (.age). + Encrypted bool `json:"encrypted"` + // Previous says the object sits under an old prefix of the bucket: found + // and restorable, never pruned unless that prefix is marked owned. + Previous bool `json:"previous_prefix,omitempty"` } // listedIDRE matches the object names the CLI writes: the backup-id // grammar (timestamp or timestamp+hex) with each branch's extension. -var listedIDRE = regexp.MustCompile(`^([0-9]{8}-[0-9]{6}(?:-[0-9a-f]{16})?)\.(tar\.gz|pgdump|sql\.gz|archive\.gz|rdb\.gz)$`) +var listedIDRE = regexp.MustCompile(`^([0-9]{8}-[0-9]{6}(?:-[0-9a-f]{16})?)\.(tar\.gz|pgdump|sql\.gz|archive\.gz|rdb\.gz)(\.age)?$`) // ListBackupsRecursive lists every object under s3://// in one -// recursive call and classifies each by its key layout. Objects whose name -// does not match the CLI's naming (strays, partial uploads) are skipped. +// recursive call per location (current prefix, then any previous one) and +// classifies each by its key layout. Objects whose name does not match the +// CLI's naming (strays, pending uploads) are skipped, and so are objects below +// their engine's plausibility floor: neither is a backup anyone should choose. func (c *Client) ListBackupsRecursive(ctx context.Context, app string, s3 S3Config) ([]ListedBackup, error) { + rows, _, err := c.listBackupsAndPending(ctx, app, s3) + return rows, err +} + +// listBackupsAndPending is ListBackupsRecursive plus the bucket-relative keys of +// verification-pending uploads, which status and doctor report. +func (c *Client) listBackupsAndPending(ctx context.Context, app string, s3 S3Config) ([]ListedBackup, []string, error) { if err := c.ensureAWSCLI(ctx); err != nil { - return nil, err - } - root := fmt.Sprintf("s3://%s/%s/", s3.Base(), app) - out, err := c.exec.Run(ctx, s3.AWS("s3 ls "+ssh.ShellQuote(root)+" --recursive")) - if err != nil { - return nil, fmt.Errorf("listing backups: %w", err) + return nil, nil, err } var rows []ListedBackup - for _, line := range strings.Split(strings.TrimSpace(out), "\n") { - fields := strings.Fields(line) - if len(fields) < 4 { - continue - } - // Layout: DATE TIME SIZE KEY (SIZE is "0" or a plain integer; the - // PRE pseudo-rows only appear in non-recursive listings). - size, err := strconv.ParseInt(fields[2], 10, 64) + var pending []string + for _, loc := range s3.locations() { + keys, err := c.listTree(ctx, loc, app+"/") if err != nil { - continue + return nil, nil, fmt.Errorf("listing backups: %w", err) } - key := fields[3] - row, ok := classifyListedKey(app, key) - if !ok { - continue + for _, k := range keys { + rel, ok := loc.relToBase(k.Key) + if !ok { + continue + } + if strings.Contains(rel, "/"+incomingDir+"/") { + pending = append(pending, k.Key) + continue + } + row, ok := classifyListedKey(app, rel) + if !ok { + continue + } + row.SizeBytes = k.Size + if k.Size < artifactFloor(k.Key) { + row.Restorable = false + } + // Listing keys are relative to the BUCKET root (they already carry + // the prefix), so the full URI is rebuilt from the bucket. + row.Key = "s3://" + loc.Bucket + "/" + k.Key + row.Previous = loc.isPrevious + rows = append(rows, row) } - row.SizeBytes = size - // Recursive listing keys are relative to the BUCKET root (they - // already carry the app prefix), so rebuild the full URI from the - // base, not from the listing prefix. - row.Key = "s3://" + s3.Base() + "/" + key - rows = append(rows, row) } sort.SliceStable(rows, func(i, j int) bool { if rows[i].CreatedAt != rows[j].CreatedAt { @@ -79,11 +95,13 @@ func (c *Client) ListBackupsRecursive(ctx context.Context, app string, s3 S3Conf } return rows[i].Key < rows[j].Key }) - return rows, nil + return rows, pending, nil } -// classifyListedKey maps one recursive-ls key (relative to the bucket -// root, so "/volumes/.tar.gz") to a ListedBackup row. +// classifyListedKey maps one key relative to the base (the bucket root, or the +// configured prefix) to a ListedBackup row: "/volumes/.tar.gz", +// "/accessories//" or +// "/accessories//databases//". func classifyListedKey(app, key string) (ListedBackup, bool) { var row ListedBackup segments := strings.Split(key, "/") @@ -96,6 +114,7 @@ func classifyListedKey(app, key string) (ListedBackup, bool) { } id := m[1] row.ID = id + row.Encrypted = m[3] != "" if ts, err := time.Parse("20060102-150405", id[:15]); err == nil { row.CreatedAt = ts.UTC().Format(time.RFC3339) } @@ -115,7 +134,11 @@ func classifyListedKey(app, key string) (ListedBackup, bool) { // backup), but `backup restore` restores volumes, not this row. row.Restorable = false case "accessories": - if len(segments) != 4 { + switch { + case len(segments) == 4: + case len(segments) == 6 && segments[3] == "databases": + row.Database = decodeDBSegment(segments[4]) + default: return row, false } row.Kind = KindAccessory diff --git a/internal/backup/postgres.go b/internal/backup/postgres.go index 6850a41..174efc9 100644 --- a/internal/backup/postgres.go +++ b/internal/backup/postgres.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "regexp" + "slices" "strconv" "strings" "time" @@ -702,7 +703,6 @@ const BackupDatabasesLabel = "teploy.backup-databases" var ( pgLeftoverName = regexp.MustCompile(`^(.*)_teploy_(old|new)_[0-9a-f]{8}$`) - pgSafeDBName = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]*$`) ) // pgCoverage says what a backup of t.db leaves out of the cluster. @@ -711,6 +711,17 @@ type pgCoverage struct { Uncovered []string // in the cluster, not backed up KeptOld []string // _teploy_old_*: a previous database kept by a restore PartialNew []string // _teploy_new_*: a replacement left by a killed restore + // Missing are databases listed by name that the server does not have. + Missing []string + // PrimaryMissing says the accessory's own database does not exist (with + // AllDatabases, every database then has a series of its own). + PrimaryMissing bool + // PrimaryIsSingle says a SingleDatabase backup is of the primary database. + PrimaryIsSingle bool + // Throwaway are mysql dry-run databases a killed restore left. + Throwaway []string + // DroppedThrowaway are mongo dry-run databases this backup cleared. + DroppedThrowaway []string } // pgReadCoverage lists the cluster's databases. A "postgres" maintenance @@ -731,16 +742,18 @@ func (c *Client) pgReadCoverage(ctx context.Context, t pgTarget, listed []string } want[l] = true } + primaryFound := false for _, name := range names { switch { case name == t.db: + primaryFound = true case pgLeftoverName.MatchString(name): if strings.Contains(name, "_teploy_old_") { cov.KeptOld = append(cov.KeptOld, name) } else { cov.PartialNew = append(cov.PartialNew, name) } - case want[name] && pgSafeDBName.MatchString(name): + case want[name] && addressableDB(name): cov.Extra = append(cov.Extra, name) case name == "postgres": tables := "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind IN ('r','p') AND n.nspname NOT IN ('pg_catalog','information_schema') AND n.nspname NOT LIKE 'pg\\_%' AND NOT EXISTS (SELECT 1 FROM pg_depend d WHERE d.classid = 'pg_class'::regclass AND d.objid = c.oid AND d.deptype = 'e')" @@ -751,12 +764,18 @@ func (c *Client) pgReadCoverage(ctx context.Context, t pgTarget, listed []string cov.Uncovered = append(cov.Uncovered, name) } } - case all && pgSafeDBName.MatchString(name): + case all && addressableDB(name): cov.Extra = append(cov.Extra, name) default: cov.Uncovered = append(cov.Uncovered, name) } } + cov.PrimaryMissing = t.db != "" && !primaryFound + for _, l := range listed { + if l != "all" && !slices.Contains(names, l) && l != t.db { + cov.Missing = append(cov.Missing, l) + } + } return cov, nil } @@ -789,14 +808,11 @@ type pgBackupMeta struct { LogicalBytes int64 `json:"logical_bytes"` } -func pgSidecarKey(s3 S3Config, app, name, kind, id, ext string) string { - return fmt.Sprintf("s3://%s/%s/accessories/%s/%s/%s%s", s3.Base(), app, name, kind, id, ext) -} - -// pgRecordedSize reads the size recorded at backup time (0 when absent). -func (c *Client) pgRecordedSize(ctx context.Context, s3 S3Config, app, name, id, workDir string) int64 { +// pgRecordedSize reads the size recorded at backup time (0 when absent), for +// the primary database or one with its own series. +func (c *Client) pgRecordedSize(ctx context.Context, s3 S3Config, app, name, db, id, workDir string) int64 { local := workDir + "/meta.json" - if _, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(pgSidecarKey(s3, app, name, "meta", id, ".json")), ssh.ShellQuote(local)))); err != nil { + if !c.fetchSidecar(ctx, s3, seriesPrefix(s3, app, name, db)+"meta/"+id+".json", local) { return 0 } out, err := c.exec.Run(ctx, "cat "+ssh.ShellQuote(local)) @@ -842,7 +858,7 @@ func (c *Client) AccessoryReports(ctx context.Context) ([]AccessoryReport, error if err != nil { continue } - container := app + "-" + name + container := ContainerName(app, name) r := AccessoryReport{App: app, Accessory: name} switch det.Engine { case EnginePostgres: @@ -853,6 +869,11 @@ func (c *Client) AccessoryReports(ctx context.Context) ([]AccessoryReport, error r.Leftovers = append(append(r.Leftovers, cov.KeptOld...), cov.PartialNew...) case EngineMySQL, EngineMariaDB: r.Uncovered, r.Leftovers, r.Err = c.mysqlReadCoverage(ctx, det.Engine, container, mysqlDB(app, env)) + labels, _ := c.inspectEngineHints(ctx, container) + if listed := listedDatabases(labels); len(listed) > 0 && r.Err == nil { + cov, err := c.mysqlCoverage(ctx, det.Engine, container, mysqlDB(app, env), listed) + r.Err, r.Uncovered = err, cov.Uncovered + } case EngineMongo: r.Leftovers, r.Err = c.mongoCheckDatabases(ctx, container) default: diff --git a/internal/backup/postgres_restore_test.go b/internal/backup/postgres_restore_test.go index 03e3a57..e4e4618 100644 --- a/internal/backup/postgres_restore_test.go +++ b/internal/backup/postgres_restore_test.go @@ -45,7 +45,7 @@ func pgRestoreMock(overrides ...ssh.MockCommand) *ssh.MockExecutor { ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 2048 20260101-000000.pgdump\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: "download: done\n"}, - ssh.MockCommand{Match: "mktemp -d '/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, + ssh.MockCommand{Match: "mktemp -d '/var/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, ssh.MockCommand{Match: "gzip -t", Output: ""}, ssh.MockCommand{Match: "docker exec -i 'myapp-postgres' pg_restore -C -l", Output: pgTOC}, ssh.MockCommand{Match: "docker exec 'myapp-postgres' psql -X -qtA -v ON_ERROR_STOP=1 -U 'postgres' -d 'postgres' -c 'SELECT current_setting", Output: "170004|t|t|appowner|UTF8|en_US.utf8|en_US.utf8|1000000\n"}, @@ -59,7 +59,7 @@ func pgRestoreMock(overrides ...ssh.MockCommand) *ssh.MockExecutor { ssh.MockCommand{Match: "gunzip -c", Output: ""}, ssh.MockCommand{Match: "rm -rf", Output: ""}, ) - return ssh.NewMockExecutor("1.2.3.4", cmds...) + return newStoreMock("1.2.3.4", cmds...) } func pgClient(mock *ssh.MockExecutor) *Client { @@ -265,26 +265,26 @@ func TestLegacyTarOfReclassifiedEngineIsAFileRestore(t *testing.T) { env := map[string]string{"PG_MAJOR": "17"} listing := ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 4096 20260101-000000.tar.gz\n"} - art, err := NewClient(ssh.NewMockExecutor("h", listing), io.Discard).ResolveAccessoryArtifact(ctx, "myapp", "db", "pgvector/pgvector:pg17", "20260101-000000", env, s3) + art, err := NewClient(newStoreMock("h", listing), io.Discard).ResolveAccessoryArtifact(ctx, "myapp", "db", "pgvector/pgvector:pg17", "20260101-000000", env, s3) if err != nil || !art.FileRestore || art.Engine.Engine != EnginePostgres || art.Ext != ".tar.gz" { t.Fatalf("artifact = %+v, %v", art, err) } - native, err := NewClient(ssh.NewMockExecutor("h", ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 4096 20260101-000000.pgdump\n"}), io.Discard).ResolveAccessoryArtifact(ctx, "myapp", "db", "pgvector/pgvector:pg17", "20260101-000000", env, s3) + native, err := NewClient(newStoreMock("h", ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 4096 20260101-000000.pgdump\n"}), io.Discard).ResolveAccessoryArtifact(ctx, "myapp", "db", "pgvector/pgvector:pg17", "20260101-000000", env, s3) if err != nil || native.FileRestore { t.Fatalf("a .pgdump is an engine restore: %+v %v", native, err) } - running := ssh.NewMockExecutor("h", listing, + running := newStoreMock("h", listing, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "docker inspect -f '{{.State.Running}}'", Output: "true\n"}) if _, err := NewClient(running, io.Discard).AccessoryRestoreKeep(ctx, "myapp", "db", "pgvector/pgvector:pg17", "20260101-000000", env, s3); err == nil || !strings.Contains(err.Error(), "must be stopped") { t.Fatalf("a file restore into a running accessory must be refused, got %v", err) } - stopped := ssh.NewMockExecutor("h", listing, + stopped := newStoreMock("h", listing, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "docker inspect -f '{{.State.Running}}'", Output: "false\n"}, - ssh.MockCommand{Match: "mktemp -d '/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, + ssh.MockCommand{Match: "mktemp -d '/var/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: ""}, ssh.MockCommand{Match: "rm -rf", Output: ""}, ssh.MockCommand{Match: "mkdir -p", Output: ""}, @@ -318,7 +318,7 @@ func mysqlRestoreMock(overrides ...ssh.MockCommand) *ssh.MockExecutor { ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 2048 20260101-000000.sql.gz\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: ""}, - ssh.MockCommand{Match: "mktemp -d '/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, + ssh.MockCommand{Match: "mktemp -d '/var/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, ssh.MockCommand{Match: "umask 077; gunzip -c", Output: ""}, ssh.MockCommand{Match: "docker exec -i 'myapp-db' sh -c 'IFS= read -r teploy_credential || exit 1; MYSQL_PWD=${teploy_credential#MYSQL_PWD=}; export MYSQL_PWD; exec mysql -u root -N -B", Output: "utf8mb4|utf8mb4_0900_ai_ci|16384|0|0\n"}, ssh.MockCommand{Match: "df -Pk '/var/tmp'", Output: "Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/vda1 100000 10 90000 1% /\n"}, @@ -328,7 +328,7 @@ func mysqlRestoreMock(overrides ...ssh.MockCommand) *ssh.MockExecutor { ssh.MockCommand{Match: "rm -f ", Output: ""}, ssh.MockCommand{Match: "rm -rf", Output: ""}, ) - return ssh.NewMockExecutor("1.2.3.4", cmds...) + return newStoreMock("1.2.3.4", cmds...) } func mysqlRestore(mock *ssh.MockExecutor) (RestoreRecovery, error) { @@ -404,7 +404,7 @@ func TestMongoRestore_FailureClassification(t *testing.T) { ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 2048 20260101-000000.archive.gz\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: ""}, - ssh.MockCommand{Match: "mktemp -d '/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, + ssh.MockCommand{Match: "mktemp -d '/var/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, ssh.MockCommand{Match: "gzip -t", Output: ""}, ssh.MockCommand{Match: "umask 077; mktemp -d '/var/tmp/teploy-pre-restore.myapp@db@XXXXXX'", Output: "/var/tmp/teploy-pre-restore.g\n"}, ssh.MockCommand{Match: "docker exec 'myapp-db' mongodump", Output: ""}, @@ -412,7 +412,7 @@ func TestMongoRestore_FailureClassification(t *testing.T) { ssh.MockCommand{Match: "docker exec -i 'myapp-db' mongorestore", Output: ""}, ssh.MockCommand{Match: "rm -", Output: ""}, ) - return ssh.NewMockExecutor("h", cmds...) + return newStoreMock("h", cmds...) } run := func(m *ssh.MockExecutor) (RestoreRecovery, error) { return NewClient(m, io.Discard).AccessoryRestoreKeep(context.Background(), "myapp", "db", "mongo:7", "20260101-000000", map[string]string{"MONGO_VERSION": "7.0"}, S3Config{Bucket: "b", Region: "us-east-1"}) @@ -433,11 +433,11 @@ func TestMongoRestore_FailureClassification(t *testing.T) { // Dragonfly: when DFLY LOAD and the pre-restore reload both fail the // script's marker makes the error recovery-incomplete. func TestDragonflyDoubleFailureIsRecoveryIncomplete(t *testing.T) { - mock := ssh.NewMockExecutor("h", + mock := newStoreMock("h", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 2048 20260101-000000.rdb.gz\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: ""}, - ssh.MockCommand{Match: "mktemp -d '/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, + ssh.MockCommand{Match: "mktemp -d '/var/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, ssh.MockCommand{Match: "umask 077; mktemp -d '/var/tmp/teploy-pre-restore.myapp@cache@XXXXXX'", Output: "/var/tmp/teploy-pre-restore.d\n"}, ssh.MockCommand{Match: "set -eu", Err: errors.New("Process exited with status 3: " + recoveryIncompleteMarker + ": DFLY LOAD of the backup failed")}, ssh.MockCommand{Match: "rm -", Output: ""}, @@ -740,7 +740,7 @@ func coverageMock(databases string, overrides ...ssh.MockCommand) *ssh.MockExecu ssh.MockCommand{Match: "{ docker exec", Output: ""}, ssh.MockCommand{Match: "rm -rf", Output: ""}, ) - return ssh.NewMockExecutor("h", cmds...) + return newStoreMock("h", cmds...) } func coverageBackup(mock *ssh.MockExecutor) (BackupResult, error) { @@ -845,7 +845,7 @@ func copiesMock(databases string, overrides ...ssh.MockCommand) *ssh.MockExecuto ssh.MockCommand{Match: "for d in '/var/tmp/teploy-pre-restore.myapp@postgres@'*", Output: "done /var/tmp/teploy-pre-restore.myapp@postgres@abc\n"}, ssh.MockCommand{Match: "rm -rf", Output: ""}, ) - return ssh.NewMockExecutor("h", cmds...) + return newStoreMock("h", cmds...) } func TestReleaseRecoveryCopies(t *testing.T) { @@ -896,7 +896,7 @@ func TestReleaseRefusedWhileTheLiveDatabaseIsMissing(t *testing.T) { // copy in /var/tmp. Its restore never wrote the marker: refused without --force. func TestReleaseRefusesUnconfirmedSafetyCopyWithoutForce(t *testing.T) { path := "/var/tmp/teploy-pre-restore.myapp@db@abc" - mock := ssh.NewMockExecutor("h", + mock := newStoreMock("h", ssh.MockCommand{Match: "docker exec 'myapp-db' sh -c", Output: ""}, ssh.MockCommand{Match: "for d in '/var/tmp/teploy-pre-restore.myapp@db@'*", Output: "open " + path + "\n"}, ssh.MockCommand{Match: "rm -rf", Output: ""}, @@ -964,7 +964,7 @@ func TestRollbackRestore(t *testing.T) { t.Fatalf("plan = %+v, %v", plan, err) } - e := &pgRollbackExec{MockExecutor: ssh.NewMockExecutor("h"), present: map[string]bool{"myapp": true, old: true}} + e := &pgRollbackExec{MockExecutor: newStoreMock("h"), present: map[string]bool{"myapp": true, old: true}} displaced, err := NewClient(e, io.Discard).RollbackRestore(ctx, plan) if err != nil || !strings.HasPrefix(displaced, "myapp_teploy_new_") { t.Fatalf("rollback = %s %v", displaced, err) @@ -974,7 +974,7 @@ func TestRollbackRestore(t *testing.T) { } // After a recovery-incomplete restore the live name may be gone. - e = &pgRollbackExec{MockExecutor: ssh.NewMockExecutor("h"), present: map[string]bool{old: true}} + e = &pgRollbackExec{MockExecutor: newStoreMock("h"), present: map[string]bool{old: true}} displaced, err = NewClient(e, io.Discard).RollbackRestore(ctx, plan) if err != nil || displaced != "" || pgCalled(e.Calls, `ALTER DATABASE "myapp" RENAME TO`) { t.Fatalf("rollback with no live database = %q %v: %v", displaced, err, e.Calls) @@ -1002,7 +1002,7 @@ func TestRollbackRestore(t *testing.T) { // A name with a space must not read as an unknown state after a committed swap. func TestPgSwapStateHandlesSpacesInNames(t *testing.T) { - mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "docker exec 'c' psql", Output: "my db\nmy db_old\n"}) + mock := newStoreMock("h", ssh.MockCommand{Match: "docker exec 'c' psql", Output: "my db\nmy db_old\n"}) st, err := NewClient(mock, io.Discard).pgSwapState(context.Background(), pgTarget{container: "c", user: "u", db: "my db"}, "my db_new", "my db_old") if err != nil || !st.live || !st.previous || st.replacement { t.Fatalf("state = %+v, %v", st, err) @@ -1167,7 +1167,7 @@ func TestMySQLRestore_MarksTheSafetyCopyCompleted(t *testing.T) { func TestMySQLBackup_SecondSchemaWarns(t *testing.T) { q := mysqlEnvClient(EngineMariaDB, "myapp-db", "X", "-N", "-B") prefix := q[:strings.Index(q, "-e ")] - mock := ssh.NewMockExecutor("h", + mock := newStoreMock("h", ssh.MockCommand{Match: prefix, Output: "appdb\t0\nreports\t0\nappdb_teploy_check\t1\n"}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, @@ -1192,11 +1192,11 @@ func TestMySQLBackup_SecondSchemaWarns(t *testing.T) { // that kills mongod is recovery-incomplete, not "untouched". func TestMongoRestore_SpaceAndLiveness(t *testing.T) { base := func(o ...ssh.MockCommand) *ssh.MockExecutor { - return ssh.NewMockExecutor("h", append(o, + return newStoreMock("h", append(o, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 2048 20260101-000000.archive.gz\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: ""}, - ssh.MockCommand{Match: "mktemp -d '/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, + ssh.MockCommand{Match: "mktemp -d '/var/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, ssh.MockCommand{Match: "gzip -t", Output: ""}, ssh.MockCommand{Match: "wc -c", Output: "62000000\n"}, ssh.MockCommand{Match: "docker exec 'myapp-db' df -Pk", Output: "Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/x 1000000 10 25000 1% /data/db\n"}, @@ -1236,7 +1236,7 @@ func TestMongoRestore_SpaceAndLiveness(t *testing.T) { // One throwaway database with a short random name fits any database name in // 63 bytes; leftovers are excluded from every real restore and dropped. func TestMongoRestore_ThrowawayNameAndExclusions(t *testing.T) { - mock := ssh.NewMockExecutor("h", + mock := newStoreMock("h", ssh.MockCommand{Match: "gzip -t", Output: ""}, ssh.MockCommand{Match: "wc -c", Output: "100\n"}, ssh.MockCommand{Match: "docker exec 'c' df -Pk", Err: errors.New("exit status 1")}, @@ -1271,7 +1271,7 @@ func TestMongoRestore_ThrowawayNameAndExclusions(t *testing.T) { // whole server, so it would land in this backup silently. func TestMongoBackup_ClearsThrowawaysFirst(t *testing.T) { var order []string - mock := ssh.NewMockExecutor("h", + mock := newStoreMock("h", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-backup.abc123\n"}, ssh.MockCommand{Match: "docker exec 'myapp-db' sh -c", Output: "teploy_check_0123abcd\n"}, @@ -1302,7 +1302,7 @@ func TestMongoBackup_ClearsThrowawaysFirst(t *testing.T) { // A missing --database dump is found before anything stops. func TestResolveRestoreArtifact_ExtraDatabaseMustExist(t *testing.T) { - mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "aws s3 ls", Err: errors.New("exit status 1")}) + mock := newStoreMock("h", ssh.MockCommand{Match: "aws s3 ls", Err: errors.New("exit status 1")}) _, err := NewClient(mock, io.Discard).ResolveRestoreArtifact(context.Background(), "myapp", "db", "postgres:17", "20260101-000000", map[string]string{"PG_MAJOR": "17", "POSTGRES_DB": "appdb"}, S3Config{Bucket: "b", Region: "r"}, RestoreOptions{Database: "reports"}) if err == nil || !strings.Contains(err.Error(), "no backup") { t.Fatalf("got %v", err) @@ -1329,12 +1329,12 @@ func TestMySQLRestore_DryRunRefusesBeforeDrop(t *testing.T) { // Mongo: the archive is loaded into throwaway namespaces first. func TestMongoRestore_DryRunRefusesBeforeTouching(t *testing.T) { - mock := ssh.NewMockExecutor("h", + mock := newStoreMock("h", ssh.MockCommand{Match: "docker exec -i 'myapp-db' mongorestore --archive --gzip --drop --nsExclude", Err: errors.New("exit status 1: Failed: corruption")}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 2048 20260101-000000.archive.gz\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: ""}, - ssh.MockCommand{Match: "mktemp -d '/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, + ssh.MockCommand{Match: "mktemp -d '/var/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, ssh.MockCommand{Match: "gzip -t", Output: ""}, ssh.MockCommand{Match: "docker exec 'myapp-db' sh -c", Output: ""}, ssh.MockCommand{Match: "rm -", Output: ""}, diff --git a/internal/backup/restore.go b/internal/backup/restore.go index 67ea5c8..5bf7b2b 100644 --- a/internal/backup/restore.go +++ b/internal/backup/restore.go @@ -62,6 +62,9 @@ func (c *Client) ReleaseRecovery(ctx context.Context, r RestoreRecovery) error { type AccessoryArtifact struct { Key, Ext string Engine EngineDetection + // Loc is the location the object was found in (the current prefix or a + // previous one); its sidecars live beside it. + Loc S3Config // FileRestore is true when the artifact is a file archive (.tar.gz): // restoring it replaces files on disk, so the accessory container // must be stopped first. Every backup a pre-G6 CLI took of a @@ -97,12 +100,11 @@ func (c *Client) ResolveAccessoryArtifact(ctx context.Context, app, name, image, if err != nil { return AccessoryArtifact{}, err } - prefix := fmt.Sprintf("s3://%s/%s/accessories/%s/", s3.Base(), app, name) - key, ext, err := c.resolveAccessoryKey(ctx, s3, prefix, date, artifactExts(det.Engine)...) + key, ext, loc, err := c.findObject(ctx, s3, app+"/accessories/"+name+"/", date, artifactExts(det.Engine)...) if err != nil { return AccessoryArtifact{}, err } - return AccessoryArtifact{Key: key, Ext: ext, Engine: det, FileRestore: ext == ".tar.gz"}, nil + return AccessoryArtifact{Key: key, Ext: ext, Engine: det, Loc: loc, FileRestore: ext == ".tar.gz"}, nil } // AccessoryRestore restores an accessory backup and releases the recovery @@ -134,14 +136,20 @@ type RestoreOptions struct { // IgnoreSpaceCheck skips the free-space preflights: when the estimate // is known to be wrong, or the disk is being recovered by this restore. IgnoreSpaceCheck bool - // Database restores one extra database stored by backup_databases: - // instead of the accessory's primary (postgres only). + // Database restores one database with its own series (backup_databases:, + // policy-managed runs, adopted containers) instead of the primary. Database string + // NoPreBackup skips the automatic backup the CLI takes before a restore. + NoPreBackup bool } // AccessoryRestoreWith is AccessoryRestoreKeep with options. func (c *Client) AccessoryRestoreWith(ctx context.Context, app, name, image, date string, env map[string]string, s3 S3Config, opts RestoreOptions) (RestoreRecovery, error) { var none RestoreRecovery + // A wedged download or load must not keep the app down indefinitely; each + // engine's restore already says, on cancel, what state it left. + ctx, cancel := context.WithTimeout(ctx, c.backupDeadline()) + defer cancel() if err := c.ensureAWSCLI(ctx); err != nil { return none, err } @@ -149,7 +157,7 @@ func (c *Client) AccessoryRestoreWith(ctx context.Context, app, name, image, dat if err != nil { return none, err } - container := app + "-" + name + container := ContainerName(app, name) if art.FileRestore { out, _ := c.exec.Run(ctx, "docker inspect -f '{{.State.Running}}' "+ssh.ShellQuote(container)+" 2>/dev/null") if strings.TrimSpace(out) == "true" { @@ -160,7 +168,7 @@ func (c *Client) AccessoryRestoreWith(ctx context.Context, app, name, image, dat // All restore scratch files live under one per-invocation temp dir // (shared fixed names let concurrent or retried restores read each // other's files). On failure it is kept and named; on success removed. - tmpOut, err := c.exec.Run(ctx, "mktemp -d "+ssh.ShellQuote("/tmp/teploy-restore.XXXXXX")) + tmpOut, err := c.exec.Run(ctx, "mktemp -d "+ssh.ShellQuote(backupTmpDir()+"/teploy-restore.XXXXXX")) if err != nil { return none, fmt.Errorf("creating restore temp dir: %w", err) } @@ -173,7 +181,7 @@ func (c *Client) AccessoryRestoreWith(ctx context.Context, app, name, image, dat } artifact := tmpdir + "/restore" + art.Ext fmt.Fprintf(c.out, "Downloading %s...\n", art.Key) - if _, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 cp %s %s", ssh.ShellQuote(art.Key), ssh.ShellQuote(artifact)))); err != nil { + if err := c.fetch(ctx, art.Loc, art.Key, artifact); err != nil { return fail(none, fmt.Errorf("downloading backup: %w", err)) } @@ -193,8 +201,9 @@ func (c *Client) AccessoryRestoreWith(ctx context.Context, app, name, image, dat pgOpts := pgRestoreOpts{ignoreSpace: opts.IgnoreSpaceCheck} if opts.Database != "" && opts.Database != db { db = opts.Database + pgOpts.logicalBytes = c.pgRecordedSize(ctx, art.Loc, app, name, db, date, tmpdir) } else { - pgOpts.logicalBytes = c.pgRecordedSize(ctx, s3, app, name, date, tmpdir) + pgOpts.logicalBytes = c.pgRecordedSize(ctx, art.Loc, app, name, "", date, tmpdir) } rec, err = c.restorePostgres(ctx, newPGTarget(container, db, user, env), artifact, art.Ext, tmpdir, pgOpts) if err != nil { @@ -203,6 +212,9 @@ func (c *Client) AccessoryRestoreWith(ctx context.Context, app, name, image, dat case art.Engine.Engine == EngineMySQL || art.Engine.Engine == EngineMariaDB: m := mysqlTarget{engine: art.Engine.Engine, container: container, db: mysqlDB(app, env)} + if opts.Database != "" { + m.db = opts.Database + } if pwd := mysqlRootPassword(env); pwd != "" { if err := c.exec.Upload(ctx, strings.NewReader("MYSQL_PWD="+pwd+"\n"), credFile, "0600"); err != nil { return fail(none, err) @@ -277,29 +289,41 @@ func (c *Client) markRestoreCompleted(ctx context.Context, rec *RestoreRecovery) } } -// resolveExtraDatabase is the stored dump of a backup_databases: entry. +// resolveExtraDatabase finds one database's object: its own series, or the +// accessory's primary series when it is the primary database. func (c *Client) resolveExtraDatabase(ctx context.Context, app, name, image, date string, env map[string]string, s3 S3Config, db string) (AccessoryArtifact, error) { if err := ValidateDate(date); err != nil { return AccessoryArtifact{}, err } - if !pgSafeDBName.MatchString(db) { + if !addressableDB(db) { return AccessoryArtifact{}, fmt.Errorf("invalid database name %q", db) } det, err := c.ResolveEngine(ctx, app, name, image, env) if err != nil { return AccessoryArtifact{}, err } - if det.Engine != EnginePostgres { - return AccessoryArtifact{}, fmt.Errorf("--database applies to postgres accessories; %s is %s", name, det.Engine) + var primary string + var exts []string + switch det.Engine { + case EnginePostgres: + primary, _ = postgresDBAndUser(app, env) + exts = []string{".pgdump"} + case EngineMySQL, EngineMariaDB: + primary = mysqlDB(app, env) + exts = []string{".sql.gz"} + default: + return AccessoryArtifact{}, fmt.Errorf("--database applies to postgres and mysql/mariadb accessories; %s is %s", name, det.Engine) } - if primary, _ := postgresDBAndUser(app, env); db == primary { - return c.ResolveAccessoryArtifact(ctx, app, name, image, date, env, s3) + if db == primary && app != AdoptedApp { + if art, err := c.ResolveAccessoryArtifact(ctx, app, name, image, date, env, s3); err == nil { + return art, nil + } } - key := pgExtraKey(s3, app, name, db, date) - if _, _, err := c.resolveAccessoryKey(ctx, s3, strings.TrimSuffix(key, date+".pgdump"), date, ".pgdump"); err != nil { + key, ext, loc, err := c.findObject(ctx, s3, app+"/accessories/"+name+"/databases/"+dbSegment(db)+"/", date, exts...) + if err != nil { return AccessoryArtifact{}, err } - return AccessoryArtifact{Key: key, Ext: ".pgdump", Engine: det}, nil + return AccessoryArtifact{Key: key, Ext: ext, Engine: det, Loc: loc}, nil } // ResolveRestoreArtifact is the artifact a restore with opts would use. @@ -309,6 +333,11 @@ func (c *Client) ResolveRestoreArtifact(ctx context.Context, app, name, image, d if opts.Database != "" { return c.resolveExtraDatabase(ctx, app, name, image, date, env, s3, opts.Database) } + if app == AdoptedApp { + if det, err := c.ResolveEngine(ctx, app, name, image, env); err == nil && (det.Engine == EnginePostgres || det.Engine == EngineMySQL || det.Engine == EngineMariaDB) { + return AccessoryArtifact{}, fmt.Errorf("%s holds several databases, each with its own backups: name one with --database", name) + } + } return c.ResolveAccessoryArtifact(ctx, app, name, image, date, env, s3) } @@ -786,7 +815,7 @@ func (rc RecoveryCopies) All() []string { // scopes postgres copies to one database ("" means every database in the // cluster, as a `--database` restore keeps copies of its own). func (c *Client) FindRecoveryCopies(ctx context.Context, app, name, image, database string, env map[string]string) (RecoveryCopies, error) { - rc := RecoveryCopies{container: app + "-" + name} + rc := RecoveryCopies{container: ContainerName(app, name)} det, err := c.ResolveEngine(ctx, app, name, image, env) if err != nil { return rc, err @@ -995,6 +1024,62 @@ func mysqlEnvClient(engine, container, sql string, flags ...string) string { return "docker exec " + ssh.ShellQuote(container) + " sh -c " + ssh.ShellQuote("MYSQL_PWD=${MARIADB_ROOT_PASSWORD:-${MYSQL_ROOT_PASSWORD:-${MYSQL_PASSWORD:-}}}; export MYSQL_PWD; exec "+client+" -u root "+strings.Join(flags, " ")+" -e "+ssh.ShellQuote(sql)) } +// mysqlDumpableDB is addressableDB minus what mysqldump itself mangles: it +// backslash-escapes a quote in the schema name inside its own USE statement and +// then fails with "Unknown database" (seen on mysql 8 and mariadb-dump). Such a +// schema is reported as not backed up, loudly. +func mysqlDumpableDB(db string) bool { + return addressableDB(db) && !strings.ContainsAny(db, `'\`) +} + +// mysqlCoverage decides which of the server's schemas a backup with this listing +// stores, the shape pgReadCoverage gives postgres. A schema it cannot hand to +// the tools literally stays uncovered, whatever the listing says. +func (c *Client) mysqlCoverage(ctx context.Context, engine, container, db string, listed []string) (pgCoverage, error) { + var cov pgCoverage + others, leftovers, err := c.mysqlReadCoverage(ctx, engine, container, db) + if err != nil { + return cov, err + } + cov.Throwaway = leftovers + all := false + want := map[string]bool{} + for _, l := range listed { + if l == "all" { + all = true + } + want[l] = true + } + for _, name := range others { + if (all || want[name]) && mysqlDumpableDB(name) { + cov.Extra = append(cov.Extra, name) + } else { + cov.Uncovered = append(cov.Uncovered, name) + } + } + if db != "" { + exists, err := c.mysqlSchemaExists(ctx, engine, container, db) + if err != nil { + return cov, err + } + cov.PrimaryMissing = !exists + } + for _, l := range listed { + if l != "all" && l != db && !slices.Contains(others, l) { + cov.Missing = append(cov.Missing, l) + } + } + return cov, nil +} + +func (c *Client) mysqlSchemaExists(ctx context.Context, engine, container, db string) (bool, error) { + out, err := c.exec.Run(ctx, mysqlEnvClient(engine, container, "SELECT COUNT(*) FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = '"+strings.ReplaceAll(db, "'", "''")+"'", "-N", "-B")) + if err != nil { + return false, err + } + return strings.TrimSpace(out) != "0", nil +} + // mysqlReadCoverage lists the server's other schemas: the dump holds one, so // the rest are not backed up. A schema carrying the dry-run marker is a killed // restore's leftover, reported apart. diff --git a/internal/backup/restore_staging_test.go b/internal/backup/restore_staging_test.go index c7c706d..8bbfb70 100644 --- a/internal/backup/restore_staging_test.go +++ b/internal/backup/restore_staging_test.go @@ -16,7 +16,8 @@ import ( // live directory (via a promote step) once that succeeds. func TestRestoreVolumes_StagesBeforePromoting(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", + ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 4096 20260101-000000.tar.gz\n"}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: "download: done\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-volume-restore.abc123\n"}, @@ -127,7 +128,8 @@ func TestRestoreVolumes_StagesBeforePromoting(t *testing.T) { } func TestRestoreVolumes_ExtractionFailureLeavesLiveDirUntouched(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", + ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 4096 20260101-000000.tar.gz\n"}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: "download: done\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-volume-restore.abc123\n"}, @@ -155,7 +157,8 @@ func TestRestoreVolumes_ExtractionFailureLeavesLiveDirUntouched(t *testing.T) { // old recovery unconditionally ran `find live -mindepth 1 -delete` first, // destroying the only copy of the unmoved originals. func TestRestoreVolumes_PartialMoveAsideNeverDeletesLiveEntries(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", + ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 4096 20260101-000000.tar.gz\n"}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: "download: done\n"}, ssh.MockCommand{Match: "umask 077", Output: "/tmp/teploy-volume-restore.abc123\n"}, @@ -204,11 +207,11 @@ func TestRestoreVolumes_PartialMoveAsideNeverDeletesLiveEntries(t *testing.T) { // restorePath, so the download and restore operated on empty arguments — // every MySQL restore was deterministically broken. func TestAccessoryRestore_MySQL_DownloadsAndRestoresRealPaths(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 99 20260101-000000.sql.gz\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: "download: done\n"}, - ssh.MockCommand{Match: "mktemp -d '/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, + ssh.MockCommand{Match: "mktemp -d '/var/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, ssh.MockCommand{Match: "umask 077; gunzip -c", Output: ""}, ssh.MockCommand{Match: "docker exec", Output: ""}, ssh.MockCommand{Match: "rm -f ", Output: ""}, @@ -244,13 +247,13 @@ func TestAccessoryRestore_MySQL_DownloadsAndRestoresRealPaths(t *testing.T) { } func TestAccessoryRestore_GenericStagesBeforePromoting(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 99 20260101-000000.tar.gz\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: "download: done\n"}, ssh.MockCommand{Match: "rm -rf", Output: ""}, ssh.MockCommand{Match: "mkdir -p", Output: ""}, - ssh.MockCommand{Match: "mktemp -d '/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, + ssh.MockCommand{Match: "mktemp -d '/var/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, ssh.MockCommand{Match: "mktemp -d", Output: "/deployments/myapp/accessories/cache.restore-old.abc123\n"}, ssh.MockCommand{Match: "find ", Output: ""}, ssh.MockCommand{Match: "cp -a ", Output: ""}, @@ -294,11 +297,11 @@ func TestAccessoryRestore_GenericStagesBeforePromoting(t *testing.T) { // the current dump, and restore + restart the original when the replacement // copy fails. func TestAccessoryRestore_RedisRestartsAfterCopyFailure(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "2026-01-01 00:00:00 99 20260101-000000.rdb.gz\n"}, ssh.MockCommand{Match: "aws s3 cp", Output: "download: done\n"}, - ssh.MockCommand{Match: "mktemp -d '/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, + ssh.MockCommand{Match: "mktemp -d '/var/tmp/teploy-restore.XXXXXX'", Output: "/tmp/teploy-restore.abc123\n"}, // A40's preflight must prove appendonly=no before the script runs. ssh.MockCommand{Match: "docker exec 'myapp-redis' redis-cli --raw config get appendonly", Output: "appendonly no"}, ssh.MockCommand{Match: "set -eu", Err: fmt.Errorf("exit status 1: docker cp failed")}, diff --git a/internal/backup/retention.go b/internal/backup/retention.go index 7cdf272..b79e5e0 100644 --- a/internal/backup/retention.go +++ b/internal/backup/retention.go @@ -2,6 +2,7 @@ package backup import ( "context" + "errors" "fmt" "regexp" "sort" @@ -12,33 +13,47 @@ import ( ) // RetentionPolicy decides which backups to keep. The zero value keeps -// everything (a prune with a zero policy is a no-op). +// everything (a prune with a zero policy is a no-op). The rules are a union +// over the GOOD backups, newest first, with restic `forget` semantics: // -// KeepLast is a floor: the N most recent backups are ALWAYS kept, regardless -// of age. MaxAgeDays deletes backups older than that many days — but never -// drops below the KeepLast floor. Setting only MaxAgeDays (KeepLast 0) will -// delete every backup older than the window, so pair it with KeepLast unless -// you truly want no age-based floor. +// - KeepLast keeps the N most recent, regardless of age. +// - KeepDaily, KeepWeekly, KeepMonthly each keep the most recent backup of +// the last N calendar days, ISO weeks or months that have one (UTC, the +// timezone backup ids are written in). A period with no backup does not +// use up N, and one backup can satisfy several rules. +// - MaxAgeDays keeps everything younger than that many days (restic's +// keep-within): it is what makes "hourly for 48h, daily for a week" a +// policy. Backups older than the window go, unless another rule keeps them. +// +// With only MaxAgeDays set, everything older than the window is deleted; with +// only KeepLast, everything past the first N. Independently of any rule, the +// newest good backup is never deleted. type RetentionPolicy struct { - KeepLast int - MaxAgeDays int + KeepLast int + KeepDaily int + KeepWeekly int + KeepMonthly int + MaxAgeDays int } // IsZero reports whether the policy prunes nothing. func (p RetentionPolicy) IsZero() bool { - return p.KeepLast <= 0 && p.MaxAgeDays <= 0 + return p.KeepLast <= 0 && p.KeepDaily <= 0 && p.KeepWeekly <= 0 && p.KeepMonthly <= 0 && p.MaxAgeDays <= 0 } // backupTimestampRe pulls the 20060102-150405 timestamp out of a backup key. // It matches both naming conventions in use: the Go path's bare // ".tar.gz" and the scheduled shell path's // "-backup-.tar.gz", plus the accessory suffixes -// (.pgdump/.sql.gz/.archive.gz/.rdb.gz). +// (.pgdump/.sql.gz/.archive.gz/.rdb.gz, each optionally .age). var backupTimestampRe = regexp.MustCompile(`\d{8}-\d{6}`) type backupEntry struct { name string time time.Time + // bad marks an object below its engine's plausibility floor: an empty-output + // failure that must never count as a good copy. + bad bool } // parseBackupEntries extracts (name, timestamp) pairs from `aws s3 ls` output. @@ -52,110 +67,360 @@ func parseBackupEntries(lsOutput string) []backupEntry { continue } name := fields[len(fields)-1] - stamp := backupTimestampRe.FindString(name) - if stamp == "" { - continue - } - t, err := time.Parse("20060102-150405", stamp) - if err != nil { - continue + e, ok := newBackupEntry(name) + if ok { + entries = append(entries, e) } - entries = append(entries, backupEntry{name: name, time: t}) } return entries } -// selectForDeletion returns the backup names to delete, newest-first order -// preserved. The newest KeepLast entries are never returned (the floor). +func newBackupEntry(name string) (backupEntry, bool) { + stamp := backupTimestampRe.FindString(name) + if stamp == "" { + return backupEntry{}, false + } + t, err := time.Parse("20060102-150405", stamp) + if err != nil { + return backupEntry{}, false + } + return backupEntry{name: name, time: t}, true +} + +// selectKept returns the names the policy keeps among the good entries. +func selectKept(entries []backupEntry, p RetentionPolicy, now time.Time) map[string]bool { + keep := map[string]bool{} + var good []backupEntry + for _, e := range entries { + if !e.bad { + good = append(good, e) + } + } + sort.SliceStable(good, func(i, j int) bool { + if !good[i].time.Equal(good[j].time) { + return good[i].time.After(good[j].time) + } + return good[i].name > good[j].name + }) + if len(good) == 0 { + return keep + } + keep[good[0].name] = true // the newest good backup is never deleted + + var cutoff time.Time + if p.MaxAgeDays > 0 { + cutoff = now.Add(-time.Duration(p.MaxAgeDays) * 24 * time.Hour) + } + type rule struct { + left int + bucket func(time.Time) string + last string + } + rules := []*rule{ + {left: p.KeepDaily, bucket: func(t time.Time) string { return t.UTC().Format("2006-01-02") }}, + {left: p.KeepWeekly, bucket: func(t time.Time) string { + y, w := t.UTC().ISOWeek() + return fmt.Sprintf("%d-%02d", y, w) + }}, + {left: p.KeepMonthly, bucket: func(t time.Time) string { return t.UTC().Format("2006-01") }}, + } + for i, e := range good { + if i < p.KeepLast { + keep[e.name] = true + } + if p.MaxAgeDays > 0 && !e.time.Before(cutoff) { + keep[e.name] = true + } + for _, r := range rules { + if r.left <= 0 { + continue + } + if b := r.bucket(e.time); b != r.last { + r.last = b + r.left-- + keep[e.name] = true + } + } + } + return keep +} + +// selectForDeletion returns the backup names to delete, newest first. Entries +// the policy keeps stay; so does a bad entry newer than the newest good one (it +// may still be somebody's in-flight state, and it is reported, not trusted). If +// there is no good entry at all, nothing is deleted. func selectForDeletion(entries []backupEntry, p RetentionPolicy, now time.Time) []string { // A zero policy never deletes — guard here too, not only in PruneBackups, // so this can't silently wipe everything if called directly. if p.IsZero() { return nil } - sort.Slice(entries, func(i, j int) bool { return entries[i].time.After(entries[j].time) }) + keep := selectKept(entries, p, now) + if len(keep) == 0 { + return nil + } + sort.SliceStable(entries, func(i, j int) bool { return entries[i].time.After(entries[j].time) }) + var newestGood time.Time + for _, e := range entries { + if !e.bad { + newestGood = e.time + break + } + } + var out []string + for _, e := range entries { + if keep[e.name] { + continue + } + if e.bad && !e.time.Before(newestGood) { + continue + } + out = append(out, e.name) + } + return out +} + +// pruneObject is one object in a prune's listing, with its bucket-relative key. +type pruneObject struct { + key string + id string + entry backupEntry + loc S3Config +} + +// pruneTree is a prefix's objects sorted into what prune decides on. +type pruneTree struct { + root []pruneObject // primary objects + series map[string][]pruneObject // databases//: per-database objects + sidecars []pruneObject // globals/, meta/, databases//meta/, teploy/ siblings + incoming []pruneObject // verification-pending uploads +} - protected := p.KeepLast - if protected < 0 { - protected = 0 +func idOf(file string) string { + id, _, _ := strings.Cut(file, ".") + return id +} + +// loadPruneTree lists the prefix across the current location and owned previous +// ones. Anything it does not recognise is never touched. +func (c *Client) loadPruneTree(ctx context.Context, app, prefix string, s3 S3Config) (pruneTree, error) { + tree := pruneTree{series: map[string][]pruneObject{}} + for _, loc := range s3.locations() { + if loc.isPrevious && !loc.previousOwned { + continue + } + keys, err := c.listTree(ctx, loc, app+"/"+prefix+"/") + if err != nil { + return tree, err + } + for _, k := range keys { + rel, ok := loc.relToBase(k.Key) + if !ok { + continue + } + rest, ok := strings.CutPrefix(rel, app+"/"+prefix+"/") + if !ok { + continue + } + segs := strings.Split(rest, "/") + file := segs[len(segs)-1] + e, ok := newBackupEntry(file) + if !ok { + continue + } + e.bad = k.Size < artifactFloor(file) + obj := pruneObject{key: "s3://" + loc.Bucket + "/" + k.Key, id: idOf(file), entry: e, loc: loc} + switch { + case len(segs) == 1: + tree.root = append(tree.root, obj) + case segs[0] == incomingDir: + tree.incoming = append(tree.incoming, obj) + case len(segs) == 3 && segs[0] == "databases": + obj.entry.name = segs[1] + "/" + file + tree.series[segs[1]] = append(tree.series[segs[1]], obj) + case len(segs) == 4 && segs[0] == "databases" && segs[2] == "meta": + tree.sidecars = append(tree.sidecars, obj) + case len(segs) == 2 && (segs[0] == "globals" || segs[0] == "meta"): + tree.sidecars = append(tree.sidecars, obj) + } + } + // A volume backup's server-data sibling sits beside its prefix. + if prefix == "volumes" { + keys, err := c.listTree(ctx, loc, app+"/teploy/") + if err != nil { + return tree, err + } + for _, k := range keys { + rel, ok := loc.relToBase(k.Key) + if !ok || !strings.HasPrefix(rel, app+"/teploy/") { + continue + } + segs := strings.Split(k.Key, "/") + if e, ok := newBackupEntry(segs[len(segs)-1]); ok { + tree.sidecars = append(tree.sidecars, pruneObject{key: "s3://" + loc.Bucket + "/" + k.Key, id: idOf(segs[len(segs)-1]), entry: e, loc: loc}) + } + } + } } + return tree, nil +} - var cutoff time.Time - if p.MaxAgeDays > 0 { - cutoff = now.Add(-time.Duration(p.MaxAgeDays) * 24 * time.Hour) +func entriesOf(objs []pruneObject) []backupEntry { + out := make([]backupEntry, len(objs)) + for i, o := range objs { + out[i] = o.entry } + return out +} - var out []string - for i, e := range entries { - if i < protected { - continue // always keep the newest N +// planPrune decides what goes: +// +// - With primary objects, the primary series is selected by the policy; each +// database's objects follow their id's fate, except that a database's newest +// good object always stays (a database whose dump has been failing keeps its +// last good copy) and ids newer than the newest primary stay (a killed run's +// extras are harmless until a newer primary supersedes them). +// - Without primary objects (adopted containers), each database is its own +// series, selected by the policy. +// - Sidecars go when no surviving object has their id and they are older than +// the newest backup. +func planPrune(tree pruneTree, p RetentionPolicy, now time.Time) (primaries, sidecars []pruneObject) { + surviving := map[string]bool{} + byName := func(objs []pruneObject, del []string) []pruneObject { + names := map[string]bool{} + for _, n := range del { + names[n] = true } - if p.MaxAgeDays > 0 { - if e.time.Before(cutoff) { - out = append(out, e.name) + byEntry := map[string]pruneObject{} + for _, o := range objs { + if names[o.entry.name] { + byEntry[o.entry.name] = o + } else { + surviving[o.id] = true } + } + out := make([]pruneObject, 0, len(del)) + for _, n := range del { // the selection's order: newest first + if o, ok := byEntry[n]; ok { + out = append(out, o) + } + } + return out + } + rootEntries := entriesOf(tree.root) + primaries = byName(tree.root, selectForDeletion(rootEntries, p, now)) + rootKept := map[string]bool{} + var newestRoot time.Time + for _, o := range tree.root { + if surviving[o.id] { + rootKept[o.id] = true + } + if !o.entry.bad && o.entry.time.After(newestRoot) { + newestRoot = o.entry.time + } + } + dbs := make([]string, 0, len(tree.series)) + for db := range tree.series { + dbs = append(dbs, db) + } + sort.Strings(dbs) + for _, db := range dbs { + objs := tree.series[db] + if newestRoot.IsZero() { + primaries = append(primaries, byName(objs, selectForDeletion(entriesOf(objs), p, now))...) continue } - // Count-based only: everything beyond the floor goes. - out = append(out, e.name) + keep := selectKept(entriesOf(objs), RetentionPolicy{}, now) // the newest good one + var del []string + for _, o := range objs { + if rootKept[o.id] || keep[o.entry.name] || !o.entry.time.Before(newestRoot) { + continue + } + del = append(del, o.entry.name) + } + primaries = append(primaries, byName(objs, del)...) + } + var newest time.Time + for _, objs := range append([][]pruneObject{tree.root}, seriesValues(tree.series)...) { + for _, o := range objs { + if o.entry.time.After(newest) { + newest = o.entry.time + } + } + } + for _, o := range tree.sidecars { + if !surviving[o.id] && o.entry.time.Before(newest) { + sidecars = append(sidecars, o) + } + } + // Pending uploads a killed run left behind. Younger ones may be a live run. + for _, o := range tree.incoming { + if now.Sub(o.entry.time) > tmpLeftoverMinutes*time.Minute { + sidecars = append(sidecars, o) + } + } + return primaries, sidecars +} + +func seriesValues(m map[string][]pruneObject) [][]pruneObject { + out := make([][]pruneObject, 0, len(m)) + for _, v := range m { + out = append(out, v) } return out } -// PruneBackups deletes backups under s3:///// that fall -// outside the retention policy, returning the deleted keys' names. A zero -// policy is a no-op. Errors stop pruning and return what was deleted so far. +// CleanStaleIncoming removes pending uploads a killed run left under a prefix. +// Prune does this too; this is for a target with no retention rule. Best effort. +func (c *Client) CleanStaleIncoming(ctx context.Context, app, prefix string, s3 S3Config) { + tree, err := c.loadPruneTree(ctx, app, prefix, s3) + if err != nil { + return + } + now := time.Now().UTC() + for _, o := range tree.incoming { + if now.Sub(o.entry.time) > tmpLeftoverMinutes*time.Minute { + c.exec.Run(ctx, o.loc.AWS("s3 rm "+ssh.ShellQuote(o.key))) + } + } +} + +// PruneBackups deletes backups under s3:///// that fall +// outside the retention policy, with every sidecar they keep beside them, +// returning the deleted backups' names. A zero policy is a no-op. A namespace +// another server also writes to is refused. A refused delete (bucket locks, +// keys without delete rights) does not stop the rest: errors are returned +// together with what was deleted. func (c *Client) PruneBackups(ctx context.Context, app, prefix string, s3 S3Config, p RetentionPolicy) ([]string, error) { if p.IsZero() { return nil, nil } - - lines, err := c.ListBackups(ctx, app, prefix, s3) + if err := c.ensureAWSCLI(ctx); err != nil { + return nil, err + } + if err := c.checkNamespaceOwner(ctx, s3, app); err != nil { + return nil, err + } + tree, err := c.loadPruneTree(ctx, app, prefix, s3) if err != nil { return nil, err } - - entries := parseBackupEntries(strings.Join(lines, "\n")) - toDelete := selectForDeletion(entries, p, time.Now().UTC()) + primaries, sidecars := planPrune(tree, p, time.Now().UTC()) var deleted []string - var extras *[]string - for _, name := range toDelete { - key := fmt.Sprintf("s3://%s/%s/%s/%s", s3.Base(), app, prefix, name) - if _, err := c.exec.Run(ctx, s3.AWS("s3 rm "+ssh.ShellQuote(key))); err != nil { - return deleted, fmt.Errorf("deleting %s: %w", name, err) - } - fmt.Fprintf(c.out, "Pruned %s\n", key) - deleted = append(deleted, name) - if strings.HasPrefix(prefix, "accessories/") { - c.pruneAccessorySidecars(ctx, app, prefix, name, s3, &extras) + var errs []error + for _, o := range primaries { + if _, err := c.exec.Run(ctx, o.loc.AWS("s3 rm "+ssh.ShellQuote(o.key))); err != nil { + errs = append(errs, fmt.Errorf("deleting %s: %w", o.key, err)) + continue } + fmt.Fprintf(c.out, "Pruned %s\n", o.key) + deleted = append(deleted, o.entry.name) } - return deleted, nil -} - -// pruneAccessorySidecars removes what a pruned accessory backup keeps -// beside itself: postgres globals and size record, and any backup_databases -// dumps. Absent for other engines; deleting a missing key succeeds. -func (c *Client) pruneAccessorySidecars(ctx context.Context, app, prefix, file string, s3 S3Config, extras **[]string) { - id, _, _ := strings.Cut(file, ".") - base := fmt.Sprintf("s3://%s/%s/%s/", s3.Base(), app, prefix) - if *extras == nil { - names := []string{} - if out, err := c.exec.Run(ctx, s3.AWS("s3 ls "+ssh.ShellQuote(base+"databases/"))); err == nil { - for _, line := range strings.Split(out, "\n") { - if f := strings.Fields(line); len(f) == 2 && f[0] == "PRE" { - names = append(names, strings.TrimSuffix(f[1], "/")) - } - } + for _, o := range sidecars { + if _, err := c.exec.Run(ctx, o.loc.AWS("s3 rm "+ssh.ShellQuote(o.key))); err != nil { + errs = append(errs, fmt.Errorf("deleting %s: %w", o.key, err)) } - *extras = &names - } - keys := []string{base + "globals/" + id + ".sql.gz", base + "meta/" + id + ".json"} - for _, db := range **extras { - keys = append(keys, base+"databases/"+db+"/"+id+".pgdump") - } - for _, key := range keys { - c.exec.Run(ctx, s3.AWS("s3 rm "+ssh.ShellQuote(key))) } + return deleted, errors.Join(errs...) } diff --git a/internal/backup/retention_test.go b/internal/backup/retention_test.go index a801222..fe8ca60 100644 --- a/internal/backup/retention_test.go +++ b/internal/backup/retention_test.go @@ -79,12 +79,12 @@ func TestSelectForDeletion(t *testing.T) { func TestPruneBackups(t *testing.T) { ls := strings.Join([]string{ - "2024-01-01 00:00:00 100 20240101-000000.tar.gz", - "2024-01-02 00:00:00 100 20240102-000000.tar.gz", - "2024-01-03 00:00:00 100 20240103-000000.tar.gz", + "2024-01-01 00:00:00 100 myapp/volumes/20240101-000000.tar.gz", + "2024-01-02 00:00:00 100 myapp/volumes/20240102-000000.tar.gz", + "2024-01-03 00:00:00 100 myapp/volumes/20240103-000000.tar.gz", }, "\n") - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: ls}, ssh.MockCommand{Match: "aws s3 rm", Output: "delete: done\n"}, @@ -121,7 +121,7 @@ func TestPruneBackups(t *testing.T) { } func TestPruneBackupsZeroPolicyNoop(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4") // no commands registered — any call would error + mock := newStoreMock("1.2.3.4") // no commands registered — any call would error var buf bytes.Buffer client := NewClient(mock, &buf) deleted, err := client.PruneBackups(context.Background(), "myapp", "volumes", @@ -136,9 +136,9 @@ func TestPruneBackupsZeroPolicyNoop(t *testing.T) { // Pruning an accessory backup also removes its postgres globals sidecar. func TestPruneRemovesGlobalsSidecar(t *testing.T) { - mock := ssh.NewMockExecutor("h", + mock := newStoreMock("h", ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, - ssh.MockCommand{Match: "aws s3 ls", Output: "2024-01-01 00:00:00 100 20240101-000000.pgdump\n2024-01-02 00:00:00 100 20240102-000000.pgdump\n"}, + ssh.MockCommand{Match: "aws s3 ls", Output: "2024-01-01 00:00:00 1000 myapp/accessories/db/20240101-000000.pgdump\n2024-01-02 00:00:00 1000 myapp/accessories/db/20240102-000000.pgdump\n2024-01-01 00:00:00 500 myapp/accessories/db/globals/20240101-000000.sql.gz\n"}, ssh.MockCommand{Match: "aws s3 rm", Output: ""}, ) deleted, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "myapp", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1}) diff --git a/internal/backup/scheduler_test.go b/internal/backup/scheduler_test.go index 8c49e51..a040fa8 100644 --- a/internal/backup/scheduler_test.go +++ b/internal/backup/scheduler_test.go @@ -39,7 +39,7 @@ func TestRequireScheduler(t *testing.T) { {name: "running", out: "daemon=yes\nenabled=enabled\ncrontab=yes\nteploy_entries=0\n"}, } { t.Run(tc.name, func(t *testing.T) { - mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "sched_daemon=no", Output: tc.out, Err: tc.err}) + mock := newStoreMock("h", ssh.MockCommand{Match: "sched_daemon=no", Output: tc.out, Err: tc.err}) err := RequireScheduler(ctx, mock) if tc.wantSub == "" { if err != nil { diff --git a/internal/backup/store.go b/internal/backup/store.go new file mode 100644 index 0000000..222cb3a --- /dev/null +++ b/internal/backup/store.go @@ -0,0 +1,726 @@ +package backup + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "path" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "filippo.io/age" + "github.com/useteploy/teploy/internal/ssh" +) + +// The object store layer (Slice 4). Every backup artifact and sidecar goes +// through put and comes back through fetch, so the three guarantees below hold +// for every engine, sidecar and DR member alike: +// +// - Encryption: when the policy names an owner recovery key, the object is +// age-encrypted to that key AND the server's own identity, the ciphertext +// is proven to decrypt back to the plaintext before anything leaves the +// server, and only the ciphertext is uploaded (.age suffix). A failure to +// encrypt fails the backup; it never falls back to plaintext. +// +// - A final key means a verified object: the upload goes to a .incoming/ key +// beside the final one, its size and checksum are checked there, and only +// then is it copied server-side to its final name. A killed run therefore +// can never leave a partial or unverified object where list, latest, +// restore or prune would take it for a backup. Leftover .incoming/ objects +// are invisible to those, and pruned by the next run. +// +// - Commit order: a backup's sidecars and extra databases are stored BEFORE +// its primary object, so a primary implies everything that belongs to it. + +// ageSuffix marks an object encrypted with age. +const ageSuffix = ".age" + +// incomingDir holds uploads until they are verified. +const incomingDir = ".incoming" + +// readBackMax is the largest object whose bytes are read back and hashed after +// upload; above it the stored size and the checksum recorded at upload stand +// in (plus the aws CLI's own transfer checksums). Bandwidth, not safety. +const readBackMax = 256 << 20 + +// agelessFloorOverhead is what age adds to a tiny object (header with two +// X25519 stanzas plus the payload nonce and tag), so a floor measured on +// plaintext is not silently failed or passed by ciphertext. +const ageFloorOverhead = 200 + +// recipientRE is an X25519 age recipient: the only form teploy generates. +var recipientRE = regexp.MustCompile(`^age1[0-9a-z]{58}$`) + +// ValidateRecipient checks an owner recovery public key. +func ValidateRecipient(r string) error { + if !recipientRE.MatchString(r) { + return fmt.Errorf("recipient must be an age public key (age1..., from `teploy backup keygen`)") + } + if _, err := age.ParseX25519Recipient(r); err != nil { + return fmt.Errorf("recipient is not a valid age public key: %w", err) + } + return nil +} + +// Encrypted reports whether objects written with this config are encrypted. +func (s3 S3Config) Encrypted() bool { return s3.OwnerRecipient != "" } + +// objectExtra is what put reports about a stored object. +type storedObject struct { + Key string + Size int64 + SHA256 string + Encrypted bool +} + +// serverRecipient returns the server's own age public key, creating the +// identity if the server has none (shared with stored target credentials and +// the secret store, so a server never grows two). +func (c *Client) serverRecipient(ctx context.Context) (string, error) { + if c.serverRecip != "" { + return c.serverRecip, nil + } + if out, err := c.exec.Run(ctx, "command -v age >/dev/null 2>&1 && command -v age-keygen >/dev/null 2>&1 && echo ok"); err != nil || strings.TrimSpace(out) != "ok" { + return "", fmt.Errorf("the age tool is not installed on this server, so nothing can be encrypted (run `teploy setup`, or apt-get install age)") + } + id, err := ensureAgeIdentity(ctx, c.exec) + if err != nil { + return "", err + } + out, err := c.exec.Run(ctx, "age-keygen -y "+ssh.ShellQuote(id)) + if err != nil { + return "", fmt.Errorf("reading the server's age identity %s: %w", id, err) + } + r := strings.TrimSpace(out) + if !recipientRE.MatchString(r) { + return "", fmt.Errorf("the server's age identity %s did not yield a usable public key", id) + } + c.serverRecip = r + return r, nil +} + +// PrepareEncryption is prepareEncryption for a run's own preflight. +func (c *Client) PrepareEncryption(ctx context.Context, s3 S3Config) error { + return c.prepareEncryption(ctx, s3) +} + +// prepareEncryption fails BEFORE any dump runs when the policy asks for +// encryption this server cannot do. +func (c *Client) prepareEncryption(ctx context.Context, s3 S3Config) error { + if !s3.Encrypted() { + return nil + } + if err := ValidateRecipient(s3.OwnerRecipient); err != nil { + return err + } + _, err := c.serverRecipient(ctx) + return err +} + +// encryptLocal encrypts local in place to local.age, proves the ciphertext +// decrypts back to the plaintext with the server's identity, and removes the +// plaintext. Files, not pipes: a pipeline reports only its last command's +// status, and a truncated stream would upload as a "backup". +func (c *Client) encryptLocal(ctx context.Context, s3 S3Config, local string) (string, error) { + server, err := c.serverRecipient(ctx) + if err != nil { + return "", err + } + q := ssh.ShellQuote + enc := local + ageSuffix + id := ageIdentityShell + cmd := fmt.Sprintf("age -r %s -r %s -o %s %s", q(s3.OwnerRecipient), q(server), q(enc), q(local)) + if _, err := c.exec.Run(ctx, cmd); err != nil { + c.exec.Run(context.WithoutCancel(ctx), "rm -f "+q(enc)) + return "", fmt.Errorf("encrypting the backup (nothing was uploaded): %w", err) + } + // cmp reads the decrypted stream to its end and compares it with the + // plaintext: a decrypt failure shows up as a short stream, so losing age's + // exit status in this pipe cannot hide anything. + check := fmt.Sprintf(`age -d -i "%s" %s | cmp -s - %s`, id, q(enc), q(local)) + if _, err := c.exec.Run(ctx, check); err != nil { + c.exec.Run(context.WithoutCancel(ctx), "rm -f "+q(enc)) + return "", fmt.Errorf("the encrypted backup did not decrypt back to its plaintext with the server's identity (nothing was uploaded): %w", err) + } + if _, err := c.exec.Run(ctx, "rm -f "+q(local)); err != nil { + return "", fmt.Errorf("removing the plaintext after encrypting: %w", err) + } + return enc, nil +} + +// ageIdentityShell expands to the server's identity path (same try-order as +// ensureAgeIdentity). It is a command substitution, for use inside "...". +const ageIdentityShell = `$(if [ -r /etc/teploy/age-key ]; then printf '%s' /etc/teploy/age-key; else printf '%s' /deployments/.age-key; fi)` + +// localFacts measures a file: size and SHA-256, in one command. +func (c *Client) localFacts(ctx context.Context, local string) (int64, string, error) { + q := ssh.ShellQuote(local) + out, err := c.exec.Run(ctx, "sha256sum "+q+" && wc -c < "+q) + if err != nil { + return 0, "", fmt.Errorf("measuring %s: %w", path.Base(local), err) + } + f := strings.Fields(out) + if len(f) < 3 { + return 0, "", fmt.Errorf("measuring %s: unreadable output", path.Base(local)) + } + size, err := strconv.ParseInt(f[len(f)-1], 10, 64) + if err != nil { + return 0, "", fmt.Errorf("measuring %s: unreadable size %q", path.Base(local), f[len(f)-1]) + } + return size, f[0], nil +} + +// incomingKey is the verification-pending name of a final key. +func incomingKey(key string) string { + dir, file := path.Split(key) + return dir + incomingDir + "/" + file +} + +// objectHead is the stored object's size and recorded checksum. +type objectHead struct { + Size int64 + SHA256 string +} + +// head reads an object's size and recorded checksum. +func (c *Client) head(ctx context.Context, s3 S3Config, key string) (objectHead, error) { + bucket, k, err := splitS3URI(key) + if err != nil { + return objectHead{}, err + } + res := ssh.RunDetailed(ctx, c.exec, s3.AWS("s3api head-object --bucket "+ssh.ShellQuote(bucket)+" --key "+ssh.ShellQuote(k))) + if res.Failed() { + return objectHead{}, fmt.Errorf("reading %s: %s", key, oneLine(res.ExitErrorText())) + } + var h struct { + ContentLength int64 `json:"ContentLength"` + Metadata map[string]string `json:"Metadata"` + } + if err := json.Unmarshal(res.Stdout, &h); err != nil { + return objectHead{}, fmt.Errorf("reading %s: unexpected reply: %w", key, err) + } + return objectHead{Size: h.ContentLength, SHA256: h.Metadata["sha256"]}, nil +} + +// splitS3URI splits s3://bucket/key. +func splitS3URI(uri string) (bucket, key string, err error) { + rest, ok := strings.CutPrefix(uri, "s3://") + if !ok { + return "", "", fmt.Errorf("not an s3:// key: %q", uri) + } + bucket, key, ok = strings.Cut(rest, "/") + if !ok || bucket == "" || key == "" { + return "", "", fmt.Errorf("not an s3:// object key: %q", uri) + } + return bucket, key, nil +} + +func oneLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexAny(s, "\r\n"); i >= 0 { + s = s[:i] + } + if len(s) > 200 { + s = s[:200] + "..." + } + return s +} + +// put stores local at key (key.age when encrypting) with the guarantees in +// this file's header. The plaintext is gone from disk afterwards only when +// encrypting; the caller's workspace owns the rest. +func (c *Client) put(ctx context.Context, s3 S3Config, local, key string) (storedObject, error) { + var none storedObject + src, final, encrypted := local, key, false + if s3.Encrypted() { + enc, err := c.encryptLocal(ctx, s3, local) + if err != nil { + return none, err + } + src, final, encrypted = enc, key+ageSuffix, true + } + size, sum, err := c.localFacts(ctx, src) + if err != nil { + return none, err + } + pending := incomingKey(final) + q := ssh.ShellQuote + if _, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 cp %s %s --metadata %s", q(src), q(pending), q("sha256="+sum)))); err != nil { + return none, fmt.Errorf("uploading: %w", err) + } + removePending := func() { + rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) + defer cancel() + c.exec.Run(rctx, s3.AWS("s3 rm "+q(pending))) + } + check := func(k string) error { + h, err := c.head(ctx, s3, k) + if err != nil { + return err + } + if h.Size != size { + return fmt.Errorf("stored %s is %d bytes, %d were sent", k, h.Size, size) + } + if h.SHA256 != "" && h.SHA256 != sum { + return fmt.Errorf("stored %s records checksum %s, %s was sent", k, h.SHA256, sum) + } + return nil + } + if err := check(pending); err != nil { + removePending() + return none, fmt.Errorf("verifying the upload: %w", err) + } + if size <= readBackMax { + // A pipe, but compared by value: anything short of the exact bytes + // (a failed read included) produces a different hash. + out, err := c.exec.Run(ctx, "{ "+s3.AWS("s3 cp "+q(pending)+" -")+"; } | sha256sum") + if err != nil { + removePending() + return none, fmt.Errorf("reading the upload back: %w", err) + } + if f := strings.Fields(lastLine(out)); len(f) == 0 || f[0] != sum { + removePending() + return none, fmt.Errorf("the uploaded object's bytes do not match what was sent (checksum mismatch)") + } + } + if _, err := c.exec.Run(ctx, s3.AWS("s3 cp "+q(pending)+" "+q(final))); err != nil { + removePending() + return none, fmt.Errorf("publishing the verified upload: %w", err) + } + if err := check(final); err != nil { + rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) + defer cancel() + c.exec.Run(rctx, s3.AWS("s3 rm "+q(final))) + removePending() + return none, fmt.Errorf("verifying the published object: %w", err) + } + removePending() + return storedObject{Key: final, Size: size, SHA256: sum, Encrypted: encrypted}, nil +} + +func lastLine(s string) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + return lines[len(lines)-1] +} + +// fetch downloads key to dest, checks it against the checksum recorded at +// upload (objects from older CLIs have none), and decrypts a .age object with +// the server's identity. dest never holds ciphertext. +func (c *Client) fetch(ctx context.Context, s3 S3Config, key, dest string) error { + q := ssh.ShellQuote + encrypted := strings.HasSuffix(key, ageSuffix) + raw := dest + if encrypted { + raw = dest + ageSuffix + } + h, headErr := c.head(ctx, s3, key) + if headErr == nil { + // The download sits beside its plaintext until decrypted, on a disk a + // failing restore may already have short of room. + need := h.Size + if encrypted { + need *= 2 + } + if free, known := c.freeBytes(ctx, "", path.Dir(dest)); known && free < need { + return fmt.Errorf("not enough free space in %s for %s: %d bytes free, %d needed (nothing was changed)", path.Dir(dest), key, free, need) + } + } + if _, err := c.exec.Run(ctx, s3.AWS("s3 cp "+q(key)+" "+q(raw))); err != nil { + return fmt.Errorf("downloading %s: %w", key, err) + } + if headErr == nil && h.SHA256 != "" { + if _, sum, err := c.localFacts(ctx, raw); err == nil && sum != h.SHA256 { + c.exec.Run(context.WithoutCancel(ctx), "rm -f "+q(raw)) + return fmt.Errorf("%s is corrupt: its checksum does not match the one recorded when it was stored", key) + } + } + if !encrypted { + return nil + } + ids := `-i "` + ageIdentityShell + `"` + if s3.ExtraIdentity != "" { + ids += " -i " + q(s3.ExtraIdentity) + } + if _, err := c.exec.Run(ctx, "umask 077; age -d "+ids+" -o "+q(dest)+" "+q(raw)); err != nil { + c.exec.Run(context.WithoutCancel(ctx), "rm -f "+q(raw)+" "+q(dest)) + if errors.Is(ctx.Err(), context.Canceled) { + return ctx.Err() + } + return fmt.Errorf("decrypting %s with this server's identity failed (encrypted to other recipients, or corrupt; on another machine use `teploy backup fetch --identity`): %w", key, err) + } + c.exec.Run(ctx, "rm -f "+q(raw)) + return nil +} + +// fetchSidecar is fetch for a sidecar that may be plaintext (older CLIs) or +// encrypted, and may not exist at all. +func (c *Client) fetchSidecar(ctx context.Context, s3 S3Config, key, dest string) bool { + if c.fetch(ctx, s3, key, dest) == nil { + return true + } + return c.fetch(ctx, s3, key+ageSuffix, dest) == nil +} + +// ---- layout ---- + +// AdoptedApp is the key namespace of adopted containers (policy `adopted:`): +// containers with no teploy labels, addressed by their exact docker name. The +// underscore cannot occur in an app name, so it cannot collide with one. S3: +// s3:///_adopted/accessories//... +const AdoptedApp = "_adopted" + +// FleetApp is the pseudo-app whose event log holds `backup run --all` summaries. +const FleetApp = "_fleet" + +// ContainerName is the docker name of an accessory: -, or the bare +// container name in the adopted namespace. +func ContainerName(app, name string) string { + if app == AdoptedApp { + return name + } + return app + "-" + name +} + +// dbSegmentSafe are the bytes a database name keeps in an S3 key. +func dbSegmentSafe(b byte) bool { + return b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' || b >= '0' && b <= '9' || b == '_' || b == '-' || b == '.' +} + +// dbSegment encodes a database name as one S3 key segment. Schemas can hold any +// byte: "/" must not add a level, spaces would break `aws s3 ls` parsing, and +// ".." must not look like a path. +func dbSegment(db string) string { + var b strings.Builder + for i := 0; i < len(db); i++ { + c := db[i] + if dbSegmentSafe(c) && !(c == '.' && (i == 0 || strings.HasPrefix(db[i:], ".."))) { + b.WriteByte(c) + } else { + fmt.Fprintf(&b, "%%%02X", c) + } + } + return b.String() +} + +// decodeDBSegment reverses dbSegment. +func decodeDBSegment(seg string) string { + if d, err := url.PathUnescape(seg); err == nil { + return d + } + return seg +} + +// addressableDB reports whether a database name can be handed to the engine +// tools as one literal shell-quoted argument. pg_dump and psql read a name with +// "=" as a connection string, and every tool reads a leading "-" as an option; +// control characters cannot survive line-based listings. Everything else, +// including quotes, spaces and $(...), is safe once shell-quoted. A name that +// fails this is reported as not backed up, never skipped silently. +func addressableDB(db string) bool { + if db == "" || len(db) > 64 || db[0] == '-' || strings.Contains(db, "=") { + return false + } + for _, r := range db { + if r < 0x20 || r == 0x7f { + return false + } + } + return true +} + +// seriesPrefix is where one accessory's backups live: the primary series, or +// one database's series below it. +func seriesPrefix(s3 S3Config, app, name, db string) string { + p := fmt.Sprintf("s3://%s/%s/accessories/%s/", s3.Base(), app, name) + if db != "" { + p += "databases/" + dbSegment(db) + "/" + } + return p +} + +// ---- listing ---- + +// storedKey is one object in a recursive listing. +type storedKey struct { + Key string // relative to the bucket root + Size int64 +} + +// listTree lists every object under s3:/// recursively. A prefix with +// no objects is an empty result: `aws s3 ls` exits 1 with no output for it, +// which every other failure mode does not. +func (c *Client) listTree(ctx context.Context, s3 S3Config, rel string) ([]storedKey, error) { + root := "s3://" + s3.Base() + "/" + rel + res := ssh.RunDetailedLimit(ctx, c.exec, s3.AWS("s3 ls "+ssh.ShellQuote(root)+" --recursive"), 64<<20) + if res.Err != nil { + return nil, fmt.Errorf("listing %s: %w", root, res.Err) + } + if res.Truncated { + return nil, fmt.Errorf("listing %s: too many objects to read safely", root) + } + if res.ExitCode != 0 { + if res.ExitCode == 1 && strings.TrimSpace(string(res.Stdout)) == "" && strings.TrimSpace(string(res.Stderr)) == "" { + return nil, nil + } + return nil, fmt.Errorf("listing %s: %s", root, oneLine(res.ExitErrorText())) + } + var out []storedKey + for _, line := range strings.Split(string(res.Stdout), "\n") { + f := strings.Fields(line) + if len(f) < 4 { + continue + } + size, err := strconv.ParseInt(f[2], 10, 64) + if err != nil { + continue + } + out = append(out, storedKey{Key: f[3], Size: size}) + } + return out, nil +} + +// locations lists where this config's backups may be: the current prefix, then +// any previous prefix of the same bucket. +func (s3 S3Config) locations() []S3Config { + out := []S3Config{s3} + for _, p := range s3.PreviousPrefixes { + loc := s3 + loc.Prefix = p.Prefix + loc.PreviousPrefixes = nil + loc.isPrevious = true + loc.previousOwned = p.Owned + out = append(out, loc) + } + return out +} + +// relToBase strips the configured prefix from a bucket-relative key. A key +// outside the prefix is not this config's. +func (s3 S3Config) relToBase(key string) (string, bool) { + if s3.Prefix == "" { + return key, true + } + return strings.CutPrefix(key, s3.Prefix+"/") +} + +// findObject finds backup id in a series across the current and previous +// locations, trying each extension and its encrypted form. It returns the +// location it was found in (sidecars live beside it). +func (c *Client) findObject(ctx context.Context, s3 S3Config, rel, id string, exts ...string) (key, ext string, loc S3Config, err error) { + var listErr error + for _, l := range s3.locations() { + prefix := "s3://" + l.Base() + "/" + rel + out, lsErr := c.exec.Run(ctx, l.AWS("s3 ls "+ssh.ShellQuote(prefix+id+"."))) + if lsErr != nil { + listErr = lsErr + continue + } + names := map[string]bool{} + for _, line := range strings.Split(out, "\n") { + if f := strings.Fields(line); len(f) > 0 { + names[f[len(f)-1]] = true + } + } + for _, e := range exts { + if names[id+e] { + return prefix + id + e, e, l, nil + } + if names[id+e+ageSuffix] { + return prefix + id + e + ageSuffix, e, l, nil + } + } + } + detail := "" + if listErr != nil { + detail = fmt.Sprintf(" (listing: %v)", listErr) + } + return "", "", s3, fmt.Errorf("no backup %s found under s3://%s/%s (looked for %s)%s", id, s3.Base(), rel, strings.Join(exts, ", "), detail) +} + +// artifactFloor is the smallest object of this name that can be a real backup. +// Anything below it is an empty-output failure (U11: fourteen 20-byte files) +// and never counts as a good copy. +func artifactFloor(name string) int64 { + base := strings.TrimSuffix(name, ageSuffix) + floor := int64(0) + switch { + case strings.HasSuffix(base, ".pgdump"): + floor = minPostgresDump + case strings.HasSuffix(base, ".sql.gz"): + floor = minMySQLDump + case strings.HasSuffix(base, ".archive.gz"): + floor = minMongoDump + case strings.HasSuffix(base, ".rdb.gz"): + floor = minRDBSnapshot + case strings.HasSuffix(base, ".tar.gz"): + floor = minTarArchive + } + if base != name { + floor += ageFloorOverhead + } + return floor +} + +// ---- janitor ---- + +// tmpLeftoverAge is how old an unattended workspace or .incoming object must be +// before it is treated as a killed run's leftover: longer than the deadline of +// any one target, shorter than a disk can afford to wait. +const tmpLeftoverMinutes = 420 + +// workspaceGlobs are what the run cleans up: private run workspaces a killed +// backup can leave (dumps, possibly a credential file), and a recovery key a +// killed `dr restore --identity` left on this server. Only ever this: a killed +// restore's own files can be the only copy of something. +const workspaceGlobs = "/var/tmp/teploy-backup.* /tmp/teploy-backup.* /var/tmp/teploy-dr-identity.*" + +// leftoverGlobs are what doctor and status report: the above, plus a kept or +// killed restore's and verify's scratch files. +const leftoverGlobs = workspaceGlobs + " /var/tmp/teploy-restore.* /tmp/teploy-restore.* /tmp/teploy-volume-restore.* /var/tmp/teploy-verify.*" + +// CleanStaleWorkspaces removes run workspaces older than the leftover age and +// returns what it removed. Only called while holding the run lock, so a live +// `backup run --all` cannot be hit; the age protects a concurrent manual one. +func (c *Client) CleanStaleWorkspaces(ctx context.Context) []string { + out, err := c.exec.Run(ctx, fmt.Sprintf(`for d in %s; do [ -d "$d" ] || continue; if [ -n "$(find "$d" -maxdepth 0 -mmin +%d 2>/dev/null)" ]; then rm -rf -- "$d" && printf '%%s\n' "$d"; fi; done; true`, workspaceGlobs, tmpLeftoverMinutes)) + if err != nil { + return nil + } + return splitLines(out) +} + +// LeftoverWorkspacesProbe lists workspaces past the leftover age, for doctor. +func LeftoverWorkspacesProbe() string { + return fmt.Sprintf(`for d in %s; do [ -d "$d" ] || continue; if [ -n "$(find "$d" -maxdepth 0 -mmin +%d 2>/dev/null)" ]; then printf '%%s\n' "$d"; fi; done; docker ps -a --filter name=teploy-backup-verify- --format 'container {{.Names}}' 2>/dev/null; true`, leftoverGlobs, tmpLeftoverMinutes) +} + +// AbortStaleUploads aborts multipart uploads a killed run left under the base +// (they are invisible in listings and bill storage). Best effort; returns how +// many were aborted. +func (c *Client) AbortStaleUploads(ctx context.Context, s3 S3Config, now time.Time) int { + q := ssh.ShellQuote + args := "s3api list-multipart-uploads --bucket " + q(s3.Bucket) + " --output json" + if s3.Prefix != "" { + args += " --prefix " + q(s3.Prefix+"/") + } + res := ssh.RunDetailed(ctx, c.exec, s3.AWS(args)) + if res.Failed() { + return 0 + } + var list struct { + Uploads []struct { + Key string `json:"Key"` + UploadID string `json:"UploadId"` + Initiated string `json:"Initiated"` + } `json:"Uploads"` + } + if json.Unmarshal(res.Stdout, &list) != nil { + return 0 + } + sort.Slice(list.Uploads, func(i, j int) bool { return list.Uploads[i].Key < list.Uploads[j].Key }) + aborted := 0 + for _, u := range list.Uploads { + t, err := time.Parse(time.RFC3339, u.Initiated) + if err != nil || now.Sub(t) < tmpLeftoverMinutes*time.Minute { + continue + } + if _, err := c.exec.Run(ctx, s3.AWS("s3api abort-multipart-upload --bucket "+q(s3.Bucket)+" --key "+q(u.Key)+" --upload-id "+q(u.UploadID))); err == nil { + aborted++ + } + } + return aborted +} + +// ---- namespace ownership ---- + +// ownersDir holds one marker per server that has written into an app namespace: +// //.owners/. Prune refuses a namespace with another +// server's marker, so a shared bucket or a reused prefix can never make one +// server delete another's backups. +const ownersDir = ".owners" + +var ownerMarkerRE = regexp.MustCompile(`^[0-9a-f]{16}$`) + +// fingerprint identifies this server: a hash of its machine id, else its age +// recipient. Empty when neither can be read. +func (c *Client) fingerprint(ctx context.Context) string { + if c.serverFP != "" { + return c.serverFP + } + out, err := c.exec.Run(ctx, `{ cat /etc/machine-id 2>/dev/null || hostname; } | sha256sum`) + if err != nil { + return "" + } + f := strings.Fields(out) + if len(f) == 0 || len(f[0]) < 16 { + return "" + } + if !ownerMarkerRE.MatchString(f[0][:16]) { + return "" + } + c.serverFP = f[0][:16] + return c.serverFP +} + +func namespaceOwnersPrefix(s3 S3Config, app string) string { + return fmt.Sprintf("s3://%s/%s/%s/", s3.Base(), app, ownersDir) +} + +// namespaceOwners lists the servers that have marked this namespace. +func (c *Client) namespaceOwners(ctx context.Context, s3 S3Config, app string) []string { + out, err := c.exec.Run(ctx, s3.AWS("s3 ls "+ssh.ShellQuote(namespaceOwnersPrefix(s3, app)))) + if err != nil { + return nil + } + var owners []string + for _, line := range strings.Split(out, "\n") { + f := strings.Fields(line) + if len(f) > 0 && ownerMarkerRE.MatchString(f[len(f)-1]) { + owners = append(owners, f[len(f)-1]) + } + } + return owners +} + +// markNamespaceOwner records this server as a writer of the namespace. Best +// effort; called after a successful backup. +func (c *Client) markNamespaceOwner(ctx context.Context, s3 S3Config, app string) { + fp := c.fingerprint(ctx) + if fp == "" { + return + } + for _, o := range c.namespaceOwners(ctx, s3, app) { + if o == fp { + return + } + } + key := namespaceOwnersPrefix(s3, app) + fp + c.exec.Run(ctx, "printf '%s\\n' "+ssh.ShellQuote(time.Now().UTC().Format(time.RFC3339))+" | "+s3.AWS("s3 cp - "+ssh.ShellQuote(key))) +} + +// ErrSharedNamespace marks a prune refused because another server writes there. +var ErrSharedNamespace = errors.New("shared backup namespace") + +// checkNamespaceOwner refuses when another server has marked this namespace. +func (c *Client) checkNamespaceOwner(ctx context.Context, s3 S3Config, app string) error { + owners := c.namespaceOwners(ctx, s3, app) + if len(owners) == 0 { + return nil + } + fp := c.fingerprint(ctx) + var others []string + for _, o := range owners { + if o != fp { + others = append(others, o) + } + } + if len(others) == 0 { + return nil + } + return fmt.Errorf("%w: another server (%s) also stores %s backups under s3://%s, so pruning here could delete its copies; nothing was pruned. Give each server its own prefix (`teploy server backup-target set --prefix `)", ErrSharedNamespace, strings.Join(others, ", "), app, s3.Base()) +} diff --git a/internal/backup/store_test.go b/internal/backup/store_test.go new file mode 100644 index 0000000..395bc7a --- /dev/null +++ b/internal/backup/store_test.go @@ -0,0 +1,754 @@ +package backup + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/ssh" +) + +func entriesAt(stamps ...string) []backupEntry { + out := make([]backupEntry, 0, len(stamps)) + for _, s := range stamps { + e, ok := newBackupEntry(s + ".pgdump") + if !ok { + panic(s) + } + out = append(out, e) + } + return out +} + +func kept(entries []backupEntry, p RetentionPolicy, now time.Time) []string { + del := map[string]bool{} + for _, d := range selectForDeletion(append([]backupEntry{}, entries...), p, now) { + del[d] = true + } + var out []string + for _, e := range entries { + if !del[e.name] { + out = append(out, strings.TrimSuffix(e.name, ".pgdump")) + } + } + sort.Sort(sort.Reverse(sort.StringSlice(out))) + return out +} + +// Restic forget's documented example shape: rules are a union, each rule takes +// the newest backup of each period that has one, a period with no backup does +// not use up the count, and one backup can satisfy several rules. +func TestGFSExactSets(t *testing.T) { + now := mustTime(t, "20260701-120000") + entries := entriesAt( + "20260701-030000", "20260701-010000", // two on the newest day + "20260630-030000", + "20260628-030000", // Sunday: end of ISO week 26 + "20260622-030000", // Monday of week 26 + "20260621-030000", // Sunday: end of week 25 + "20260614-030000", + "20260531-030000", + "20260430-030000", + "20260401-030000", + "20250630-030000", + ) + cases := []struct { + name string + p RetentionPolicy + want []string + }{ + {"keep last 2", RetentionPolicy{KeepLast: 2}, []string{"20260701-030000", "20260701-010000"}}, + {"daily 3 skips the second backup of a day and an empty day", RetentionPolicy{KeepDaily: 3}, + []string{"20260701-030000", "20260630-030000", "20260628-030000"}}, + {"weekly 3: newest of each ISO week", RetentionPolicy{KeepWeekly: 3}, + []string{"20260701-030000", "20260628-030000", "20260621-030000"}}, + {"monthly 3: newest of each month, gaps do not count", RetentionPolicy{KeepMonthly: 3}, + []string{"20260701-030000", "20260630-030000", "20260531-030000"}}, + {"union, one backup satisfies several rules", RetentionPolicy{KeepLast: 1, KeepDaily: 2, KeepWeekly: 2, KeepMonthly: 2}, + []string{"20260701-030000", "20260630-030000", "20260628-030000"}}, + {"more periods than exist keeps each period's newest", RetentionPolicy{KeepMonthly: 100}, + []string{"20260701-030000", "20260630-030000", "20260531-030000", "20260430-030000", "20250630-030000"}}, + {"max age keeps everything younger", RetentionPolicy{MaxAgeDays: 2}, + []string{"20260701-030000", "20260701-010000", "20260630-030000"}}, + {"deploy-ovh: 48h, 7 daily, 4 weekly, 12 monthly", RetentionPolicy{MaxAgeDays: 2, KeepDaily: 7, KeepWeekly: 4, KeepMonthly: 12}, + []string{"20260701-030000", "20260701-010000", "20260630-030000", "20260628-030000", "20260622-030000", "20260621-030000", "20260614-030000", "20260531-030000", "20260430-030000", "20250630-030000"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := kept(entries, tc.p, now) + if strings.Join(got, ",") != strings.Join(tc.want, ",") { + t.Fatalf("kept\n %v\nwant\n %v", got, tc.want) + } + }) + } +} + +// ISO weeks cross years: 2026-01-01 belongs to 2026 W01, 2025-12-29 too. +func TestGFSWeekAcrossYear(t *testing.T) { + entries := entriesAt("20260101-000000", "20251229-000000", "20251228-000000") + got := kept(entries, RetentionPolicy{KeepWeekly: 2}, mustTime(t, "20260102-000000")) + if strings.Join(got, ",") != "20260101-000000,20251228-000000" { + t.Fatalf("kept %v", got) + } +} + +func TestPruneNeverDeletesNewestGood(t *testing.T) { + now := mustTime(t, "20300101-000000") + entries := entriesAt("20200103-000000", "20200102-000000", "20200101-000000") + if got := kept(entries, RetentionPolicy{MaxAgeDays: 1}, now); strings.Join(got, ",") != "20200103-000000" { + t.Fatalf("everything past max age must still leave the newest: %v", got) + } +} + +// A below-floor object is not a good copy: it neither counts toward keep_last +// nor shields an older good one, and an all-bad series deletes nothing. +func TestPruneBadObjectsAreNotGoodCopies(t *testing.T) { + now := mustTime(t, "20300101-000000") + bad := func(s string) backupEntry { + e, _ := newBackupEntry(s + ".pgdump") + e.bad = true + return e + } + entries := append(entriesAt("20200101-000000"), bad("20200102-000000"), bad("20200103-000000")) + del := selectForDeletion(append([]backupEntry{}, entries...), RetentionPolicy{KeepLast: 1}, now) + if len(del) != 0 { + t.Fatalf("a bad object newer than the good one is left alone, the good one kept: %v", del) + } + entries = append(entriesAt("20200105-000000", "20200101-000000"), bad("20200102-000000")) + del = selectForDeletion(append([]backupEntry{}, entries...), RetentionPolicy{KeepLast: 1}, now) + sort.Strings(del) + if strings.Join(del, ",") != "20200101-000000.pgdump,20200102-000000.pgdump" { + t.Fatalf("del = %v", del) + } + if del := selectForDeletion([]backupEntry{bad("20200102-000000")}, RetentionPolicy{KeepLast: 1}, now); len(del) != 0 { + t.Fatalf("with no good copy nothing goes: %v", del) + } +} + +func treeMock(listing string) *ssh.MockExecutor { + return newStoreMock("h", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "aws s3 ls", Output: listing}, + ssh.MockCommand{Match: "aws s3 rm", Output: ""}, + ssh.MockCommand{Match: "{ cat /etc/machine-id", Output: "0123456789abcdef0123456789abcdef -\n"}, + ) +} + +func rmKeys(mock *ssh.MockExecutor) []string { + var out []string + for _, c := range mock.Calls { + if strings.HasPrefix(c, "aws s3 rm ") { + f := strings.Fields(c) + out = append(out, strings.Trim(f[3], "'")) + } + } + sort.Strings(out) + return out +} + +// A pruned backup takes globals, meta, per-database objects and their meta; the +// kept backup keeps all of theirs; unknown objects are never touched. +func TestPruneTakesEverySidecarAndNothingElse(t *testing.T) { + r := "a/accessories/db/" + listing := strings.Join([]string{ + "d t 1000 " + r + "20240101-000000.pgdump.age", + "d t 1000 " + r + "20240102-000000.pgdump.age", + "d t 500 " + r + "globals/20240101-000000.sql.gz.age", + "d t 500 " + r + "globals/20240102-000000.sql.gz.age", + "d t 500 " + r + "meta/20240101-000000.json", + "d t 500 " + r + "meta/20240102-000000.json.age", + "d t 1000 " + r + "databases/x/20240101-000000.pgdump", + "d t 1000 " + r + "databases/x/20240102-000000.pgdump.age", + "d t 500 " + r + "databases/x/meta/20240101-000000.json.age", + "d t 1000 " + r + "not-a-backup.txt", + "d t 1000 " + r + "weird/20200101-000000.pgdump", + }, "\n") + mock := treeMock(listing) + if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1}); err != nil { + t.Fatal(err) + } + want := []string{ + "s3://b/" + r + "20240101-000000.pgdump.age", + "s3://b/" + r + "databases/x/20240101-000000.pgdump", + "s3://b/" + r + "databases/x/meta/20240101-000000.json.age", + "s3://b/" + r + "globals/20240101-000000.sql.gz.age", + "s3://b/" + r + "meta/20240101-000000.json", + } + sort.Strings(want) + if got := rmKeys(mock); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("removed\n%s\nwant\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} + +// With no primary objects (an adopted container), each database is its own +// series; globals stay as long as any database still needs that id. +func TestPruneAdoptedSeriesIndependently(t *testing.T) { + r := "_adopted/accessories/m/" + listing := strings.Join([]string{ + "d t 1000 " + r + "databases/a/20240101-000000.sql.gz", + "d t 1000 " + r + "databases/a/20240102-000000.sql.gz", + "d t 1000 " + r + "databases/b/20240101-000000.sql.gz", // b's dump has been failing: its only copy + "d t 500 " + r + "globals/20240101-000000.sql.gz", + }, "\n") + mock := treeMock(listing) + if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), AdoptedApp, "accessories/m", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1}); err != nil { + t.Fatal(err) + } + got := rmKeys(mock) + if strings.Join(got, ",") != "s3://b/"+r+"databases/a/20240101-000000.sql.gz" { + t.Fatalf("removed %v", got) + } +} + +// A database whose dump keeps failing keeps its newest good object even while +// the primary series moves on. +func TestPruneKeepsADatabasesLastGoodCopy(t *testing.T) { + r := "a/accessories/db/" + listing := strings.Join([]string{ + "d t 1000 " + r + "20240103-000000.pgdump", + "d t 1000 " + r + "20240102-000000.pgdump", + "d t 1000 " + r + "20240101-000000.pgdump", + "d t 1000 " + r + "databases/x/20240101-000000.pgdump", + "d t 1000 " + r + "databases/y/20240101-000000.pgdump", + "d t 1000 " + r + "databases/y/20240103-000000.pgdump", + "d t 1000 " + r + "databases/y/20240104-000000.pgdump", // a killed run's: newer than any primary + }, "\n") + mock := treeMock(listing) + if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 2}); err != nil { + t.Fatal(err) + } + want := []string{"s3://b/" + r + "20240101-000000.pgdump", "s3://b/" + r + "databases/y/20240101-000000.pgdump"} + if got := rmKeys(mock); strings.Join(got, ",") != strings.Join(want, ",") { + t.Fatalf("removed %v want %v", got, want) + } +} + +func TestPruneRefusesASharedNamespace(t *testing.T) { + mock := newStoreMock("h", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "aws s3 ls 's3://b/a/.owners/'", Output: "d t 20 0123456789abcdef\nd t 20 aaaaaaaaaaaaaaaa\n"}, + ssh.MockCommand{Match: "{ cat /etc/machine-id", Output: "0123456789abcdef0123456789abcdef -\n"}, + ) + _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "volumes", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1}) + if !errors.Is(err, ErrSharedNamespace) { + t.Fatalf("err = %v", err) + } + if len(rmKeys(mock)) != 0 { + t.Fatal("nothing may be deleted") + } +} + +// A refused delete (bucket locks) does not stop the rest, and is reported. +func TestPruneToleratesRefusedDeletes(t *testing.T) { + r := "a/volumes/" + mock := newStoreMock("h", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "aws s3 ls", Output: "d t 1000 " + r + "20240103-000000.tar.gz\nd t 1000 " + r + "20240102-000000.tar.gz\nd t 1000 " + r + "20240101-000000.tar.gz\n"}, + ssh.MockCommand{Match: "aws s3 rm 's3://b/" + r + "20240102-000000.tar.gz'", Err: errors.New("exit status 1: AccessDenied: object is locked")}, + ssh.MockCommand{Match: "aws s3 rm", Output: ""}, + ) + deleted, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "volumes", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1}) + if err == nil || !strings.Contains(err.Error(), "locked") { + t.Fatalf("err = %v", err) + } + if len(deleted) != 1 || deleted[0] != "20240101-000000.tar.gz" { + t.Fatalf("deleted = %v", deleted) + } +} + +// ---- put / fetch ---- + +func putMock(extra ...ssh.MockCommand) *ssh.MockExecutor { + base := []ssh.MockCommand{ + {Match: "sha256sum", Output: "abc f\n4096\n"}, + {Match: "{ ", Output: "abc -\n"}, + {Match: "aws s3api head-object", Output: `{"ContentLength":4096,"Metadata":{"sha256":"abc"}}`}, + {Match: "aws s3 cp", Output: "ok\n"}, + {Match: "aws s3 rm", Output: ""}, + {Match: "rm -f", Output: ""}, + {Match: "command -v age", Output: "ok\n"}, + {Match: "if [ -r", Output: "yes\n"}, + {Match: "age-keygen -y", Output: testServerRecipient + "\n"}, + {Match: "age -r", Output: ""}, + {Match: "age -d", Output: ""}, + } + return ssh.NewMockExecutor("h", append(extra, base...)...) +} + +// The final key only ever receives a server-side copy of a verified .incoming/ +// object, and plaintext never goes up when encrypting. +func TestPutUploadsToIncomingThenPublishes(t *testing.T) { + mock := putMock() + s3 := S3Config{Bucket: "b", Region: "r", OwnerRecipient: testOwnerRecipient} + obj, err := NewClient(mock, io.Discard).put(context.Background(), s3, "/w/x.pgdump", "s3://b/a/accessories/db/1.pgdump") + if err != nil { + t.Fatal(err) + } + if obj.Key != "s3://b/a/accessories/db/1.pgdump.age" || !obj.Encrypted { + t.Fatalf("obj = %+v", obj) + } + var order []string + for _, c := range mock.Calls { + switch { + case strings.HasPrefix(c, "age -r"): + if !strings.Contains(c, testOwnerRecipient) || !strings.Contains(c, testServerRecipient) { + t.Fatalf("must encrypt to owner and server: %s", c) + } + order = append(order, "encrypt") + case strings.HasPrefix(c, "age -d"): + order = append(order, "prove") + case strings.HasPrefix(c, "rm -f '/w/x.pgdump'"): + order = append(order, "rm-plain") + case strings.HasPrefix(c, "aws s3 cp '/w/x.pgdump.age' 's3://b/a/accessories/db/.incoming/1.pgdump.age' --metadata 'sha256=abc'"): + order = append(order, "upload") + case strings.HasPrefix(c, "aws s3 cp '/w/x.pgdump'"): + t.Fatalf("plaintext uploaded: %s", c) + case strings.HasPrefix(c, "{ aws s3 cp"): + order = append(order, "readback") + case strings.HasPrefix(c, "aws s3 cp 's3://b/a/accessories/db/.incoming/1.pgdump.age' 's3://b/a/accessories/db/1.pgdump.age'"): + order = append(order, "publish") + case strings.HasPrefix(c, "aws s3 rm 's3://b/a/accessories/db/.incoming/1.pgdump.age'"): + order = append(order, "clean") + } + } + if strings.Join(order, ",") != "encrypt,prove,rm-plain,upload,readback,publish,clean" { + t.Fatalf("order = %v", order) + } +} + +func TestPutRefusesWhatDoesNotVerify(t *testing.T) { + for _, tc := range []struct { + name string + mock ssh.MockCommand + want string + }{ + {"short object", ssh.MockCommand{Match: "aws s3api head-object", Output: `{"ContentLength":100,"Metadata":{"sha256":"abc"}}`}, "100 bytes"}, + {"wrong checksum recorded", ssh.MockCommand{Match: "aws s3api head-object", Output: `{"ContentLength":4096,"Metadata":{"sha256":"zzz"}}`}, "checksum"}, + {"bytes differ", ssh.MockCommand{Match: "{ ", Output: "zzz -\n"}, "checksum mismatch"}, + {"head refused", ssh.MockCommand{Match: "aws s3api head-object", Err: errors.New("exit status 254: Forbidden")}, "verifying the upload"}, + } { + t.Run(tc.name, func(t *testing.T) { + mock := putMock(tc.mock) + _, err := NewClient(mock, io.Discard).put(context.Background(), S3Config{Bucket: "b", Region: "r"}, "/w/x", "s3://b/k/1.pgdump") + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want %q", err, tc.want) + } + for _, c := range mock.Calls { + if strings.HasPrefix(c, "aws s3 cp 's3://b/k/.incoming/1.pgdump' 's3://b/k/1.pgdump'") { + t.Fatalf("an unverified object was published: %s", c) + } + } + removed := false + for _, c := range mock.Calls { + removed = removed || strings.HasPrefix(c, "aws s3 rm 's3://b/k/.incoming/1.pgdump'") + } + if !removed { + t.Fatal("the unverified upload was left behind") + } + }) + } +} + +// Encryption failing never falls back to plaintext. +func TestPutEncryptFailureUploadsNothing(t *testing.T) { + for _, tc := range []ssh.MockCommand{ + {Match: "age -r", Err: errors.New("exit status 1: no space")}, + {Match: "age -d", Err: errors.New("exit status 1: no identity matched")}, + } { + mock := putMock(tc) + _, err := NewClient(mock, io.Discard).put(context.Background(), S3Config{Bucket: "b", Region: "r", OwnerRecipient: testOwnerRecipient}, "/w/x", "s3://b/k/1.pgdump") + if err == nil { + t.Fatalf("%s: expected a failure", tc.Match) + } + for _, c := range mock.Calls { + if strings.HasPrefix(c, "aws s3 cp") { + t.Fatalf("%s: uploaded anyway: %s", tc.Match, c) + } + } + } +} + +func TestPrepareEncryptionRefusesEarly(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "command -v age", Err: errors.New("exit status 1")}) + err := NewClient(mock, io.Discard).prepareEncryption(context.Background(), S3Config{OwnerRecipient: testOwnerRecipient}) + if err == nil || !strings.Contains(err.Error(), "not installed") { + t.Fatalf("err = %v", err) + } + if err := NewClient(mock, io.Discard).prepareEncryption(context.Background(), S3Config{OwnerRecipient: "not-a-key"}); err == nil { + t.Fatal("a malformed recipient must be refused") + } +} + +func TestFetchRefusesACorruptObject(t *testing.T) { + mock := putMock(ssh.MockCommand{Match: "sha256sum", Output: "different f\n4096\n"}) + err := NewClient(mock, io.Discard).fetch(context.Background(), S3Config{Bucket: "b", Region: "r"}, "s3://b/k/1.pgdump", "/w/x") + if err == nil || !strings.Contains(err.Error(), "corrupt") { + t.Fatalf("err = %v", err) + } +} + +func TestFetchDecryptsWithServerIdentityAndNamesTheWayOut(t *testing.T) { + mock := putMock(ssh.MockCommand{Match: "umask 077; age -d", Output: ""}) + if err := NewClient(mock, io.Discard).fetch(context.Background(), S3Config{Bucket: "b", Region: "r"}, "s3://b/k/1.pgdump.age", "/w/x"); err != nil { + t.Fatal(err) + } + dec := false + for _, c := range mock.Calls { + dec = dec || strings.HasPrefix(c, "umask 077; age -d -i \"$(if [ -r /etc/teploy/age-key ]") + } + if !dec { + t.Fatalf("no decrypt: %v", mock.Calls) + } + mock = putMock(ssh.MockCommand{Match: "umask 077; age -d", Err: errors.New("exit status 1: no identity matched")}) + err := NewClient(mock, io.Discard).fetch(context.Background(), S3Config{Bucket: "b", Region: "r"}, "s3://b/k/1.pgdump.age", "/w/x") + if err == nil || !strings.Contains(err.Error(), "backup fetch --identity") { + t.Fatalf("err = %v", err) + } +} + +func TestFindObjectPrefersCurrentThenPrevious(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "aws s3 ls 's3://b/new/a/volumes/1.'", Err: errors.New("exit status 1")}, + ssh.MockCommand{Match: "aws s3 ls 's3://b/a/volumes/1.'", Output: "d t 100 1.tar.gz.age\n"}, + ) + s3 := S3Config{Bucket: "b", Region: "r", Prefix: "new", PreviousPrefixes: []PreviousPrefix{{Prefix: ""}}} + key, ext, loc, err := NewClient(mock, io.Discard).findObject(context.Background(), s3, "a/volumes/", "1", ".tar.gz") + if err != nil || key != "s3://b/a/volumes/1.tar.gz.age" || ext != ".tar.gz" || loc.Prefix != "" { + t.Fatalf("%q %q %+v %v", key, ext, loc, err) + } +} + +// The recursive listing's keys are relative to the bucket: a prefix must not be +// doubled in the URI nor hide the rows. +func TestListWithPrefixAndPreviousPrefix(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "aws s3 ls 's3://b/srv1/a/' --recursive", Output: "2026-01-01 00:00:00 1000 srv1/a/accessories/db/20260102-000000.pgdump.age\n2026-01-01 00:00:00 1000 srv1/a/accessories/db/.incoming/20260103-000000.pgdump.age\n2026-01-01 00:00:00 10 srv1/a/accessories/db/20260104-000000.pgdump\n"}, + ssh.MockCommand{Match: "aws s3 ls 's3://b/a/' --recursive", Output: "2026-01-01 00:00:00 1000 a/accessories/db/databases/wp%20x/20260101-000000.sql.gz\n"}, + ) + s3 := S3Config{Bucket: "b", Region: "r", Prefix: "srv1", PreviousPrefixes: []PreviousPrefix{{Prefix: ""}}} + rows, err := NewClient(mock, io.Discard).ListBackupsRecursive(context.Background(), "a", s3) + if err != nil { + t.Fatal(err) + } + byID := map[string]ListedBackup{} + for _, r := range rows { + byID[r.ID] = r + } + if r := byID["20260102-000000"]; r.Key != "s3://b/srv1/a/accessories/db/20260102-000000.pgdump.age" || !r.Encrypted || r.Previous || !r.Restorable { + t.Fatalf("current: %+v", r) + } + if _, ok := byID["20260103-000000"]; ok { + t.Fatal("a pending upload listed as a backup") + } + if r := byID["20260104-000000"]; r.Restorable { + t.Fatalf("a below-floor object must not be restorable: %+v", r) + } + if r := byID["20260101-000000"]; !r.Previous || r.Database != "wp x" || r.Key != "s3://b/a/accessories/db/databases/wp%20x/20260101-000000.sql.gz" { + t.Fatalf("previous: %+v", r) + } +} + +// `aws s3 ls` exits 1 with no output on an empty prefix: that is an empty list, +// and any other failure is not. +func TestListTreeEmptyIsNotAnError(t *testing.T) { + dir := t.TempDir() + write := func(body string) { + if err := os.WriteFile(filepath.Join(dir, "aws"), []byte("#!/bin/sh\n"+body+"\n"), 0o755); err != nil { + t.Fatal(err) + } + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + exec := ssh.NewLocalExecutor() + write("exit 1") + keys, err := NewClient(exec, io.Discard).listTree(context.Background(), S3Config{Bucket: "b", Region: "r"}, "a/") + if err != nil || len(keys) != 0 { + t.Fatalf("%v %v", keys, err) + } + write("echo 'An error occurred (AccessDenied)' >&2; exit 1") + if _, err := NewClient(exec, io.Discard).listTree(context.Background(), S3Config{Bucket: "b", Region: "r"}, "a/"); err == nil { + t.Fatal("a refused listing must not read as empty") + } + write("exit 254") + if _, err := NewClient(exec, io.Discard).listTree(context.Background(), S3Config{Bucket: "b", Region: "r"}, "a/"); err == nil { + t.Fatal("another exit status must not read as empty") + } +} + +func TestLatestSkipsBelowFloorAndSearchesPrevious(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "aws s3 ls 's3://b/new/a/accessories/db/'", Output: "2026-01-01 00:00:00 20 20260105-000000.pgdump\n"}, + ssh.MockCommand{Match: "aws s3 ls 's3://b/a/accessories/db/'", Output: "2026-01-01 00:00:00 1000 20260101-000000.pgdump\n"}, + ) + s3 := S3Config{Bucket: "b", Region: "r", Prefix: "new", PreviousPrefixes: []PreviousPrefix{{Prefix: ""}}} + id, _, _, loc, err := NewClient(mock, io.Discard).latestBackup(context.Background(), "a", "db", "", s3) + if err != nil || id != "20260101-000000" || loc.Prefix != "" { + t.Fatalf("%q %+v %v", id, loc, err) + } +} + +// ---- addressing ---- + +func TestDBSegmentIsOneSafeKeySegment(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {"wp_blog", "wp_blog"}, + {"a b", "a%20b"}, + {"a/b", "a%2Fb"}, + {"..", "%2E."}, + {"x'; rm -rf /; '", "x%27%3B%20rm%20-rf%20%2F%3B%20%27"}, + {"$(touch x)", "%24%28touch%20x%29"}, + } { + got := dbSegment(tc.in) + if got != tc.want { + t.Errorf("dbSegment(%q) = %q, want %q", tc.in, got, tc.want) + } + if back := decodeDBSegment(got); back != tc.in { + t.Errorf("decode(%q) = %q, want %q", got, back, tc.in) + } + } +} + +func TestAddressableDB(t *testing.T) { + for _, ok := range []string{"wp_blog", "a b", "x'; touch y", "$(touch y)", "`z`", "日本語"} { + if !addressableDB(ok) { + t.Errorf("%q should be addressable once quoted", ok) + } + } + for _, bad := range []string{"", "-rf", "--all-databases", "a=b", "a\nb", strings.Repeat("x", 65)} { + if addressableDB(bad) { + t.Errorf("%q must not be addressable", bad) + } + } + if mysqlDumpableDB("x'y") || mysqlDumpableDB(`x\y`) || !mysqlDumpableDB("a b") { + t.Error("mysqlDumpableDB wrong") + } +} + +func TestContainerName(t *testing.T) { + if ContainerName("app", "db") != "app-db" || ContainerName(AdoptedApp, "omni_postgres") != "omni_postgres" { + t.Fatal("wrong container names") + } +} + +// Hostile adopted names and schemas reach the shell only inside single quotes; +// non-addressable schemas are reported, never run. +func TestAdoptedMySQLHostileSchemasAreQuotedAndKeyed(t *testing.T) { + hostile := "x y$(touch /tmp/p)" + unaddressable := "--all-databases" + mock := newStoreMock("h", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "docker inspect -f", Output: "{}\n[]\n[]\n"}, + ssh.MockCommand{Match: "umask 077", Output: "/var/tmp/teploy-backup.t\n"}, + ssh.MockCommand{Match: "docker exec 'm_y' sh -c 'MYSQL_PWD=${MARIADB_ROOT_PASSWORD:-${MYSQL_ROOT_PASSWORD:-${MYSQL_PASSWORD:-}}}; export MYSQL_PWD; exec mysql -u root -N -B -e '\"'\"'SELECT s.schema_name", Output: "wp_a\t0\n" + hostile + "\t0\n" + unaddressable + "\t0\n"}, + ssh.MockCommand{Match: "docker exec", Output: ""}, + ssh.MockCommand{Match: "mkdir -p", Output: ""}, + ssh.MockCommand{Match: "rm -rf", Output: ""}, + ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, + ssh.MockCommand{Match: "df -Pk", Output: "Filesystem 1K-blocks Used Available Use% Mounted\n/dev/x 1 1 99999999 1% /\n"}, + ssh.MockCommand{Match: "aws s3", Output: ""}, + ssh.MockCommand{Match: "printf", Output: ""}, + ) + c := NewClient(mock, io.Discard) + if err := c.SetEngineOverride("m_y", EngineMySQL); err != nil { + t.Fatal(err) + } + res, err := c.AccessoryBackupWith(context.Background(), AdoptedApp, "m_y", "x", map[string]string{"MYSQL_ROOT_PASSWORD": "pw"}, S3Config{Bucket: "b", Region: "r"}, BackupOptions{AllDatabases: true}) + if err != nil { + t.Fatalf("backup: %v", err) + } + got := map[string]string{} + for _, o := range res.Objects { + got[o.Database] = o.Key + } + if got["wp_a"] == "" || got[hostile] == "" { + t.Fatalf("objects: %+v", res.Objects) + } + if !strings.Contains(got[hostile], "/_adopted/accessories/m_y/databases/x%20y%24%28touch%20%2Ftmp%2Fp%29/") { + t.Fatalf("key: %s", got[hostile]) + } + if len(res.UncoveredDatabases) != 1 || res.UncoveredDatabases[0] != unaddressable { + t.Fatalf("uncovered: %v", res.UncoveredDatabases) + } + for _, call := range mock.Calls { + if strings.Contains(call, "mysqldump") && strings.Contains(call, unaddressable) { + t.Fatalf("an option-shaped name reached the dump: %s", call) + } + if strings.Contains(call, "mysqldump") && strings.Contains(call, hostile) && !strings.Contains(call, ssh.ShellQuote(ssh.ShellQuote(hostile))) && !strings.Contains(call, ssh.ShellQuote(hostile)) { + t.Fatalf("a hostile name is not quoted: %s", call) + } + } +} + +// One database failing does not stop the next, fails the backup, and is named. +func TestOneDatabaseFailingDoesNotStopTheOthers(t *testing.T) { + mock := newStoreMock("h", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "docker inspect -f", Output: "{}\n[]\n[]\n"}, + ssh.MockCommand{Match: "umask 077", Output: "/var/tmp/teploy-backup.t\n"}, + ssh.MockCommand{Match: "docker exec 'a-db' pg_dumpall", Output: ""}, + ssh.MockCommand{Match: "docker exec 'a-db' psql -X -qtA -v ON_ERROR_STOP=1 -U 'postgres' -d 'postgres' -c 'SELECT datname", Output: "app\nbad\ngood\n"}, + ssh.MockCommand{Match: "docker exec 'a-db' pg_dump -Fc -U 'postgres' 'bad'", Err: errors.New("exit status 1: pg_dump: error")}, + ssh.MockCommand{Match: "docker exec", Output: ""}, + ssh.MockCommand{Match: "{ docker", Output: ""}, + ssh.MockCommand{Match: "mkdir -p", Output: ""}, + ssh.MockCommand{Match: "rm -rf", Output: ""}, + ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, + ssh.MockCommand{Match: "df -Pk", Output: "Filesystem 1K-blocks Used Available Use% Mounted\n/dev/x 1 1 99999999 1% /\n"}, + ssh.MockCommand{Match: "aws s3", Output: ""}, + ssh.MockCommand{Match: "printf", Output: ""}, + ) + res, err := NewClient(mock, io.Discard).AccessoryBackupWith(context.Background(), "a", "db", "postgres:17", map[string]string{"POSTGRES_DB": "app"}, S3Config{Bucket: "b", Region: "r"}, BackupOptions{AllDatabases: true}) + if err == nil || !strings.Contains(err.Error(), "bad") { + t.Fatalf("err = %v", err) + } + if res.FailedDatabases["bad"] == "" { + t.Fatalf("failed: %+v", res.FailedDatabases) + } + names := []string{} + for _, o := range res.Objects { + names = append(names, o.Database) + } + if strings.Join(names, ",") != "good," { + t.Fatalf("objects stored: %q", names) + } + // Commit order: the primary object is stored after every sidecar and database. + lastPut := -1 + for i, c := range mock.Calls { + if strings.Contains(c, ".incoming/") && strings.Contains(c, "--metadata") { + lastPut = i + } + } + if !strings.Contains(mock.Calls[lastPut], "a/accessories/db/.incoming/") || strings.Contains(mock.Calls[lastPut], "databases/") { + t.Fatalf("the primary must be the last upload: %s", mock.Calls[lastPut]) + } +} + +// Preflight: measures the backup (plain + ciphertext), refuses before dumping. +func TestPreflightRefusesBeforeDumping(t *testing.T) { + mock := newStoreMock("h", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "docker inspect -f", Output: "{}\n[]\n[]\n"}, + ssh.MockCommand{Match: "umask 077", Output: "/var/tmp/teploy-backup.t\n"}, + ssh.MockCommand{Match: "docker exec 'a-db' psql -X -qtA -v ON_ERROR_STOP=1 -U 'postgres' -d 'app' -c 'SELECT pg_database_size", Output: "10737418240\n"}, + ssh.MockCommand{Match: "df -Pk", Output: "Filesystem 1K-blocks Used Available Use% Mounted\n/dev/x 1 1 1048576 1% /\n"}, + ssh.MockCommand{Match: "docker exec", Output: ""}, + ssh.MockCommand{Match: "rm -rf", Output: ""}, + ssh.MockCommand{Match: "aws s3", Output: ""}, + ) + _, err := NewClient(mock, io.Discard).AccessoryBackup(context.Background(), "a", "db", "postgres:17", map[string]string{"POSTGRES_DB": "app"}, S3Config{Bucket: "b", Region: "r"}) + if err == nil || !strings.Contains(err.Error(), "not enough free space") || !strings.Contains(err.Error(), "--ignore-space-check") { + t.Fatalf("err = %v", err) + } + for _, c := range mock.Calls { + if strings.Contains(c, "pg_dump -Fc") { + t.Fatalf("dumped anyway: %s", c) + } + } +} + +// ---- volumes: .env ---- + +func TestUnencryptedVolumeBackupLeavesEnvOutAndSays(t *testing.T) { + mock := newStoreMock("h", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "umask 077", Output: "/var/tmp/teploy-backup.t\n"}, + ssh.MockCommand{Match: "du -sk", Output: "10\t/x\n"}, + ssh.MockCommand{Match: "df -Pk", Output: "Filesystem 1K-blocks Used Available Use% Mounted\n/dev/x 1 1 99999999 1% /\n"}, + ssh.MockCommand{Match: "if [ -f", Output: "yes\n"}, + ssh.MockCommand{Match: "tar -czf", Output: ""}, + ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, + ssh.MockCommand{Match: "if [ -e", Output: "absent\n"}, + ssh.MockCommand{Match: "aws s3", Output: ""}, + ssh.MockCommand{Match: "rm -rf", Output: ""}, + ssh.MockCommand{Match: "printf", Output: ""}, + ) + res, err := NewClient(mock, io.Discard).BackupVolumes(context.Background(), "a", S3Config{Bucket: "b", Region: "r"}) + if err != nil { + t.Fatal(err) + } + if res.Encrypted || len(res.Warnings) != 1 || !strings.Contains(res.Warnings[0], ".env was left out") { + t.Fatalf("res = %+v", res) + } + for _, c := range mock.Calls { + if strings.HasPrefix(c, "tar -czf") && strings.Contains(c, ".env") { + t.Fatalf(".env in an unencrypted archive: %s", c) + } + } +} + +// ---- DR bundles ---- + +func TestBundleAgeKeyRefusedWithoutARecipient(t *testing.T) { + mock := drBaseMock() + _, err := NewClient(mock, io.Discard).CreateBundle(context.Background(), BundleOptions{App: "myapp", Config: drTestConfig(), IncludeSecrets: true, IncludeAgeKey: true}, DirBundleStore{Root: "/var/drstore"}) + if err == nil || !strings.Contains(err.Error(), "recovery key") { + t.Fatalf("err = %v", err) + } +} + +// The private key goes from its file straight to ciphertext, to the owner only. +func TestBundleAgeKeyIsWrappedToTheOwnerOnly(t *testing.T) { + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "umask 077; age -r", Output: ""}) + store := S3BundleStore{S3: S3Config{Bucket: "b", Region: "r", OwnerRecipient: testOwnerRecipient}} + cmds := []ssh.MockCommand{ + {Match: "aws s3 cp", Output: ""}, {Match: "aws s3 rm", Output: ""}, {Match: "rm -f", Output: ""}, + {Match: "command -v age", Output: "ok\n"}, {Match: "if [ -r", Output: "yes\n"}, {Match: "age-keygen -y", Output: testServerRecipient + "\n"}, + {Match: "age -r", Output: ""}, {Match: "age -d", Output: ""}, + } + mock = newStoreMock("1.2.3.4", cmds...) + // Members: encrypted through put; a member already age ciphertext, and the + // manifest, are not. + for _, member := range []string{"accessories/db.pgdump", "secrets/K.age", "age-key.age", "manifest.json"} { + if err := store.Upload(context.Background(), mock, "myapp", "20260101-000000", member, "/w/"+member); err != nil { + t.Fatalf("%s: %v", member, err) + } + } + joined := strings.Join(mock.Calls, "\n") + for _, want := range []string{ + "dr/20260101-000000/accessories/db.pgdump.age'", + "dr/20260101-000000/secrets/K.age'", + "dr/20260101-000000/age-key.age'", + "dr/20260101-000000/manifest.json'", + } { + if !strings.Contains(joined, want) { + t.Errorf("no upload to %s:\n%s", want, joined) + } + } + for _, unwanted := range []string{"age-key.age.age", "secrets/K.age.age", "manifest.json.age"} { + if strings.Contains(joined, unwanted) { + t.Errorf("double wrapped: %s", unwanted) + } + } +} + +func TestBundleDownloadFindsEncryptedAndLegacyMembers(t *testing.T) { + store := S3BundleStore{S3: S3Config{Bucket: "b", Region: "r"}} + mock := newStoreMock("h", + ssh.MockCommand{Match: "aws s3 ls", Output: "d t 100 db.pgdump.age\n"}, + ssh.MockCommand{Match: "aws s3 cp", Output: ""}, + ssh.MockCommand{Match: "umask 077; age -d", Output: ""}, + ssh.MockCommand{Match: "rm -f", Output: ""}, + ) + if err := store.Download(context.Background(), mock, "a", "1", "accessories/db.pgdump", "/w/x"); err != nil { + t.Fatal(err) + } + if !strings.Contains(strings.Join(mock.Calls, "\n"), "accessories/db.pgdump.age") { + t.Fatalf("did not fetch the encrypted member: %v", mock.Calls) + } + legacy := newStoreMock("h", + ssh.MockCommand{Match: "aws s3 ls", Err: errors.New("exit status 1")}, + ssh.MockCommand{Match: "aws s3 cp", Output: ""}, + ) + if err := store.Download(context.Background(), legacy, "a", "1", "accessories/db.pgdump", "/w/x"); err != nil { + t.Fatal(err) + } + if strings.Contains(strings.Join(legacy.Calls, "\n"), "age -d") { + t.Fatalf("a plain member must not be decrypted: %v", legacy.Calls) + } +} diff --git a/internal/backup/store_test_helpers_test.go b/internal/backup/store_test_helpers_test.go new file mode 100644 index 0000000..7cae720 --- /dev/null +++ b/internal/backup/store_test_helpers_test.go @@ -0,0 +1,23 @@ +package backup + +import "github.com/useteploy/teploy/internal/ssh" + +// newStoreMock is ssh.NewMockExecutor with the verified-upload path answered: +// the local measurement and the stored object agree, so a test about something +// else does not have to spell the upload's checks out. A test about the upload +// itself uses ssh.NewMockExecutor. +func newStoreMock(host string, cmds ...ssh.MockCommand) *ssh.MockExecutor { + return ssh.NewMockExecutor(host, append(append([]ssh.MockCommand{}, storeStubs...), cmds...)...) +} + +var storeStubs = []ssh.MockCommand{ + {Match: "sha256sum", Output: "abc f\n4096\n"}, + {Match: "{ ", Output: "abc -\n"}, + {Match: "aws s3api head-object", Output: `{"ContentLength":4096,"Metadata":{"sha256":"abc"}}`}, +} + +// Valid X25519 age recipients (fixed values: public keys, no matching identity is kept). +const ( + testOwnerRecipient = "age1lggyhqrw2nlhcxprm67z43rta597azn8gknawjehu9d9dl0jq3yqqvfafg" + testServerRecipient = "age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p" +) diff --git a/internal/backup/target.go b/internal/backup/target.go index 1d29904..1c0d4d5 100644 --- a/internal/backup/target.go +++ b/internal/backup/target.go @@ -46,6 +46,9 @@ type Target struct { Region string `json:"region"` Endpoint string `json:"endpoint,omitempty"` Prefix string `json:"prefix,omitempty"` + // PreviousPrefixes are prefixes this target used before, in this bucket: + // list, latest and restore search them; prune only touches owned ones. + PreviousPrefixes []PreviousPrefix `json:"-"` // HasCredentials reports whether the stored file carries an encrypted // credentials blob. The blob itself is never decoded into this struct. HasCredentials bool `json:"-"` @@ -53,6 +56,15 @@ type Target struct { Path string `json:"-"` } +// PreviousPrefix is a key prefix this server's backups used to go under. Owned +// means the operator has said only this server ever wrote there, which is what +// allows its old objects to age out; otherwise they may be another server's, so +// they are found and restorable but never deleted. +type PreviousPrefix struct { + Prefix string `json:"prefix"` + Owned bool `json:"owned,omitempty"` +} + // targetFile is the on-disk shape. Credentials holds age armor (empty when // the target relies on the server's ambient AWS credential chain). type targetFile struct { @@ -62,6 +74,8 @@ type targetFile struct { Endpoint string `json:"endpoint"` Prefix string `json:"prefix"` Credentials string `json:"credentials,omitempty"` + // PreviousPrefixes is additive: an older CLI ignores it. + PreviousPrefixes []PreviousPrefix `json:"previous_prefixes,omitempty"` } // credentialValueRE is the charset a stored S3 access key or secret may @@ -138,6 +152,7 @@ func ReadTarget(ctx context.Context, exec ssh.Executor) (Target, error) { return Target{}, fmt.Errorf("%s: unsupported backup target version %q", t.Path, f.V) } t.Bucket, t.Region, t.Endpoint, t.Prefix = f.Bucket, f.Region, f.Endpoint, f.Prefix + t.PreviousPrefixes = f.PreviousPrefixes t.HasCredentials = f.Credentials != "" return t, nil } @@ -153,7 +168,7 @@ echo absent` // age-encrypted credentials blob) on the server and returns the path // written plus whether the non-root fallback location was used. func WriteTarget(ctx context.Context, exec ssh.Executor, t Target, credsArmor string) (path string, fallback bool, err error) { - f := targetFile{V: "1", Bucket: t.Bucket, Region: t.Region, Endpoint: t.Endpoint, Prefix: t.Prefix, Credentials: credsArmor} + f := targetFile{V: "1", Bucket: t.Bucket, Region: t.Region, Endpoint: t.Endpoint, Prefix: t.Prefix, Credentials: credsArmor, PreviousPrefixes: t.PreviousPrefixes} payload, err := json.Marshal(f) if err != nil { return "", false, err diff --git a/internal/backup/verify.go b/internal/backup/verify.go index b5ac80f..c73328c 100644 --- a/internal/backup/verify.go +++ b/internal/backup/verify.go @@ -26,8 +26,10 @@ type VerifyResult struct { Kind string `json:"kind"` // postgres|mysql|mongo|redis|valkey|dragonfly|nucleus|volume // Engine is the detected engine (mariadb is Kind "mysql"; generic is // Kind "volume") and Detection the evidence it came from. - Engine string `json:"engine"` - Detection string `json:"detection"` + Engine string `json:"engine"` + Detection string `json:"detection"` + // Database is the database verified, when it has its own series. + Database string `json:"database,omitempty"` Date string `json:"date"` S3Key string `json:"s3_key"` SizeBytes int64 `json:"size_bytes"` @@ -35,6 +37,13 @@ type VerifyResult struct { DurationMs int64 `json:"duration_ms"` OK bool `json:"ok"` Detail string `json:"detail,omitempty"` + // Encrypted says the verified object was age-encrypted, and was decrypted + // with this server's own identity. + Encrypted bool `json:"encrypted"` + // series is the bucket-relative prefix of the backup's object. + series string + // loc is where the object was found (current or a previous prefix). + loc S3Config } // InspectAccessory reads the image and environment of the RUNNING accessory @@ -44,7 +53,7 @@ type VerifyResult struct { // gives auth parity with the backup commands (same POSTGRES_USER, same // passwordless-root mysql, etc.). func (c *Client) InspectAccessory(ctx context.Context, app, name string) (image string, env map[string]string, err error) { - containerName := app + "-" + name + containerName := ContainerName(app, name) img, err := c.exec.Run(ctx, fmt.Sprintf( "docker inspect -f '{{.Config.Image}}' %s", ssh.ShellQuote(containerName))) if err != nil { @@ -73,41 +82,55 @@ func (c *Client) InspectAccessory(ctx context.Context, app, name string) (image // by listing the S3 prefix. Returns the bare timestamp (the restore // arg) and the object's size in bytes. func (c *Client) LatestBackupDate(ctx context.Context, app, name string, s3 S3Config) (date string, size int64, err error) { - date, size, _, err = c.latestBackup(ctx, app, name, s3) + date, size, _, _, err = c.latestBackup(ctx, app, name, "", s3) return date, size, err } -// latestBackup is LatestBackupDate plus the object's file name (its -// extension says which artifact format the backup is). -func (c *Client) latestBackup(ctx context.Context, app, name string, s3 S3Config) (date string, size int64, file string, err error) { - prefix := fmt.Sprintf("s3://%s/%s/accessories/%s/", s3.Base(), app, name) - out, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 ls %s", ssh.ShellQuote(prefix)))) - if err != nil { - return "", 0, "", fmt.Errorf("listing backups at %s: %w", prefix, err) - } - // Lines look like: 2026-07-10 04:00:01 52428800 20260710-040000.sql.gz - // The teploy timestamp format sorts lexically, so track the max. - var best, bestFile string +// latestBackup is LatestBackupDate for one series (the primary, or one +// database's) across the current and previous prefixes, plus the object's file +// name and where it was found. Objects below the engine's plausibility floor are +// not backups, and pending uploads never appear (they sit one level down). +func (c *Client) latestBackup(ctx context.Context, app, name, db string, s3 S3Config) (date string, size int64, file string, found S3Config, err error) { + var listed bool + var lastErr error var bestSize int64 - for _, line := range strings.Split(strings.TrimSpace(out), "\n") { - fields := strings.Fields(line) - if len(fields) < 4 { - continue - } - fname := fields[len(fields)-1] - ts, _, ok := strings.Cut(fname, ".") - if !ok || ValidateDate(ts) != nil { + var best, bestFile string + for _, loc := range s3.locations() { + prefix := seriesPrefix(loc, app, name, db) + out, lsErr := c.exec.Run(ctx, loc.AWS(fmt.Sprintf("s3 ls %s", ssh.ShellQuote(prefix)))) + if lsErr != nil { + lastErr = fmt.Errorf("listing backups at %s: %w", prefix, lsErr) continue } - if ts > best { - best, bestFile = ts, fname - bestSize, _ = strconv.ParseInt(fields[len(fields)-2], 10, 64) + listed = true + // Lines look like: 2026-07-10 04:00:01 52428800 20260710-040000.sql.gz + // The teploy timestamp format sorts lexically, so track the max. + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + fields := strings.Fields(line) + if len(fields) < 4 { + continue + } + fname := fields[len(fields)-1] + ts, _, ok := strings.Cut(fname, ".") + if !ok || ValidateDate(ts) != nil { + continue + } + sz, _ := strconv.ParseInt(fields[len(fields)-2], 10, 64) + if sz < artifactFloor(fname) { + continue + } + if ts > best { + best, bestFile, bestSize, found = ts, fname, sz, loc + } } } if best == "" { - return "", 0, "", fmt.Errorf("no backups found at %s", prefix) + if !listed && lastErr != nil { + return "", 0, "", s3, lastErr + } + return "", 0, "", s3, fmt.Errorf("no backups found at %s", seriesPrefix(s3, app, name, db)) } - return best, bestSize, bestFile, nil + return best, bestSize, bestFile, found, nil } // VerifyBackup downloads a backup (latest when date is empty), restores it @@ -128,7 +151,15 @@ func verifyTmpDir() string { return "/var/tmp" } -func (c *Client) VerifyBackup(ctx context.Context, app, name, date string, s3 S3Config) (retResult *VerifyResult, retErr error) { +func (c *Client) VerifyBackup(ctx context.Context, app, name, date string, s3 S3Config) (*VerifyResult, error) { + return c.VerifyBackupDatabase(ctx, app, name, date, "", s3) +} + +// VerifyBackupDatabase is VerifyBackup for one database with its own series +// (database empty: the accessory's primary). +func (c *Client) VerifyBackupDatabase(ctx context.Context, app, name, date, database string, s3 S3Config) (retResult *VerifyResult, retErr error) { + ctx, cancel := context.WithTimeout(ctx, c.backupDeadline()) + defer cancel() if err := c.ensureAWSCLI(ctx); err != nil { return nil, err } @@ -143,11 +174,32 @@ func (c *Client) VerifyBackup(ctx context.Context, app, name, date string, s3 S3 if err != nil { return nil, err } + series := det.Engine == EnginePostgres || det.Engine == EngineMySQL || det.Engine == EngineMariaDB + if database != "" && !series { + return nil, fmt.Errorf("--database applies to postgres and mysql/mariadb accessories; %s is %s", name, det.Engine) + } + if database != "" && !addressableDB(database) { + return nil, fmt.Errorf("invalid database name %q", database) + } + if app == AdoptedApp && series && database == "" { + return nil, fmt.Errorf("%s holds several databases, each with its own backups: name one with --database", name) + } + seriesDB := database + if database != "" && app != AdoptedApp { + primary, _ := postgresDBAndUser(app, env) + if det.Engine != EnginePostgres { + primary = mysqlDB(app, env) + } + if database == primary { + seriesDB = "" + } + } var size int64 var latestFile string + loc := s3 if date == "" { - date, size, latestFile, err = c.latestBackup(ctx, app, name, s3) + date, size, latestFile, loc, err = c.latestBackup(ctx, app, name, seriesDB, s3) if err != nil { return nil, err } @@ -157,7 +209,8 @@ func (c *Client) VerifyBackup(ctx context.Context, app, name, date string, s3 S3 res := &VerifyResult{ App: app, Accessory: name, Image: image, Date: date, SizeBytes: size, - Engine: det.Engine, Detection: det.Source, + Engine: det.Engine, Detection: det.Source, Database: database, + series: strings.TrimPrefix(seriesPrefix(S3Config{Bucket: "b"}, app, name, seriesDB), "s3://b/"), loc: loc, } var nonce [16]byte @@ -197,18 +250,22 @@ func (c *Client) VerifyBackup(ctx context.Context, app, name, date string, s3 S3 if det.IsDB() && det.Engine != EngineNucleus { ext := "" for _, e := range artifactExts(det.Engine) { - if strings.HasSuffix(latestFile, e) { + if strings.HasSuffix(strings.TrimSuffix(latestFile, ageSuffix), e) { ext = e } } + key := "" if ext == "" && latestFile == "" { - if _, e, rerr := c.resolveAccessoryKey(ctx, s3, fmt.Sprintf("s3://%s/%s/accessories/%s/", s3.Base(), app, name), date, artifactExts(det.Engine)...); rerr == nil { - ext = e + if k, e, l, rerr := c.findObject(ctx, s3, res.series, date, artifactExts(det.Engine)...); rerr == nil { + ext, key, res.loc = e, k, l } } if ext == ".tar.gz" { res.Kind = det.Engine - res.S3Key = fmt.Sprintf("s3://%s/%s/accessories/%s/%s.tar.gz", s3.Base(), app, name, date) + if key == "" { + key = seriesPrefix(loc, app, name, seriesDB) + latestFile + } + res.S3Key = key res.DurationMs = time.Since(start).Milliseconds() res.Detail = "this " + det.Engine + " backup is a raw file copy of the live data directory (taken before teploy detected the engine) — it can be restored as files but not verified as a database; take a new backup" return res, nil @@ -221,7 +278,7 @@ func (c *Client) VerifyBackup(ctx context.Context, app, name, date string, s3 S3 switch det.Engine { case EnginePostgres: res.Kind = "postgres" - err = c.verifyPostgres(ctx, res, app, name, image, env, s3, scratch, tmpBase, latestFile) + err = c.verifyPostgres(ctx, res, app, name, image, env, s3, scratch, tmpBase) case EngineMySQL, EngineMariaDB: res.Kind = "mysql" err = c.verifyMySQL(ctx, res, app, name, det.Engine, image, env, s3, scratch, tmpBase) @@ -246,6 +303,11 @@ func (c *Client) VerifyBackup(ctx context.Context, app, name, date string, s3 S3 } res.DurationMs = time.Since(start).Milliseconds() + if res.SizeBytes == 0 && res.S3Key != "" { + if h, herr := c.head(ctx, res.loc, res.S3Key); herr == nil { + res.SizeBytes = h.Size + } + } if err != nil { res.OK = false res.Detail = err.Error() @@ -283,14 +345,20 @@ func (c *Client) verifyEnv(ctx context.Context, env map[string]string, tmpBase s return "--label teploy.verify.owner=" + ssh.ShellQuote(env["TEPLOY_VERIFY_OWNER"]) + " --env-file " + ssh.ShellQuote(path), nil } -func (c *Client) download(ctx context.Context, s3Key, dest string, s3 S3Config) error { - fmt.Fprintf(c.out, "Downloading %s...\n", s3Key) - _, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 cp %s %s", - ssh.ShellQuote(s3Key), ssh.ShellQuote(dest)))) +// fetchVerifyObject downloads (and decrypts) the backup under verification, +// trying exts in order, recording its key and encryption on res. +func (c *Client) fetchVerifyObject(ctx context.Context, res *VerifyResult, s3 S3Config, dest func(ext string) string, exts ...string) (string, error) { + key, ext, loc, err := c.findObject(ctx, s3, res.series, res.Date, exts...) if err != nil { - return fmt.Errorf("downloading backup: %w", err) + return "", err } - return nil + res.S3Key, res.loc = key, loc + res.Encrypted = strings.HasSuffix(key, ageSuffix) + fmt.Fprintf(c.out, "Downloading %s...\n", key) + if err := c.fetch(ctx, loc, key, dest(ext)); err != nil { + return "", fmt.Errorf("downloading backup: %w", err) + } + return ext, nil } // waitReady polls a readiness probe inside the scratch container. @@ -304,31 +372,25 @@ func (c *Client) waitReady(ctx context.Context, scratch, probe string, attempts return nil } -func (c *Client) verifyPostgres(ctx context.Context, res *VerifyResult, app, name, image string, env map[string]string, s3 S3Config, scratch, tmpBase, latestFile string) error { +func (c *Client) verifyPostgres(ctx context.Context, res *VerifyResult, app, name, image string, env map[string]string, s3 S3Config, scratch, tmpBase string) error { db, user := postgresDBAndUser(app, env) - prefix := fmt.Sprintf("s3://%s/%s/accessories/%s/", s3.Base(), app, name) - var s3Key, ext string - switch { - case strings.HasSuffix(latestFile, ".pgdump"): - s3Key, ext = prefix+latestFile, ".pgdump" - case strings.HasSuffix(latestFile, ".sql.gz"): - s3Key, ext = prefix+latestFile, ".sql.gz" - default: - var err error - if s3Key, ext, err = c.resolveAccessoryKey(ctx, s3, prefix, res.Date, ".pgdump", ".sql.gz"); err != nil { - return err + seriesDB := "" + if res.Database != "" { + if res.Database != db || app == AdoptedApp { + seriesDB = res.Database } + db = res.Database } - res.S3Key = s3Key - dump := tmpBase + ext - if err := c.download(ctx, s3Key, dump, s3); err != nil { + ext, err := c.fetchVerifyObject(ctx, res, s3, func(e string) string { return tmpBase + e }, ".pgdump", ".sql.gz") + if err != nil { return err } + dump := tmpBase + ext // The cluster's roles (pg_dumpall --globals-only, stored beside the // dump) let the scratch restore keep real ownership. Older backups // have none. globals := tmpBase + ".globals.sql.gz" - haveGlobals := c.download(ctx, prefix+"globals/"+res.Date+".sql.gz", globals, s3) == nil + haveGlobals := c.fetchSidecar(ctx, res.loc, postgresGlobalsKey(res.loc, app, name, res.Date), globals) flags, err := c.verifyEnv(ctx, env, tmpBase, "POSTGRES_", "POSTGRESQL_", "ALLOW_EMPTY_PASSWORD") if err != nil { @@ -371,7 +433,7 @@ func (c *Client) verifyPostgres(ctx context.Context, res *VerifyResult, app, nam return fmt.Errorf("preparing the scratch database: %w", err) } fmt.Fprintf(c.out, "Restoring into scratch as %s...\n", user) - if _, err := c.restorePostgres(ctx, newPGTarget(scratch, db, user, env), dump, ext, path.Dir(tmpBase), pgRestoreOpts{logicalBytes: c.pgRecordedSize(ctx, s3, app, name, res.Date, path.Dir(tmpBase))}); err != nil { + if _, err := c.restorePostgres(ctx, newPGTarget(scratch, db, user, env), dump, ext, path.Dir(tmpBase), pgRestoreOpts{logicalBytes: c.pgRecordedSize(ctx, res.loc, app, name, seriesDB, res.Date, path.Dir(tmpBase))}); err != nil { return fmt.Errorf("restore into scratch failed: %w", err) } @@ -389,10 +451,11 @@ func (c *Client) verifyPostgres(ctx context.Context, res *VerifyResult, app, nam func (c *Client) verifyMySQL(ctx context.Context, res *VerifyResult, app, name, engine, image string, env map[string]string, s3 S3Config, scratch, tmpBase string) error { db := mysqlDB(app, env) - s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.sql.gz", s3.Base(), app, name, res.Date) - res.S3Key = s3Key + if res.Database != "" { + db = res.Database + } dump := tmpBase + ".sql.gz" - if err := c.download(ctx, s3Key, dump, s3); err != nil { + if _, err := c.fetchVerifyObject(ctx, res, s3, func(string) string { return dump }, ".sql.gz"); err != nil { return err } @@ -415,6 +478,12 @@ func (c *Client) verifyMySQL(ctx context.Context, res *VerifyResult, app, name, if err := c.waitReady(ctx, scratch, "sh -c "+ssh.ShellQuote(prelude+"exec "+client+" --protocol=TCP -h 127.0.0.1 -u root -e 'SELECT 1'"), 40); err != nil { return err } + // The scratch server only made MYSQL_DATABASE: a dump of one schema has no + // CREATE DATABASE of its own. + createDB := prelude + "exec " + client + " -u root -e " + ssh.ShellQuote("CREATE DATABASE IF NOT EXISTS "+mysqlIdent(db)) + if _, err := c.exec.Run(ctx, "docker exec "+ssh.ShellQuote(scratch)+" sh -c "+ssh.ShellQuote(createDB)); err != nil { + return fmt.Errorf("preparing the scratch database: %w", err) + } restore := prelude + "exec " + client + " -u root " + ssh.ShellQuote(db) if _, err := c.exec.Run(ctx, "gunzip -c "+ssh.ShellQuote(dump)+" > "+ssh.ShellQuote(tmpBase+".sql")+" && docker exec -i "+ssh.ShellQuote(scratch)+" sh -c "+ssh.ShellQuote(restore)+" < "+ssh.ShellQuote(tmpBase+".sql")); err != nil { return fmt.Errorf("restore into scratch failed: %w", err) @@ -435,10 +504,8 @@ func (c *Client) verifyMySQL(ctx context.Context, res *VerifyResult, app, name, func (c *Client) verifyRedis(ctx context.Context, res *VerifyResult, app, name, engine, image string, env map[string]string, s3 S3Config, scratch, tmpBase string) error { cli := redisCLI(engine) - s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.rdb.gz", s3.Base(), app, name, res.Date) - res.S3Key = s3Key dump := tmpBase + ".rdb.gz" - if err := c.download(ctx, s3Key, dump, s3); err != nil { + if _, err := c.fetchVerifyObject(ctx, res, s3, func(string) string { return dump }, ".rdb.gz"); err != nil { return err } @@ -467,10 +534,8 @@ func (c *Client) verifyRedis(ctx context.Context, res *VerifyResult, app, name, // verifyDragonfly boots a scratch dragonfly and loads the snapshot with // DFLY LOAD (the same mechanism the in-place restore uses). func (c *Client) verifyDragonfly(ctx context.Context, res *VerifyResult, app, name, image string, env map[string]string, s3 S3Config, scratch, tmpBase string) error { - s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.rdb.gz", s3.Base(), app, name, res.Date) - res.S3Key = s3Key dump := tmpBase + ".rdb.gz" - if err := c.download(ctx, s3Key, dump, s3); err != nil { + if _, err := c.fetchVerifyObject(ctx, res, s3, func(string) string { return dump }, ".rdb.gz"); err != nil { return err } if _, err := c.exec.Run(ctx, "gunzip -c "+ssh.ShellQuote(dump)+" > "+ssh.ShellQuote(tmpBase+".rdb")); err != nil { @@ -490,10 +555,8 @@ func (c *Client) verifyDragonfly(ctx context.Context, res *VerifyResult, app, na } func (c *Client) verifyMongo(ctx context.Context, res *VerifyResult, app, name, image string, env map[string]string, s3 S3Config, scratch, tmpBase string) error { - s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.archive.gz", s3.Base(), app, name, res.Date) - res.S3Key = s3Key dump := tmpBase + ".archive.gz" - if err := c.download(ctx, s3Key, dump, s3); err != nil { + if _, err := c.fetchVerifyObject(ctx, res, s3, func(string) string { return dump }, ".archive.gz"); err != nil { return err } @@ -536,10 +599,8 @@ func (c *Client) verifyMongo(ctx context.Context, res *VerifyResult, app, name, // tables, and stay running. The "Restored N table(s)" boot line is the // verification metric. func (c *Client) verifyNucleus(ctx context.Context, res *VerifyResult, app, name, image string, env map[string]string, s3 S3Config, scratch, tmpBase string) error { - s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.tar.gz", s3.Base(), app, name, res.Date) - res.S3Key = s3Key dump := tmpBase + ".tar.gz" - if err := c.download(ctx, s3Key, dump, s3); err != nil { + if _, err := c.fetchVerifyObject(ctx, res, s3, func(string) string { return dump }, ".tar.gz"); err != nil { return err } @@ -582,10 +643,8 @@ func (c *Client) verifyNucleus(ctx context.Context, res *VerifyResult, app, name // verifyVolume checks a generic volume archive: integrity (a truncated or // corrupt tar fails extraction) and non-emptiness. func (c *Client) verifyVolume(ctx context.Context, res *VerifyResult, app, name string, s3 S3Config, tmpBase string) error { - s3Key := fmt.Sprintf("s3://%s/%s/accessories/%s/%s.tar.gz", s3.Base(), app, name, res.Date) - res.S3Key = s3Key dump := tmpBase + ".tar.gz" - if err := c.download(ctx, s3Key, dump, s3); err != nil { + if _, err := c.fetchVerifyObject(ctx, res, s3, func(string) string { return dump }, ".tar.gz"); err != nil { return err } extractDir := tmpBase + ".d" diff --git a/internal/backup/verify_test.go b/internal/backup/verify_test.go index fc20421..e19067e 100644 --- a/internal/backup/verify_test.go +++ b/internal/backup/verify_test.go @@ -11,7 +11,7 @@ import ( ) func TestLatestBackupDate_PicksNewest(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "umask 077; mktemp -d", Output: "/var/tmp/teploy-verify.test\n"}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "" + @@ -33,7 +33,7 @@ func TestLatestBackupDate_PicksNewest(t *testing.T) { } func TestLatestBackupDate_NoBackups(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "umask 077; mktemp -d", Output: "/var/tmp/teploy-verify.test\n"}, ssh.MockCommand{Match: "aws s3 ls", Output: "\n"}, ) @@ -49,7 +49,7 @@ func TestLatestBackupDate_NoBackups(t *testing.T) { // inspect (not config), the restore targets a SCRATCH container (never the // real accessory), the verify query runs, and teardown removes the scratch. func TestVerifyBackup_Postgres(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "umask 077; mktemp -d", Output: "/var/tmp/teploy-verify.test\n"}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "docker inspect -f '{{.Config.Image}}'", Output: "postgres:16\n"}, @@ -112,7 +112,7 @@ func TestVerifyBackup_Postgres(t *testing.T) { // TestVerifyBackup_FailureIsResultAndTearsDown asserts a failed restore // yields ok=false (not an operational error) and STILL removes the scratch. func TestVerifyBackup_FailureIsResultAndTearsDown(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "umask 077; mktemp -d", Output: "/var/tmp/teploy-verify.test\n"}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "docker inspect -f '{{.Config.Image}}'", Output: "postgres:16\n"}, @@ -151,7 +151,7 @@ func TestVerifyBackup_FailureIsResultAndTearsDown(t *testing.T) { // TestVerifyBackup_NucleusBootVerify covers the engine-boot verification for // nucleus accessories (generic tar + scratch engine on the restored data dir). func TestVerifyBackup_NucleusBootVerify(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", ssh.MockCommand{Match: "umask 077; mktemp -d", Output: "/var/tmp/teploy-verify.test\n"}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "docker inspect -f '{{.Config.Image}}'", Output: "ghcr.io/neutron-build/nucleus:latest\n"}, @@ -184,7 +184,8 @@ func TestVerifyBackup_NucleusBootVerify(t *testing.T) { // TestVerifyBackup_VolumeArchive covers the generic path: extraction is the // integrity check, and an empty archive fails. func TestVerifyBackup_VolumeArchive(t *testing.T) { - mock := ssh.NewMockExecutor("1.2.3.4", + mock := newStoreMock("1.2.3.4", + ssh.MockCommand{Match: "aws s3 ls", Output: "2026-07-10 04:00:01 4096 20260710-040000.tar.gz\n"}, ssh.MockCommand{Match: "umask 077; mktemp -d", Output: "/var/tmp/teploy-verify.test\n"}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "docker inspect -f '{{.Config.Image}}'", Output: "some/custom-thing:1\n"}, diff --git a/internal/ssh/result.go b/internal/ssh/result.go index 673d792..8d0f19e 100644 --- a/internal/ssh/result.go +++ b/internal/ssh/result.go @@ -93,6 +93,13 @@ func RunInputDetailed(ctx context.Context, ex Executor, cmd string, stdin io.Rea return runDetailedWithLimit(ctx, ex, cmd, stdin, DefaultResultLimit) } +// RunDetailedLimit is RunDetailed with a caller-chosen capture limit, for the +// few callers whose stdout is legitimately large (a bucket listing). Callers +// must treat Truncated as a failure. +func RunDetailedLimit(ctx context.Context, ex Executor, cmd string, limit int64) Result { + return runDetailedWithLimit(ctx, ex, cmd, nil, limit) +} + func runDetailedWithLimit(ctx context.Context, ex Executor, cmd string, stdin io.Reader, limit int64) Result { switch v := ex.(type) { case *RemoteExecutor: From 57a322fafd4179a42273eb027f44ae205cc80da2 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Wed, 4 Nov 2026 22:34:35 -0700 Subject: [PATCH 02/17] feat(backup): fleet engine: server policy, run-time enumeration, overlap lock, heartbeats, status The engine behind `teploy backup run --all` (G4): - /etc/teploy/backup-policy.json, 0600, with the non-root fallback; an unreadable policy fails closed instead of reading as no encryption. - Enumeration at run time: labelled accessories, adopted containers, app volumes; a per-app schedule overrides, a hand-written job under a marker does not; docker failing is an error, not an empty server. - flock overlap lock: a killed run cannot block the next, a live holder past every deadline is killed after checking the pid is teploy's. - One target failing does not stop the next, nothing of a failed target is pruned, and every omission is named, so ok never hides one. - Heartbeats with status=down semantics; no URL or token in any result. - Volume size guard, killed-run janitor, multipart abort. - ReadFleetStatus: one read-only report of every DB-like container, no secrets. Co-Authored-By: Claude Opus 5.5 --- internal/backup/fleet.go | 557 +++++++++++++++++++++++++++++++ internal/backup/fleet_test.go | 434 ++++++++++++++++++++++++ internal/backup/policy.go | 376 +++++++++++++++++++++ internal/backup/runlock_other.go | 28 ++ internal/backup/runlock_unix.go | 104 ++++++ internal/backup/status.go | 508 ++++++++++++++++++++++++++++ internal/events/events.go | 10 +- 7 files changed, 2014 insertions(+), 3 deletions(-) create mode 100644 internal/backup/fleet.go create mode 100644 internal/backup/fleet_test.go create mode 100644 internal/backup/policy.go create mode 100644 internal/backup/runlock_other.go create mode 100644 internal/backup/runlock_unix.go create mode 100644 internal/backup/status.go diff --git a/internal/backup/fleet.go b/internal/backup/fleet.go new file mode 100644 index 0000000..c04d997 --- /dev/null +++ b/internal/backup/fleet.go @@ -0,0 +1,557 @@ +package backup + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/useteploy/teploy/internal/ssh" +) + +// The fleet run (G4): what `teploy backup run --all` does on a server. Targets +// are enumerated AT RUN TIME (labelled accessories, app volumes, adopted +// containers), run one at a time (a small host's disk and RAM are shared with +// production), and one failing never stops the others. + +// Target kinds. +const ( + TargetAccessory = "accessory" + TargetVolumes = "volumes" + TargetAdopted = "adopted" +) + +// Target statuses. +const ( + TargetOK = "ok" + TargetFailed = "failed" + TargetExcluded = "excluded" // left out by policy or the size guard + TargetOwnCron = "own-schedule" +) + +// Run statuses. +const ( + RunOK = "ok" + RunPartial = "partial" + RunFailed = "failed" +) + +// ContainerFact is what the inventory knows about one container. Env is parsed +// for engine detection and credentials; it is never reported. +type ContainerFact struct { + Name, Image, State string + Role, App, Accessory string + Labels map[string]string + Env map[string]string + Argv []string + Engine EngineDetection + EngineErr error +} + +// Unmanaged reports a container with no teploy role. +func (f ContainerFact) Unmanaged() bool { return f.Role == "" } + +// inventoryFormat emits one JSON object per container. +const inventoryFormat = `{"name":{{json .Name}},"image":{{json .Config.Image}},"state":{{json .State.Status}},"labels":{{json .Config.Labels}},"env":{{json .Config.Env}},"entrypoint":{{json .Config.Entrypoint}},"cmd":{{json .Config.Cmd}}}` + +// Inventory reads every container on the server in one call. +func (c *Client) Inventory(ctx context.Context) ([]ContainerFact, error) { + // Not `docker ps | xargs`: a dead daemon would read as an empty server, and a + // run with no targets reports success. + cmd := "ids=$(docker ps -aq --no-trunc) || exit 1; [ -z \"$ids\" ] || docker inspect --format " + ssh.ShellQuote(inventoryFormat) + " $ids" + res := ssh.RunDetailedLimit(ctx, c.exec, cmd, 64<<20) + if res.Err != nil { + return nil, fmt.Errorf("listing containers: %w", res.Err) + } + if res.Truncated { + return nil, fmt.Errorf("listing containers: output too large to read safely") + } + if res.ExitCode != 0 { + return nil, fmt.Errorf("listing containers: %s", oneLine(res.ExitErrorText())) + } + return parseInventory(string(res.Stdout)) +} + +func parseInventory(out string) ([]ContainerFact, error) { + var facts []ContainerFact + for _, line := range splitLines(out) { + var raw struct { + Name string `json:"name"` + Image string `json:"image"` + State string `json:"state"` + Labels map[string]string `json:"labels"` + Env []string `json:"env"` + Entrypoint []string `json:"entrypoint"` + Cmd []string `json:"cmd"` + } + if err := json.Unmarshal([]byte(line), &raw); err != nil { + return nil, fmt.Errorf("parsing the container inventory: %w", err) + } + f := ContainerFact{ + Name: strings.TrimPrefix(raw.Name, "/"), Image: raw.Image, State: raw.State, Labels: raw.Labels, + Role: raw.Labels["teploy.role"], App: raw.Labels["teploy.app"], Accessory: raw.Labels["teploy.accessory"], + Env: map[string]string{}, Argv: append(append([]string{}, raw.Entrypoint...), raw.Cmd...), + } + for _, kv := range raw.Env { + if k, v, ok := strings.Cut(kv, "="); ok { + f.Env[k] = v + } + } + // Older accessories carry no teploy.accessory label; their name says. + if f.Role == "accessory" && f.Accessory == "" && f.App != "" { + f.Accessory = strings.TrimPrefix(f.Name, f.App+"-") + } + f.Engine, f.EngineErr = DetectEngine(f.Image, f.Env, f.Labels, f.Argv) + facts = append(facts, f) + } + sort.Slice(facts, func(i, j int) bool { return facts[i].Name < facts[j].Name }) + return facts, nil +} + +// isScratch reports teploy's own throwaway containers. +func isScratch(name string) bool { + return strings.HasPrefix(name, "teploy-backup-verify-") || strings.HasPrefix(name, "teploy-dr-") +} + +// FleetTarget is one thing a run backs up. +type FleetTarget struct { + Kind string + App, Name string + Container string + Fact *ContainerFact + Adopted *AdoptedEntry + // Retention is the policy's, or the per-app override's. + Retention RetentionPolicy + Status string // pre-decided TargetExcluded / TargetOwnCron / TargetFailed + Reason string + SizeBytes int64 + // SizeGuard says the exclusion is volume_max_gb's: the run records an event + // so it is visible in the app's history, not only in this run's output. + SizeGuard bool +} + +// FleetTargetResult is one target's outcome. +type FleetTargetResult struct { + Kind string `json:"kind"` + App string `json:"app,omitempty"` + Accessory string `json:"accessory,omitempty"` + Container string `json:"container,omitempty"` + Engine string `json:"engine,omitempty"` + // Status: ok | failed | excluded | own-schedule. + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + ID string `json:"id,omitempty"` + // SizeBytes is stored bytes for a backup, the measured tree for an exclusion. + SizeBytes int64 `json:"size_bytes"` + DurationMs int64 `json:"duration_ms"` + Encrypted bool `json:"encrypted"` + Objects []BackupObject `json:"objects,omitempty"` + Pruned int `json:"pruned"` + Warnings []string `json:"warnings,omitempty"` + // FailedDatabases are databases of this target that were not backed up. + FailedDatabases map[string]string `json:"failed_databases,omitempty"` + // UncoveredDatabases are databases on this server this target left out. + UncoveredDatabases []string `json:"uncovered_databases,omitempty"` + Result BackupResult `json:"-"` + // SizeGuard says volume_max_gb excluded this target. + SizeGuard bool `json:"size_guard,omitempty"` +} + +// HeartbeatResult reports one heartbeat without its URL. +type HeartbeatResult struct { + Host string `json:"host"` + OK bool `json:"ok"` + Status int `json:"status,omitempty"` + Error string `json:"error,omitempty"` +} + +// FleetRunResult is the outcome of one run. +type FleetRunResult struct { + // Status: ok (every target ok; omissions are listed, see Warnings) | partial + // (some failed) | failed (none succeeded, or the run could not start). + Status string `json:"status"` + StartedAt time.Time `json:"started_at"` + DurationMs int64 `json:"duration_ms"` + Encrypted bool `json:"encrypted"` + Targets []FleetTargetResult `json:"targets"` + // UncoveredContainers are DB-like containers no target covers. + UncoveredContainers []string `json:"uncovered_containers,omitempty"` + // Omissions name everything left out of this run: exclusions, size-guard + // hits, uncovered containers and databases. A clean status with omissions is + // never just "ok" to a reader of this field. + Omissions []string `json:"omissions,omitempty"` + Warnings []string `json:"warnings,omitempty"` + Heartbeats []HeartbeatResult `json:"heartbeats,omitempty"` + // Cleaned counts a killed run's leftovers this run removed. + CleanedWorkspaces int `json:"cleaned_workspaces,omitempty"` + AbortedUploads int `json:"aborted_uploads,omitempty"` + Error string `json:"error,omitempty"` +} + +var ( + fleetAppRE = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + fleetNameRE = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*$`) +) + +// volumeTree is one app's non-empty volumes directory. +type volumeTree struct { + App string + KB int64 +} + +// listVolumeTrees sizes every app's volumes directory. +func (c *Client) listVolumeTrees(ctx context.Context) ([]volumeTree, error) { + out, err := c.exec.Run(ctx, `for d in /deployments/*/volumes; do [ -d "$d" ] || continue; [ -n "$(ls -A "$d" 2>/dev/null)" ] || continue; a=$(basename "$(dirname "$d")"); kb=$(du -sk "$d" 2>/dev/null | cut -f1); printf '%s\t%s\n' "${kb:-0}" "$a"; done; true`) + if err != nil { + return nil, fmt.Errorf("sizing app volumes: %w", err) + } + var trees []volumeTree + for _, line := range splitLines(out) { + kbs, app, ok := strings.Cut(line, "\t") + if !ok || !fleetAppRE.MatchString(app) { + continue + } + kb, _ := strconv.ParseInt(kbs, 10, 64) + trees = append(trees, volumeTree{App: app, KB: kb}) + } + return trees, nil +} + +func (c *Client) duKB(ctx context.Context, dir string) (int64, bool) { + out, err := c.exec.Run(ctx, "du -sk "+ssh.ShellQuote(dir)+" 2>/dev/null | cut -f1") + if err != nil { + return 0, false + } + kb, err := strconv.ParseInt(strings.TrimSpace(out), 10, 64) + return kb, err == nil +} + +// EnumerateFleet lists this run's targets and the DB-like containers none of +// them cover. It reads the server now; nothing is cached between runs. +func (c *Client) EnumerateFleet(ctx context.Context, p FleetPolicy) ([]FleetTarget, []string, error) { + facts, err := c.Inventory(ctx) + if err != nil { + return nil, nil, err + } + cron, err := ReadCrontab(ctx, c.exec) + if err != nil { + return nil, nil, err + } + ownCron := map[string]bool{} + for _, line := range cron { + // A hand-written job under a teploy marker (custom) is not an override: + // nothing says it works (two of them failed nightly for weeks), and the + // policy covering the database as well is how they get retired. + if m := FindCronMarker(line); m != nil && m.Kind != KindFleet && m.Form != CronFormCustom { + ownCron[m.App+"/"+m.Accessory] = true + } + } + policies := map[string]AppPolicy{} + readAppPolicy := func(app string) AppPolicy { + if ap, ok := policies[app]; ok { + return ap + } + ap, err := ReadPolicy(ctx, c.exec, app) + if err != nil { + fmt.Fprintf(c.out, "Warning: %v\n", err) + } + policies[app] = ap + return ap + } + base := p.Retention.Policy() + override := func(kp *KindPolicy, r RetentionPolicy) RetentionPolicy { + if kp == nil || (kp.KeepLast == 0 && kp.MaxAgeDays == 0) { + return r + } + return RetentionPolicy{KeepLast: kp.KeepLast, MaxAgeDays: kp.MaxAgeDays} + } + adopted := map[string]*AdoptedEntry{} + for i := range p.Adopted { + adopted[p.Adopted[i].Container] = &p.Adopted[i] + } + seenAdopted := map[string]bool{} + + var targets []FleetTarget + var uncovered []string + for i := range facts { + f := &facts[i] + if isScratch(f.Name) { + continue + } + if a, ok := adopted[f.Name]; ok { + seenAdopted[f.Name] = true + t := FleetTarget{Kind: TargetAdopted, App: AdoptedApp, Name: f.Name, Container: f.Name, Fact: f, Adopted: a, Retention: base} + if p.Excluded(AdoptedApp, f.Name, f.Name) { + t.Status, t.Reason = TargetExcluded, "excluded by policy" + } + targets = append(targets, t) + continue + } + if f.Role == "accessory" { + t := FleetTarget{Kind: TargetAccessory, App: f.App, Name: f.Accessory, Container: f.Name, Fact: f} + switch { + case !fleetAppRE.MatchString(f.App) || !fleetNameRE.MatchString(f.Accessory): + t.Status, t.Reason = TargetFailed, fmt.Sprintf("container %s carries unusable teploy labels", f.Name) + case f.Name != ContainerName(f.App, f.Accessory): + t.Status, t.Reason = TargetFailed, fmt.Sprintf("container %s does not match its labels (expected %s), so it cannot be addressed; adopt it by name", f.Name, ContainerName(f.App, f.Accessory)) + default: + ap := readAppPolicy(f.App) + var kp *KindPolicy + if ap.Accessories != nil { + kp = ap.Accessories[f.Accessory] + } + t.Retention = override(kp, base) + switch { + case p.Excluded(f.App, f.Accessory, f.Name): + t.Status, t.Reason = TargetExcluded, "excluded by policy" + case ownCron[f.App+"/"+f.Accessory]: + t.Status, t.Reason = TargetOwnCron, "has its own backup schedule (a per-app schedule overrides the policy)" + case p.VolumeMaxGB > 0 && f.EngineErr == nil && f.Engine.Engine == EngineGeneric: + // A generic accessory is a raw tar: the same full copy a + // volume archive is, under the same cap. + if kb, ok := c.duKB(ctx, fmt.Sprintf("%s/%s/accessories/%s", deploymentsDir, f.App, f.Accessory)); ok { + t.SizeBytes = kb * 1024 + if float64(t.SizeBytes) > p.VolumeMaxGB*(1<<30) { + t.Status, t.Reason, t.SizeGuard = TargetExcluded, fmt.Sprintf("accessory files are %.2f GB, over the policy's volume_max_gb %.2f; not backed up", float64(t.SizeBytes)/(1<<30), p.VolumeMaxGB), true + } + } + } + } + targets = append(targets, t) + continue + } + if f.EngineErr == nil && f.Engine.IsDB() && !p.Excluded("", "", f.Name) { + uncovered = append(uncovered, f.Name) + } + } + for _, a := range p.Adopted { + if !seenAdopted[a.Container] { + entry := a + targets = append(targets, FleetTarget{Kind: TargetAdopted, App: AdoptedApp, Name: a.Container, Container: a.Container, Adopted: &entry, Retention: base, + Status: TargetFailed, Reason: "adopted container " + a.Container + " does not exist on this server"}) + } + } + if p.Volumes { + trees, err := c.listVolumeTrees(ctx) + if err != nil { + return targets, uncovered, err + } + for _, tr := range trees { + t := FleetTarget{Kind: TargetVolumes, App: tr.App, SizeBytes: tr.KB * 1024} + ap := readAppPolicy(tr.App) + t.Retention = override(ap.Volumes, base) + switch { + case p.Excluded(tr.App, "volumes", ""): + t.Status, t.Reason = TargetExcluded, "excluded by policy" + case ownCron[tr.App+"/"]: + t.Status, t.Reason = TargetOwnCron, "has its own backup schedule (a per-app schedule overrides the policy)" + case p.VolumeMaxGB > 0 && float64(t.SizeBytes) > p.VolumeMaxGB*(1<<30): + t.Status, t.Reason, t.SizeGuard = TargetExcluded, fmt.Sprintf("volumes are %.2f GB, over the policy's volume_max_gb %.2f; not backed up", float64(t.SizeBytes)/(1<<30), p.VolumeMaxGB), true + } + targets = append(targets, t) + } + } + return targets, uncovered, nil +} + +// RunFleetTarget backs one target up and, only if that fully succeeded, prunes. +func (c *Client) RunFleetTarget(ctx context.Context, t FleetTarget, s3 S3Config, opts BackupOptions) FleetTargetResult { + r := FleetTargetResult{Kind: t.Kind, App: t.App, Accessory: t.Name, Container: t.Container, Status: t.Status, Reason: t.Reason, SizeBytes: t.SizeBytes, Encrypted: s3.Encrypted(), SizeGuard: t.SizeGuard} + if t.Status != "" { + return r + } + var res BackupResult + var err error + prefix := "volumes" + switch t.Kind { + case TargetVolumes: + res, err = c.BackupVolumesWith(ctx, t.App, s3, opts) + default: + prefix = "accessories/" + t.Name + o := opts + o.AllDatabases = true + if t.Adopted != nil { + if serr := c.SetEngineOverride(t.Container, t.Adopted.Engine); serr != nil { + err = serr + break + } + if !t.Adopted.Databases.All { + o.AllDatabases = false + o.Databases = t.Adopted.Databases.Names + } + } + res, err = c.AccessoryBackupWith(ctx, t.App, t.Name, t.Fact.Image, t.Fact.Env, s3, o) + } + r.Result = res + r.Engine, r.ID, r.DurationMs, r.Objects, r.Warnings, r.FailedDatabases, r.UncoveredDatabases = + res.Engine, res.ID, res.DurationMs, res.Objects, res.Warnings, res.FailedDatabases, res.UncoveredDatabases + r.SizeBytes = res.SizeBytes + if err != nil { + // Nothing is pruned when any part of this target failed: the run that + // could not make a new copy is the last one to delete old copies. + r.Status, r.Reason = TargetFailed, err.Error() + return r + } + r.Status = TargetOK + if t.Retention.IsZero() { + c.CleanStaleIncoming(ctx, t.App, prefix, s3) + return r + } + deleted, perr := c.PruneBackups(ctx, t.App, prefix, s3, t.Retention) + r.Pruned = len(deleted) + if perr != nil { + r.Warnings = append(r.Warnings, "prune: "+perr.Error()) + } + return r +} + +// SummarizeFleet sets the run status and collects omissions. +func SummarizeFleet(res *FleetRunResult) { + ok, failed := 0, 0 + for _, t := range res.Targets { + label := t.Container + if label == "" { + label = t.App + "/" + t.Kind + } + switch t.Status { + case TargetOK: + ok++ + if len(t.Objects) == 0 { + res.Omissions = append(res.Omissions, label+": holds no database yet, so nothing was stored") + } + case TargetFailed: + failed++ + case TargetExcluded: + res.Omissions = append(res.Omissions, label+": "+t.Reason) + } + for _, db := range t.UncoveredDatabases { + res.Omissions = append(res.Omissions, label+": database "+db+" not backed up") + } + } + for _, c := range res.UncoveredContainers { + res.Omissions = append(res.Omissions, c+": a database container no policy entry covers (adopt or exclude it)") + } + switch { + case failed == 0: + res.Status = RunOK + case ok > 0: + res.Status = RunPartial + default: + res.Status = RunFailed + } +} + +// ---- heartbeat ---- + +// heartbeatTimeout bounds one heartbeat attempt. +const heartbeatTimeout = 10 * time.Second + +// heartbeatURL is the URL to ping: as configured on success; on failure with +// status=down (Dash and Uptime Kuma push monitors) or healthchecks.io's /fail. +func heartbeatURL(raw string, up bool) (string, error) { + u, err := url.Parse(raw) + if err != nil { + return "", errors.New("unparseable heartbeat URL") + } + if up { + return u.String(), nil + } + if strings.HasSuffix(u.Hostname(), "hc-ping.com") { + u.Path = strings.TrimSuffix(u.Path, "/") + "/fail" + return u.String(), nil + } + q := u.Query() + q.Set("status", "down") + u.RawQuery = q.Encode() + return u.String(), nil +} + +// SendHeartbeats pings every URL. A heartbeat failing never fails the backup, +// and no URL (they carry tokens) reaches a result, a log or an error: net/http's +// errors embed the URL, so only their cause is kept. +func SendHeartbeats(ctx context.Context, urls []string, up bool, client *http.Client) []HeartbeatResult { + if client == nil { + client = &http.Client{Timeout: heartbeatTimeout} + } + out := make([]HeartbeatResult, 0, len(urls)) + for _, raw := range urls { + r := HeartbeatResult{} + parsed, perr := url.Parse(raw) + if perr != nil { + r.Error = "unparseable heartbeat URL" + out = append(out, r) + continue + } + r.Host = parsed.Host + target, err := heartbeatURL(raw, up) + if err != nil { + r.Error = err.Error() + out = append(out, r) + continue + } + for attempt := 0; attempt < 3 && !r.OK; attempt++ { + if attempt > 0 { + select { + case <-ctx.Done(): + case <-time.After(2 * time.Second): + } + } + r.Status, r.Error = 0, "" + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) + if err != nil { + r.Error = "building the request failed" + break + } + resp, err := client.Do(req) + if err != nil { + var ue *url.Error + if errors.As(err, &ue) { + r.Error = ue.Err.Error() + } else { + r.Error = "request failed" + } + continue + } + io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<16)) + resp.Body.Close() + r.Status = resp.StatusCode + r.OK = resp.StatusCode >= 200 && resp.StatusCode < 300 + if !r.OK { + r.Error = "unexpected status " + strconv.Itoa(resp.StatusCode) + } + } + out = append(out, r) + } + return out +} + +// ProbeTarget says whether the target answers through the exact path a run uses +// (aws, stored credentials), reading under the base: a key scoped to one prefix +// cannot list the bucket root, and an empty prefix is an answer. +func (c *Client) ProbeTarget(ctx context.Context, s3 S3Config) error { + pctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + _, err := c.listTree(pctx, s3, ownersDir+"-probe/") + return err +} + +// RunLockPath is where the overlap lock lives: beside the policy. +func RunLockPath(policyPath string) string { + if policyPath != "" { + return path.Dir(policyPath) + "/backup-run.lock" + } + if d := os.Getenv("HOME"); d != "" { + return d + "/.config/teploy/backup-run.lock" + } + return "/tmp/teploy-backup-run.lock" +} diff --git a/internal/backup/fleet_test.go b/internal/backup/fleet_test.go new file mode 100644 index 0000000..63b3468 --- /dev/null +++ b/internal/backup/fleet_test.go @@ -0,0 +1,434 @@ +package backup + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/ssh" +) + +// ---- policy ---- + +func TestFleetPolicyValidate(t *testing.T) { + ok := FleetPolicy{V: "1", Interval: "1h", Recipient: testOwnerRecipient, Adopted: []AdoptedEntry{{Container: "omni_postgres", Engine: "postgres", Databases: DatabaseSelector{All: true}}}, Excludes: []string{"app", "app/db", "omni_x"}} + if err := ok.Validate(); err != nil { + t.Fatal(err) + } + for name, p := range map[string]FleetPolicy{ + "interval": {Interval: "1 2 3"}, + "retention": {Retention: FleetRetention{KeepDaily: -1}}, + "recipient": {Recipient: "age1short"}, + "container": {Adopted: []AdoptedEntry{{Container: "x'; rm -rf /", Databases: DatabaseSelector{All: true}}}}, + "twice": {Adopted: []AdoptedEntry{{Container: "x", Databases: DatabaseSelector{All: true}}, {Container: "x", Databases: DatabaseSelector{All: true}}}}, + "engine": {Adopted: []AdoptedEntry{{Container: "x", Engine: "generic", Databases: DatabaseSelector{All: true}}}}, + "no-databases": {Adopted: []AdoptedEntry{{Container: "x"}}}, + "database": {Adopted: []AdoptedEntry{{Container: "x", Databases: DatabaseSelector{Names: []string{"--all"}}}}}, + "exclude": {Excludes: []string{"a/b/c"}}, + "exclude-shell": {Excludes: []string{"$(x)"}}, + "heartbeat": {HeartbeatURLs: []string{"ftp://h/token"}}, + } { + if err := p.Validate(); err == nil { + t.Errorf("%s: expected a refusal", name) + } + } + // A refused heartbeat URL never echoes itself: it is a secret. + err := (FleetPolicy{HeartbeatURLs: []string{"ftp://h/TOKEN"}}).Validate() + if strings.Contains(err.Error(), "TOKEN") { + t.Fatalf("error leaks the URL: %v", err) + } +} + +func TestDatabaseSelectorJSON(t *testing.T) { + var d DatabaseSelector + if err := json.Unmarshal([]byte(`"all"`), &d); err != nil || !d.All { + t.Fatalf("%+v %v", d, err) + } + if err := json.Unmarshal([]byte(`["a","b"]`), &d); err != nil || d.All || len(d.Names) != 2 { + t.Fatalf("%+v %v", d, err) + } + if err := json.Unmarshal([]byte(`"some"`), &d); err == nil { + t.Fatal(`only "all" is a string value`) + } + b, _ := json.Marshal(DatabaseSelector{All: true}) + if string(b) != `"all"` { + t.Fatalf("%s", b) + } +} + +func TestIntervalCron(t *testing.T) { + for _, iv := range []string{"15m", "30m", "1h", "hourly", "2h", "3h", "4h", "6h", "8h", "12h", "24h", "1d", "daily", "7d", "1w", "weekly"} { + c, err := IntervalCron(iv, "srv") + if err != nil { + t.Fatalf("%s: %v", iv, err) + } + if err := ValidateSchedule(c); err != nil { + t.Fatalf("%s -> %q: %v", iv, c, err) + } + } + a, _ := IntervalCron("1h", "srv-a") + again, _ := IntervalCron("1h", "srv-a") + if a != again { + t.Fatal("the minute must be stable per server") + } + if _, err := IntervalCron("5m", "x"); err == nil { + t.Fatal("5m is not offered") + } + if (FleetPolicy{Interval: "6h"}).IntervalDuration() != 6*time.Hour { + t.Fatal("duration") + } +} + +func TestExcludedMatches(t *testing.T) { + p := FleetPolicy{Excludes: []string{"app1", "app2/db", "app3/volumes", "omni_x"}} + for _, tc := range []struct { + app, name, container string + want bool + }{ + {"app1", "db", "app1-db", true}, + {"app1", "volumes", "", true}, + {"app2", "db", "app2-db", true}, + {"app2", "cache", "app2-cache", false}, + {"app3", "volumes", "", true}, + {"app3", "db", "app3-db", false}, + {AdoptedApp, "omni_x", "omni_x", true}, + {AdoptedApp, "other", "other", false}, + {"", "", "omni_x", true}, + } { + if got := p.Excluded(tc.app, tc.name, tc.container); got != tc.want { + t.Errorf("Excluded(%q,%q,%q) = %v", tc.app, tc.name, tc.container, got) + } + } +} + +func TestReadFleetPolicyFailsClosed(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "for p in", Err: errors.New("exit status 3: cannot read /etc/teploy/backup-policy.json")}) + if _, err := ReadFleetPolicy(context.Background(), mock); err == nil { + t.Fatal("an unreadable policy must not read as no policy") + } + var cfg S3Config + if err := ApplyPolicyEncryption(context.Background(), mock, &cfg); err == nil { + t.Fatal("an unreadable policy must refuse, not go on unencrypted") + } + mock = ssh.NewMockExecutor("h", ssh.MockCommand{Match: "for p in", Output: "path /etc/teploy/backup-policy.json\n" + `{"v":"1","recipient":"` + testOwnerRecipient + `"}`}) + if err := ApplyPolicyEncryption(context.Background(), mock, &cfg); err != nil || cfg.OwnerRecipient != testOwnerRecipient { + t.Fatalf("%v %+v", err, cfg) + } +} + +// ---- enumeration ---- + +func inventoryLine(name, image, state string, labels map[string]string, env ...string) string { + b, _ := json.Marshal(map[string]any{"name": "/" + name, "image": image, "state": state, "labels": labels, "env": env, "entrypoint": []string{}, "cmd": []string{}}) + return string(b) +} + +func TestEnumerateFleet(t *testing.T) { + labels := func(app, acc string) map[string]string { + return map[string]string{"teploy.role": "accessory", "teploy.app": app, "teploy.accessory": acc} + } + inv := strings.Join([]string{ + inventoryLine("a-db", "postgres:17", "running", labels("a", "db"), "PG_MAJOR=17"), + inventoryLine("b-db", "postgres:17", "running", labels("b", "db"), "PG_MAJOR=17"), + inventoryLine("c-db", "postgres:17", "running", labels("c", "db"), "PG_MAJOR=17"), + inventoryLine("wrongname", "postgres:17", "running", labels("d", "db"), "PG_MAJOR=17"), + inventoryLine("e-db", "postgres:17", "running", labels("e;x", "db"), "PG_MAJOR=17"), + inventoryLine("omni_postgres", "postgres:17", "running", nil, "PG_MAJOR=17"), + inventoryLine("stray", "redis:7", "running", nil, "REDIS_VERSION=7"), + inventoryLine("web", "nginx", "running", map[string]string{"teploy.role": "web", "teploy.app": "a"}), + inventoryLine("teploy-backup-verify-x", "postgres:17", "running", nil, "PG_MAJOR=17"), + }, "\n") + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: inv}, + ssh.MockCommand{Match: "crontab -l", Output: "0 3 * * * /usr/local/bin/teploy backup run --app b --accessory db # [teploy-accessory-backup:b:db]\n0 4 * * * /opt/hand.sh # [teploy-accessory-backup:c:db]\n"}, + ssh.MockCommand{Match: "cat", Output: ""}, + ssh.MockCommand{Match: "for d in /deployments/*/volumes", Output: "100\ta\n99999999\tbig\n50\tgone\n"}, + ) + p := FleetPolicy{Retention: FleetRetention{KeepLast: 3}, Volumes: true, VolumeMaxGB: 1, Excludes: []string{"gone/volumes"}, + Adopted: []AdoptedEntry{{Container: "omni_postgres", Databases: DatabaseSelector{All: true}}, {Container: "missing", Databases: DatabaseSelector{All: true}}}} + targets, uncovered, err := NewClient(mock, io.Discard).EnumerateFleet(context.Background(), p) + if err != nil { + t.Fatal(err) + } + if strings.Join(uncovered, ",") != "stray" { + t.Fatalf("uncovered = %v", uncovered) + } + byName := map[string]FleetTarget{} + for _, tr := range targets { + n := tr.Container + if n == "" { + n = tr.App + "/volumes" + } + byName[n] = tr + } + if tr := byName["a-db"]; tr.Status != "" || tr.Retention.KeepLast != 3 { + t.Fatalf("a-db: %+v", tr) + } + if tr := byName["b-db"]; tr.Status != TargetOwnCron { + t.Fatalf("a per-app schedule overrides the policy: %+v", tr) + } + if tr := byName["c-db"]; tr.Status != "" { + t.Fatalf("a hand-written job under a marker is no override: %+v", tr) + } + if tr := byName["wrongname"]; tr.Status != TargetFailed { + t.Fatalf("mislabelled: %+v", tr) + } + if tr := byName["e-db"]; tr.Status != TargetFailed { + t.Fatalf("hostile label: %+v", tr) + } + if tr := byName["omni_postgres"]; tr.Status != "" || tr.App != AdoptedApp || tr.Fact == nil { + t.Fatalf("adopted: %+v", tr) + } + if tr := byName["missing"]; tr.Status != TargetFailed || !strings.Contains(tr.Reason, "does not exist") { + t.Fatalf("an adopted container that is gone is a failure: %+v", tr) + } + if tr := byName["a/volumes"]; tr.Status != "" || tr.SizeBytes != 100*1024 { + t.Fatalf("volumes: %+v", tr) + } + if tr := byName["big/volumes"]; tr.Status != TargetExcluded || !tr.SizeGuard { + t.Fatalf("size guard: %+v", tr) + } + if tr := byName["gone/volumes"]; tr.Status != TargetExcluded || tr.SizeGuard { + t.Fatalf("exclude: %+v", tr) + } + if _, ok := byName["teploy-backup-verify-x"]; ok { + t.Fatal("a verify scratch is not a target") + } +} + +// A dead docker daemon must not read as an empty server: a run with no targets +// reports success. +func TestInventoryDockerDownIsAnError(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "ids=$(docker ps -aq", Err: errors.New("exit status 1: cannot connect to the docker daemon")}) + if _, err := NewClient(mock, io.Discard).Inventory(context.Background()); err == nil { + t.Fatal("must be an error") + } + if !strings.Contains(mock.Calls[0], "|| exit 1") || strings.Contains(mock.Calls[0], "| xargs") { + t.Fatalf("the status of docker ps must be checked: %s", mock.Calls[0]) + } +} + +func TestSummarizeFleet(t *testing.T) { + stored := []BackupObject{{Key: "k"}} + r := FleetRunResult{Targets: []FleetTargetResult{{Status: TargetOK, Container: "a", Objects: stored}, {Status: TargetExcluded, Container: "b", Reason: "excluded by policy"}}, UncoveredContainers: []string{"c"}} + SummarizeFleet(&r) + if r.Status != RunOK || len(r.Omissions) != 2 { + t.Fatalf("%+v", r) + } + // An ok target that stored nothing is named: "ok" alone would hide it. + r = FleetRunResult{Targets: []FleetTargetResult{{Status: TargetOK, Container: "empty"}}} + SummarizeFleet(&r) + if r.Status != RunOK || !strings.Contains(strings.Join(r.Omissions, ""), "nothing was stored") { + t.Fatalf("%+v", r) + } + r = FleetRunResult{Targets: []FleetTargetResult{{Status: TargetOK, Container: "a", Objects: stored, UncoveredDatabases: []string{"x"}}, {Status: TargetFailed, Container: "b"}}} + SummarizeFleet(&r) + if r.Status != RunPartial || !strings.Contains(strings.Join(r.Omissions, ""), "database x") { + t.Fatalf("%+v", r) + } + r = FleetRunResult{Targets: []FleetTargetResult{{Status: TargetFailed, Container: "b"}}} + SummarizeFleet(&r) + if r.Status != RunFailed { + t.Fatalf("%+v", r) + } +} + +// RunFleetTarget prunes nothing when the backup failed, and a failing prune does +// not fail what was backed up. +func TestRunFleetTargetNeverPrunesAFailedBackup(t *testing.T) { + mock := newStoreMock("h", + ssh.MockCommand{Match: "which aws", Err: errors.New("exit status 1")}, + ssh.MockCommand{Match: "aws s3", Output: ""}, + ) + r := NewClient(mock, io.Discard).RunFleetTarget(context.Background(), FleetTarget{Kind: TargetVolumes, App: "a", Retention: RetentionPolicy{KeepLast: 1}}, S3Config{Bucket: "b", Region: "r"}, BackupOptions{}) + if r.Status != TargetFailed || !strings.Contains(r.Reason, "aws CLI") { + t.Fatalf("%+v", r) + } + for _, c := range mock.Calls { + if strings.Contains(c, "s3 rm") { + t.Fatalf("pruned after a failed backup: %s", c) + } + } +} + +// ---- heartbeat ---- + +func TestHeartbeatSemantics(t *testing.T) { + var got []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = append(got, r.URL.RequestURI()) + if strings.Contains(r.URL.Path, "bad") { + w.WriteHeader(500) + } + })) + defer srv.Close() + up := SendHeartbeats(context.Background(), []string{srv.URL + "/api/push/TOKEN"}, true, nil) + if !up[0].OK || got[0] != "/api/push/TOKEN" { + t.Fatalf("%+v %v", up, got) + } + got = nil + down := SendHeartbeats(context.Background(), []string{srv.URL + "/api/push/TOKEN?x=1"}, false, nil) + if !down[0].OK || !strings.Contains(got[0], "status=down") || !strings.Contains(got[0], "x=1") { + t.Fatalf("%+v %v", down, got) + } + if u, _ := heartbeatURL("https://hc-ping.com/abc", false); u != "https://hc-ping.com/abc/fail" { + t.Fatalf("healthchecks: %s", u) + } + // Failures say so with no token anywhere, and do not panic or block. + dead := SendHeartbeats(context.Background(), []string{"http://127.0.0.1:1/SECRET", srv.URL + "/bad/SECRET"}, true, &http.Client{Timeout: time.Second}) + for _, r := range dead { + if r.OK || r.Error == "" { + t.Fatalf("%+v", r) + } + if strings.Contains(r.Error, "SECRET") || strings.Contains(r.Host, "SECRET") { + t.Fatalf("the token leaked: %+v", r) + } + } +} + +func TestMaskedHeartbeats(t *testing.T) { + got := MaskedHeartbeats([]string{"https://dash.example/api/push/TOKEN?x=y"}) + if got[0] != "https://dash.example/..." { + t.Fatal(got) + } +} + +// ---- run lock ---- + +func TestRunLockOverlapAndRelease(t *testing.T) { + p := filepath.Join(t.TempDir(), "lock") + now := time.Now() + l, _, err := AcquireRunLock(p, now, time.Hour) + if err != nil { + t.Fatal(err) + } + _, holder, err := AcquireRunLock(p, now, time.Hour) + if !errors.Is(err, ErrRunLocked) { + t.Fatalf("second lock: %v", err) + } + if holder.PID != os.Getpid() { + t.Fatalf("holder = %+v", holder) + } + // Past staleAfter, but the holder is not a teploy --all process: never killed. + if _, _, err := AcquireRunLock(p, now.Add(48*time.Hour), time.Hour); !errors.Is(err, ErrRunLocked) { + t.Fatalf("a recycled pid must not be killed: %v", err) + } + l.Release() + l2, _, err := AcquireRunLock(p, now, time.Hour) + if err != nil { + t.Fatalf("after release: %v", err) + } + l2.Release() + // A lock file left by a dead process blocks nothing: only a held flock does. + if err := os.WriteFile(p, []byte(`{"pid":1,"started_at":"2020-01-01T00:00:00Z"}`), 0o600); err != nil { + t.Fatal(err) + } + l3, _, err := AcquireRunLock(p, now, time.Hour) + if err != nil { + t.Fatalf("a stale file must not block: %v", err) + } + l3.Release() +} + +// ---- status ---- + +func TestFleetStatusCoverage(t *testing.T) { + labels := map[string]string{"teploy.role": "accessory", "teploy.app": "a", "teploy.accessory": "db"} + inv := strings.Join([]string{ + inventoryLine("a-db", "postgres:17", "exited", labels, "PG_MAJOR=17", "POSTGRES_PASSWORD=SECRETPW"), + inventoryLine("b_mysql", "mysql:8", "exited", nil, "MYSQL_VERSION=8", "MYSQL_ROOT_PASSWORD=SECRETPW"), + inventoryLine("stray", "redis:7", "running", nil, "REDIS_VERSION=7"), + }, "\n") + now := mustTime(t, "20260201-120000") + evA := `{"v":1,"id":"1","seq":1,"ts":"2026-02-01T11:00:00Z","app":"a","kind":"backup.run","outcome":"succeeded","detail":{"accessory":"db","kind":"accessory"}}` + evAd := `{"v":1,"id":"2","seq":1,"ts":"2026-01-01T11:00:00Z","app":"_adopted","kind":"backup.run","outcome":"failed","detail":{"accessory":"b_mysql","kind":"accessory","error":"boom"}}` + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "path /etc/teploy/backup-policy.json\n" + `{"v":"1","interval":"1h","recipient":"` + testOwnerRecipient + `","heartbeat_urls":["https://d/api/push/SECRETTOKEN"],"adopted":[{"container":"b_mysql","databases":"all"},{"container":"gone","databases":"all"}]}`}, + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"bk","region":"r","prefix":"p","credentials":"SECRETARMOR"}`}, + ssh.MockCommand{Match: "command -v aws >/dev/null", Output: "aws\nage\n"}, + ssh.MockCommand{Match: "crontab -l", Output: "5 * * * * /usr/local/bin/teploy backup run --all # [teploy-backup-all]\n"}, + ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: inv}, + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "( c=$(python3", Output: "2026-02-01 11:00:00 1000 p/a/accessories/db/20260201-110000.pgdump.age\n"}, + ssh.MockCommand{Match: "python3 -c", Output: evA + "\n" + evAd + "\n"}, + ssh.MockCommand{Match: "for d in /deployments/*/volumes", Output: ""}, + ssh.MockCommand{Match: "for d in /var/tmp/teploy-backup.", Output: "/var/tmp/teploy-backup.old\n"}, + ) + st, err := ReadFleetStatus(context.Background(), mock, now) + if err != nil { + t.Fatal(err) + } + b, _ := json.Marshal(st) + for _, secretValue := range []string{"SECRETPW", "SECRETTOKEN", "SECRETARMOR"} { + if strings.Contains(string(b), secretValue) { + t.Fatalf("status leaks %s: %s", secretValue, b) + } + } + entry := map[string]StatusEntry{} + for _, e := range st.Entries { + entry[e.Name] = e + } + if e := entry["a-db"]; e.Coverage != CoverageCovered || e.Schedule != "policy" || e.Newest == nil || !e.Newest.Encrypted { + t.Fatalf("a-db: %+v", e) + } + if e := entry["stray"]; e.Coverage != CoverageUncovered || !e.Unmanaged || e.Kind != "unmanaged" { + t.Fatalf("stray: %+v", e) + } + if e := entry["gone"]; e.Coverage != CoverageFailing || e.State != "missing" { + t.Fatalf("gone: %+v", e) + } + if !st.Policy.CronInstalled || st.Policy.HeartbeatCount != 1 || !st.Encrypted || st.Target.Prefix != "p" || !st.Target.HasCredentials { + t.Fatalf("summary: %+v %+v", st.Policy, st.Target) + } + if len(st.LeftoverWorkspaces) != 1 { + t.Fatalf("leftovers: %v", st.LeftoverWorkspaces) + } + if strings.Join(st.UncoveredDatabases, ",") != "stray" { + t.Fatalf("uncovered: %v", st.UncoveredDatabases) + } + if e := entry["b_mysql"]; e.Coverage != CoverageFailing || !e.Adopted || e.LastError != "boom" { + t.Fatalf("a failed adopted container reads failing, adopted: %+v", e) + } + // Past twice the interval with no new success. + later, err := ReadFleetStatus(context.Background(), mock, now.Add(3*time.Hour)) + if err != nil { + t.Fatal(err) + } + for _, e := range later.Entries { + if e.Name == "a-db" && e.Coverage != CoverageStale { + t.Fatalf("stale: %+v", e) + } + } +} + +func TestEntryObjectsAndEvents(t *testing.T) { + var none StatusEntry + fillEntryEvents(&none, nil, "x") + if none.LastSuccess != nil { + t.Fatal("no events, no success") + } + e := StatusEntry{} + fillEntryObjects(&e, []ListedBackup{{Kind: KindAccessory, Accessory: "db", Restorable: false, CreatedAt: "2026-02-01T11:00:00Z", ID: "x"}}, "db") + if e.Newest != nil { + t.Fatal("a non-restorable object must not count as the newest") + } + fillEntryObjects(&e, []ListedBackup{ + {Kind: KindAccessory, Accessory: "db", Database: "b", Restorable: true, CreatedAt: "2026-02-01T11:00:00Z", ID: "x"}, + {Kind: KindAccessory, Accessory: "db", Database: "a", Restorable: true, CreatedAt: "2026-01-01T11:00:00Z", ID: "y"}, + {Kind: KindAccessory, Accessory: "db", Database: "a", Restorable: true, CreatedAt: "2025-01-01T11:00:00Z", ID: "z"}, + }, "db") + if e.Newest == nil || e.Newest.ID != "x" || len(e.Databases) != 2 || e.Databases[0].Name != "a" || e.Databases[0].Newest.ID != "y" { + t.Fatalf("%+v", e) + } + if e.LastSuccess == nil { + t.Fatal("a good object is evidence of a success") + } +} diff --git a/internal/backup/policy.go b/internal/backup/policy.go new file mode 100644 index 0000000..14dae91 --- /dev/null +++ b/internal/backup/policy.go @@ -0,0 +1,376 @@ +package backup + +import ( + "context" + "encoding/json" + "fmt" + "hash/fnv" + "net/url" + "regexp" + "strings" + "time" + + "github.com/useteploy/teploy/internal/ssh" +) + +// The server backup policy (G4): /etc/teploy/backup-policy.json, read at run +// time by the one cron line `teploy backup run --all`, so a database deployed +// tomorrow is covered with no command. Mode 0600: heartbeat URLs carry tokens. +// A non-root user that cannot write /etc/teploy falls back to +// ~/.config/teploy/backup-policy.json, like the backup target. + +// Policy paths. +const ( + PolicyFileEtc = "/etc/teploy/backup-policy.json" + PolicyFileHome = ".config/teploy/backup-policy.json" +) + +// FleetRetention is the policy's retention half (see RetentionPolicy). +type FleetRetention struct { + KeepLast int `json:"keep_last,omitempty"` + KeepDaily int `json:"keep_daily,omitempty"` + KeepWeekly int `json:"keep_weekly,omitempty"` + KeepMonthly int `json:"keep_monthly,omitempty"` + MaxAgeDays int `json:"max_age_days,omitempty"` +} + +// Policy converts to the prune selection's form. +func (r FleetRetention) Policy() RetentionPolicy { + return RetentionPolicy{KeepLast: r.KeepLast, KeepDaily: r.KeepDaily, KeepWeekly: r.KeepWeekly, KeepMonthly: r.KeepMonthly, MaxAgeDays: r.MaxAgeDays} +} + +// DatabaseSelector is an adopted entry's databases: "all", or a list of names. +type DatabaseSelector struct { + All bool + Names []string +} + +// MarshalJSON renders "all" or the list. +func (d DatabaseSelector) MarshalJSON() ([]byte, error) { + if d.All { + return json.Marshal("all") + } + if d.Names == nil { + return json.Marshal([]string{}) + } + return json.Marshal(d.Names) +} + +// UnmarshalJSON accepts "all" or a list of names. +func (d *DatabaseSelector) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err == nil { + if s != "all" { + return fmt.Errorf(`databases must be "all" or a list of names (got %q)`, s) + } + *d = DatabaseSelector{All: true} + return nil + } + var names []string + if err := json.Unmarshal(b, &names); err != nil { + return fmt.Errorf(`databases must be "all" or a list of names`) + } + *d = DatabaseSelector{Names: names} + return nil +} + +// AdoptedEntry brings a container with no teploy labels under the policy. +type AdoptedEntry struct { + Container string `json:"container"` + Engine string `json:"engine,omitempty"` + Databases DatabaseSelector `json:"databases"` +} + +// FleetPolicy is the server's backup policy. +type FleetPolicy struct { + V string `json:"v"` + // Interval is how often `backup run --all` fires. Empty: no cron line (the + // policy still configures encryption, retention and manual runs). + Interval string `json:"interval,omitempty"` + Retention FleetRetention `json:"retention"` + // Excludes are targets the run leaves out, visibly: "" (everything of + // it), "/", "/volumes", or a container name. + Excludes []string `json:"excludes,omitempty"` + Adopted []AdoptedEntry `json:"adopted,omitempty"` + // Volumes turns on app volume archives (each one a full copy). + Volumes bool `json:"volumes"` + // VolumeMaxGB caps one archive: a bigger tree is warned about and excluded. + VolumeMaxGB float64 `json:"volume_max_gb,omitempty"` + // HeartbeatURLs are pinged after every run. They carry tokens: never logged, + // never shown back. + HeartbeatURLs []string `json:"heartbeat_urls,omitempty"` + // Recipient is the owner's recovery public key (`teploy backup keygen`). + // With one, every object is encrypted to it and the server's own identity. + Recipient string `json:"recipient,omitempty"` + + // Path is where the policy was read from (diagnostics only). + Path string `json:"-"` +} + +// Configured reports whether the server has a policy at all. +func (p FleetPolicy) Configured() bool { return p.V != "" } + +// Validate checks everything the run later interpolates or depends on. It is +// also the gate on what reaches the server's disk. +func (p FleetPolicy) Validate() error { + if p.Interval != "" { + if _, err := IntervalCron(p.Interval, "x"); err != nil { + return err + } + } + r := p.Retention + if r.KeepLast < 0 || r.KeepDaily < 0 || r.KeepWeekly < 0 || r.KeepMonthly < 0 || r.MaxAgeDays < 0 { + return fmt.Errorf("retention values must be >= 0") + } + if p.VolumeMaxGB < 0 { + return fmt.Errorf("volume_max_gb must be >= 0") + } + if p.Recipient != "" { + if err := ValidateRecipient(p.Recipient); err != nil { + return err + } + } + for _, e := range p.Excludes { + if err := validateExclude(e); err != nil { + return err + } + } + seen := map[string]bool{} + for _, a := range p.Adopted { + if err := ValidateContainerName(a.Container); err != nil { + return err + } + if seen[a.Container] { + return fmt.Errorf("container %s is adopted twice", a.Container) + } + seen[a.Container] = true + if a.Engine != "" && (!knownEngines[a.Engine] || a.Engine == EngineGeneric || a.Engine == EngineNucleus) { + return fmt.Errorf("adopted %s: engine %q cannot be adopted (postgres, mysql, mariadb, redis, valkey, dragonfly, mongo)", a.Container, a.Engine) + } + if !a.Databases.All && len(a.Databases.Names) == 0 { + return fmt.Errorf(`adopted %s: databases must be "all" or at least one name`, a.Container) + } + for _, db := range a.Databases.Names { + if !addressableDB(db) { + return fmt.Errorf("adopted %s: database name %q cannot be addressed safely", a.Container, db) + } + } + } + for _, u := range p.HeartbeatURLs { + if err := validateHeartbeatURL(u); err != nil { + return err + } + } + return nil +} + +var containerNameRE = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,254}$`) + +// ValidateContainerName applies docker's own name grammar. +func ValidateContainerName(name string) error { + if !containerNameRE.MatchString(name) { + return fmt.Errorf("invalid container name %q", name) + } + return nil +} + +func validateExclude(e string) error { + for _, part := range strings.Split(e, "/") { + if !containerNameRE.MatchString(part) { + return fmt.Errorf("invalid exclude %q: expected , /, /volumes or a container name", e) + } + } + if strings.Count(e, "/") > 1 { + return fmt.Errorf("invalid exclude %q: expected , /, /volumes or a container name", e) + } + return nil +} + +func validateHeartbeatURL(raw string) error { + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + // Never the value: the URL is a secret and this reaches logs. + return fmt.Errorf("heartbeat URLs must be http(s) URLs") + } + return nil +} + +// Excluded reports whether the policy leaves a target out. +func (p FleetPolicy) Excluded(app, name, container string) bool { + for _, e := range p.Excludes { + switch { + case e == container && container != "": + return true + case app != "" && app != AdoptedApp && e == app: + return true + case app != "" && e == app+"/"+name: + return true + } + } + return false +} + +var intervalMinutes = map[string]int{ + "15m": 15, "30m": 30, "1h": 60, "hourly": 60, "2h": 120, "3h": 180, "4h": 240, + "6h": 360, "8h": 480, "12h": 720, "24h": 1440, "1d": 1440, "daily": 1440, + "7d": 10080, "1w": 10080, "weekly": 10080, +} + +// IntervalDuration is the policy interval as a duration (0 when unset). +func (p FleetPolicy) IntervalDuration() time.Duration { + return time.Duration(intervalMinutes[p.Interval]) * time.Minute +} + +// IntervalCron renders an interval as five cron fields. seed (the server's +// identity) picks the minute, so a fleet does not hit its bucket in one second. +func IntervalCron(interval, seed string) (string, error) { + mins, ok := intervalMinutes[interval] + if !ok { + return "", fmt.Errorf("invalid interval %q: use 15m, 30m, 1h, 2h, 3h, 4h, 6h, 8h, 12h, 24h, 7d (or hourly, daily, weekly)", interval) + } + h := fnv.New32a() + h.Write([]byte(seed)) + v := h.Sum32() + m := int(v % 60) + hour := int((v >> 8) % 6) // the small hours + switch { + case mins < 60: + return fmt.Sprintf("%d-59/%d * * * *", m%mins, mins), nil + case mins == 60: + return fmt.Sprintf("%d * * * *", m), nil + case mins < 1440: + return fmt.Sprintf("%d %d-23/%d * * *", m, hour%(mins/60), mins/60), nil + case mins == 1440: + return fmt.Sprintf("%d %d * * *", m, hour), nil + } + return fmt.Sprintf("%d %d * * 0", m, hour), nil +} + +// readPolicyScript prints "absent" or "path

\n". A file that exists +// and cannot be read is a failure, not an absence: a non-root CLI user reading +// "no policy" from a root-owned file would upload plaintext. +const readPolicyScript = `for p in /etc/teploy/backup-policy.json "$HOME"/.config/teploy/backup-policy.json; do +if [ -e "$p" ] && [ ! -r "$p" ]; then echo "cannot read $p: run as its owner" >&2; exit 3; fi +if [ -r "$p" ]; then printf 'path %s\n' "$p"; cat "$p"; exit 0; fi +done +echo absent` + +// ReadFleetPolicy loads the server's policy. No policy is (FleetPolicy{}, nil). +func ReadFleetPolicy(ctx context.Context, exec ssh.Executor) (FleetPolicy, error) { + res := ssh.RunDetailed(ctx, exec, readPolicyScript) + if res.Err != nil { + return FleetPolicy{}, fmt.Errorf("reading the backup policy: %w", res.Err) + } + if res.ExitCode != 0 { + return FleetPolicy{}, fmt.Errorf("reading the backup policy: %s", res.ExitErrorText()) + } + out := strings.TrimSpace(string(res.Stdout)) + if out == "absent" { + return FleetPolicy{}, nil + } + line, rest, _ := strings.Cut(out, "\n") + if !strings.HasPrefix(line, "path ") { + return FleetPolicy{}, fmt.Errorf("reading the backup policy: unrecognized output framing") + } + var p FleetPolicy + p.Path = strings.TrimSpace(strings.TrimPrefix(line, "path ")) + if err := json.Unmarshal([]byte(rest), &p); err != nil { + return FleetPolicy{}, fmt.Errorf("parsing %s: %w", p.Path, err) + } + if p.V != "1" { + return FleetPolicy{}, fmt.Errorf("%s: unsupported backup policy version %q", p.Path, p.V) + } + return p, nil +} + +const writeFleetPolicyScript = `d= +if [ "$(id -u)" = 0 ]; then + mkdir -p /etc/teploy && chmod 700 /etc/teploy || exit 1 + d=/etc/teploy/backup-policy.json +elif [ -w /etc/teploy ] 2>/dev/null; then + d=/etc/teploy/backup-policy.json +else + dir=${XDG_CONFIG_HOME:-$HOME/.config}/teploy + mkdir -p "$dir" && chmod 700 "$dir" || exit 1 + d=$dir/backup-policy.json + echo fallback >&2 +fi +umask 077 +t=$(mktemp "$(dirname "$d")/.backup-policy.XXXXXXXX") || exit 1 +cat > "$t" || { rm -f -- "$t"; exit 1; } +chmod 600 "$t" || { rm -f -- "$t"; exit 1; } +mv -f -- "$t" "$d" || { rm -f -- "$t"; exit 1; } +printf 'written %s\n' "$d"` + +// WriteFleetPolicy stores the policy atomically at 0600 and returns the path and +// whether the non-root fallback was used. +func WriteFleetPolicy(ctx context.Context, exec ssh.Executor, p FleetPolicy) (string, bool, error) { + if err := p.Validate(); err != nil { + return "", false, err + } + p.V = "1" + payload, err := json.Marshal(p) + if err != nil { + return "", false, err + } + res := ssh.RunInputDetailed(ctx, exec, writeFleetPolicyScript, strings.NewReader(string(payload))) + if res.Err != nil { + return "", false, fmt.Errorf("writing the backup policy: %w", res.Err) + } + if res.ExitCode != 0 { + return "", false, fmt.Errorf("writing the backup policy: %s", res.ExitErrorText()) + } + f := strings.Fields(strings.TrimSpace(string(res.Stdout))) + if len(f) != 2 || f[0] != "written" { + return "", false, fmt.Errorf("writing the backup policy: unrecognized output framing") + } + return f[1], strings.Contains(string(res.Stderr), "fallback"), nil +} + +const unsetFleetPolicyScript = `for p in /etc/teploy/backup-policy.json "$HOME"/.config/teploy/backup-policy.json; do +if [ -e "$p" ]; then rm -f -- "$p" || exit 1; echo "removed $p"; fi +done` + +// UnsetFleetPolicy removes the policy file(s), returning what it removed. +func UnsetFleetPolicy(ctx context.Context, exec ssh.Executor) ([]string, error) { + res := ssh.RunDetailed(ctx, exec, unsetFleetPolicyScript) + if res.Err != nil { + return nil, fmt.Errorf("removing the backup policy: %w", res.Err) + } + if res.ExitCode != 0 { + return nil, fmt.Errorf("removing the backup policy: %s", res.ExitErrorText()) + } + var removed []string + for _, line := range splitLines(string(res.Stdout)) { + if v, ok := strings.CutPrefix(line, "removed "); ok { + removed = append(removed, v) + } + } + return removed, nil +} + +// ApplyPolicyEncryption reads the policy and arms cfg with its recovery key, so +// every object written with cfg is encrypted. An unreadable policy fails closed: +// better no backup now than a plaintext one on a server configured to encrypt. +func ApplyPolicyEncryption(ctx context.Context, exec ssh.Executor, cfg *S3Config) error { + p, err := ReadFleetPolicy(ctx, exec) + if err != nil { + return err + } + cfg.OwnerRecipient = p.Recipient + return nil +} + +// MaskedHeartbeats renders heartbeat URLs without their tokens: scheme and host. +func MaskedHeartbeats(urls []string) []string { + out := []string{} + for _, raw := range urls { + u, err := url.Parse(raw) + if err != nil { + out = append(out, "(unparseable)") + continue + } + out = append(out, u.Scheme+"://"+u.Host+"/...") + } + return out +} diff --git a/internal/backup/runlock_other.go b/internal/backup/runlock_other.go new file mode 100644 index 0000000..a15d08e --- /dev/null +++ b/internal/backup/runlock_other.go @@ -0,0 +1,28 @@ +//go:build windows + +package backup + +import ( + "errors" + "time" +) + +// RunLockHolder describes the process that holds the overlap lock. +type RunLockHolder struct { + PID int + StartedAt time.Time +} + +// RunLock is unavailable here: `backup run --all` runs on Linux servers. +type RunLock struct{} + +// ErrRunLocked means another run holds the lock. +var ErrRunLocked = errors.New("another `teploy backup run --all` is running") + +// AcquireRunLock refuses: this is a server-side command. +func AcquireRunLock(path string, now time.Time, staleAfter time.Duration) (*RunLock, RunLockHolder, error) { + return nil, RunLockHolder{}, errors.New("backup run --all runs on the server (Linux)") +} + +// Release is a no-op. +func (l *RunLock) Release() {} diff --git a/internal/backup/runlock_unix.go b/internal/backup/runlock_unix.go new file mode 100644 index 0000000..8aa49f2 --- /dev/null +++ b/internal/backup/runlock_unix.go @@ -0,0 +1,104 @@ +//go:build !windows + +package backup + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "strconv" + "strings" + "syscall" + "time" +) + +// RunLockHolder describes the process that holds the overlap lock. +type RunLockHolder struct { + PID int `json:"pid"` + StartedAt time.Time `json:"started_at"` +} + +// RunLock is the overlap lock of `backup run --all`: an flock on a file. The +// kernel drops it the moment the process dies, however it died, so a killed run +// cannot block the next one, and Go opens the file close-on-exec so no dump or +// upload child inherits it. What the lock cannot release by itself is a holder +// that is alive and stuck: past staleAfter (longer than the run's own deadline, +// which should have ended it) the holder is killed, after checking that the pid +// is still a teploy process. +type RunLock struct { + f *os.File +} + +// ErrRunLocked means another run holds the lock. +var ErrRunLocked = errors.New("another `teploy backup run --all` is running") + +// AcquireRunLock takes the lock at path. When held it returns ErrRunLocked and +// the holder. killStale says what to do with a holder past staleAfter. +func AcquireRunLock(path string, now time.Time, staleAfter time.Duration) (*RunLock, RunLockHolder, error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, RunLockHolder{}, fmt.Errorf("opening the run lock %s: %w", path, err) + } + try := func() error { return syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) } + err = try() + if err != nil { + holder := readRunLockHolder(path) + if !errors.Is(err, syscall.EWOULDBLOCK) { + f.Close() + return nil, holder, fmt.Errorf("locking %s: %w", path, err) + } + if holder.PID <= 0 || holder.StartedAt.IsZero() || now.Sub(holder.StartedAt) <= staleAfter || !isTeployProcess(holder.PID) { + f.Close() + return nil, holder, ErrRunLocked + } + // Alive, stuck and past every deadline it had: it will never let go. + if killErr := syscall.Kill(holder.PID, syscall.SIGKILL); killErr != nil { + f.Close() + return nil, holder, fmt.Errorf("%w (pid %d has held it since %s and could not be stopped: %v)", ErrRunLocked, holder.PID, holder.StartedAt.Format(time.RFC3339), killErr) + } + for i := 0; i < 50; i++ { + if err = try(); err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + if err != nil { + f.Close() + return nil, holder, ErrRunLocked + } + } + if err := f.Truncate(0); err == nil { + body, _ := json.Marshal(RunLockHolder{PID: os.Getpid(), StartedAt: now}) + f.WriteAt(body, 0) + f.Sync() + } + return &RunLock{f: f}, RunLockHolder{}, nil +} + +// Release drops the lock (and exiting does too). +func (l *RunLock) Release() { + if l == nil || l.f == nil { + return + } + l.f.Truncate(0) + l.f.Close() + l.f = nil +} + +func readRunLockHolder(path string) RunLockHolder { + var h RunLockHolder + if b, err := os.ReadFile(path); err == nil { + _ = json.Unmarshal(b, &h) + } + return h +} + +// isTeployProcess guards the kill against a recycled pid. +func isTeployProcess(pid int) bool { + b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/cmdline") + if err != nil { + return false + } + return strings.Contains(string(b), "teploy") && strings.Contains(string(b), "--all") +} diff --git a/internal/backup/status.go b/internal/backup/status.go new file mode 100644 index 0000000..e8e786e --- /dev/null +++ b/internal/backup/status.go @@ -0,0 +1,508 @@ +package backup + +import ( + "context" + "fmt" + "io" + "sort" + "strings" + "time" + + "github.com/useteploy/teploy/internal/events" + "github.com/useteploy/teploy/internal/ssh" +) + +// Fleet status: one read-only call Dash's backups page builds on, and the facts +// doctor checks. Env is read for detection and never reported; heartbeat URLs +// appear only as a count and hosts; credentials only as a boolean. + +// Coverage states. +const ( + CoverageCovered = "covered" + CoverageStale = "stale" + CoverageFailing = "failing" + CoverageUncovered = "uncovered" + CoverageExcluded = "excluded" + CoverageAdopted = "adopted" +) + +// StatusObject is the newest good object of a series. +type StatusObject struct { + ID string `json:"id"` + Key string `json:"key"` + SizeBytes int64 `json:"size_bytes"` + CreatedAt string `json:"created_at"` + Encrypted bool `json:"encrypted"` + Previous bool `json:"previous_prefix,omitempty"` +} + +// StatusVerify is the last verify-backup of a target. +type StatusVerify struct { + At time.Time `json:"at"` + OK bool `json:"ok"` + ID string `json:"id,omitempty"` + Metric string `json:"metric,omitempty"` + // Database is set when the verify was of one database's series. + Database string `json:"database,omitempty"` +} + +// StatusDatabase is one database with its own series. +type StatusDatabase struct { + Name string `json:"name"` + Newest *StatusObject `json:"newest,omitempty"` +} + +// StatusEntry is one DB-like container, or one app's volumes. +type StatusEntry struct { + Name string `json:"name"` + Kind string `json:"kind"` // accessory | adopted | unmanaged | volumes + App string `json:"app,omitempty"` + Accessory string `json:"accessory,omitempty"` + Engine string `json:"engine,omitempty"` + Detection string `json:"detection,omitempty"` + State string `json:"state,omitempty"` + // Unmanaged: no teploy labels. Adopted: a policy entry covers it anyway. + Unmanaged bool `json:"unmanaged"` + Adopted bool `json:"adopted"` + // Coverage: covered | stale | failing | uncovered | excluded | adopted. + Coverage string `json:"coverage"` + Reason string `json:"reason,omitempty"` + // Schedule: policy | own (a per-app cron line) | none. + Schedule string `json:"schedule"` + LastSuccess *time.Time `json:"last_success,omitempty"` + LastFailure *time.Time `json:"last_failure,omitempty"` + // LastError is the last failure's message (event-log bounded, no secrets). + LastError string `json:"last_error,omitempty"` + Newest *StatusObject `json:"newest,omitempty"` + Databases []StatusDatabase `json:"databases,omitempty"` + LastVerify *StatusVerify `json:"last_verify,omitempty"` + UncoveredDatabases []string `json:"uncovered_databases,omitempty"` + // SizeBytes is the measured tree (volumes, generic accessories). + SizeBytes int64 `json:"size_bytes,omitempty"` +} + +// StatusTarget summarizes the stored target (never credentials). +type StatusTarget struct { + Configured bool `json:"configured"` + Bucket string `json:"bucket,omitempty"` + Region string `json:"region,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Prefix string `json:"prefix,omitempty"` + HasCredentials bool `json:"has_credentials"` + PreviousPrefixes []PreviousPrefix `json:"previous_prefixes,omitempty"` + // Reachable says the listing this report needed worked. + Reachable bool `json:"reachable"` + Error string `json:"error,omitempty"` +} + +// StatusPolicy summarizes the policy (no heartbeat tokens). +type StatusPolicy struct { + Configured bool `json:"configured"` + Path string `json:"path,omitempty"` + Interval string `json:"interval,omitempty"` + CronInstalled bool `json:"cron_installed"` + Retention FleetRetention `json:"retention"` + Excludes []string `json:"excludes,omitempty"` + Adopted []AdoptedEntry `json:"adopted,omitempty"` + Volumes bool `json:"volumes"` + VolumeMaxGB float64 `json:"volume_max_gb,omitempty"` + HeartbeatCount int `json:"heartbeat_count"` + HeartbeatHosts []string `json:"heartbeat_hosts,omitempty"` + // Recipient is the owner's recovery PUBLIC key. + Recipient string `json:"recipient,omitempty"` +} + +// StatusLastRun is the last `backup run --all`. +type StatusLastRun struct { + At time.Time `json:"at"` + Status string `json:"status"` + TargetsOK int `json:"targets_ok"` + TargetsFailed int `json:"targets_failed"` + Omissions []string `json:"omissions,omitempty"` + Error string `json:"error,omitempty"` + Stale bool `json:"stale"` +} + +// FleetStatus is the whole report. +type FleetStatus struct { + Target StatusTarget `json:"target"` + Policy StatusPolicy `json:"policy"` + // Encrypted says new objects will be encrypted (a recovery key is set). + Encrypted bool `json:"encrypted"` + AWSCLI bool `json:"aws_cli"` + AgeCLI bool `json:"age_cli"` + // RunLock names a run in progress. + LastRun *StatusLastRun `json:"last_run,omitempty"` + Entries []StatusEntry `json:"entries"` + // UncoveredDatabases lists every database, across containers, no backup + // holds: "/", or "" when nothing covers it. + UncoveredDatabases []string `json:"uncovered_databases"` + // LeftoverWorkspaces are temp directories killed runs left. + LeftoverWorkspaces []string `json:"leftover_workspaces,omitempty"` + // LeftoverUploads are bucket keys of uploads a killed run never published. + LeftoverUploads []string `json:"leftover_uploads,omitempty"` + Warnings []string `json:"warnings,omitempty"` +} + +// ReadFleetStatus builds the report. A failure to read one thing (the bucket, +// one container's databases) is reported in the report; only a failure to read +// the server itself is an error. +func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (FleetStatus, error) { + c := NewClient(exec, io.Discard) + st := FleetStatus{Entries: []StatusEntry{}, UncoveredDatabases: []string{}} + + policy, err := ReadFleetPolicy(ctx, exec) + if err != nil { + return st, err + } + target, err := ReadTarget(ctx, exec) + if err != nil { + return st, err + } + st.Policy = StatusPolicy{ + Configured: policy.Configured(), Path: policy.Path, Interval: policy.Interval, Retention: policy.Retention, + Excludes: policy.Excludes, Adopted: policy.Adopted, Volumes: policy.Volumes, VolumeMaxGB: policy.VolumeMaxGB, + HeartbeatCount: len(policy.HeartbeatURLs), Recipient: policy.Recipient, + } + for _, h := range MaskedHeartbeats(policy.HeartbeatURLs) { + st.Policy.HeartbeatHosts = append(st.Policy.HeartbeatHosts, strings.TrimSuffix(h, "/...")) + } + st.Encrypted = policy.Recipient != "" + st.Target = StatusTarget{ + Configured: target.Bucket != "", Bucket: target.Bucket, Region: target.Region, Endpoint: target.Endpoint, + Prefix: target.Prefix, HasCredentials: target.HasCredentials, PreviousPrefixes: target.PreviousPrefixes, + } + if out, err := exec.Run(ctx, "command -v aws >/dev/null 2>&1 && echo aws; command -v age >/dev/null 2>&1 && echo age; true"); err == nil { + for _, t := range splitLines(out) { + st.AWSCLI = st.AWSCLI || t == "aws" + st.AgeCLI = st.AgeCLI || t == "age" + } + } + ownCron := map[string]bool{} + if cron, err := ReadCrontab(ctx, exec); err == nil { + for _, line := range cron { + if m := FindCronMarker(line); m != nil { + if m.Kind == KindFleet { + st.Policy.CronInstalled = true + } else if m.Form != CronFormCustom { + ownCron[m.App+"/"+m.Accessory] = true + } + } + } + } + facts, err := c.Inventory(ctx) + if err != nil { + return st, err + } + adopted := map[string]*AdoptedEntry{} + for i := range policy.Adopted { + adopted[policy.Adopted[i].Container] = &policy.Adopted[i] + } + + // One listing and one event log per namespace. + var cfg S3Config + if target.Bucket != "" { + region := target.Region + if region == "" { + region = "us-east-1" + } + cfg = S3Config{Bucket: target.Bucket, Region: region, Endpoint: target.Endpoint, Prefix: target.Prefix, StoredTarget: target.Path, PreviousPrefixes: target.PreviousPrefixes} + } + if target.Bucket != "" && st.AWSCLI { + if err := c.ProbeTarget(ctx, cfg); err != nil { + st.Target.Error = oneLine(err.Error()) + } else { + st.Target.Reachable = true + } + } + listed := map[string][]ListedBackup{} + listNS := func(app string) []ListedBackup { + if rows, ok := listed[app]; ok { + return rows + } + var rows []ListedBackup + if st.Target.Reachable { + lctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + r, pending, err := c.listBackupsAndPending(lctx, app, cfg) + if err != nil { + st.Target.Reachable = false + st.Target.Error = oneLine(err.Error()) + } + rows = r + for _, k := range pending { + segs := strings.Split(k, "/") + if e, ok := newBackupEntry(segs[len(segs)-1]); ok && now.Sub(e.time) > tmpLeftoverMinutes*time.Minute { + st.LeftoverUploads = append(st.LeftoverUploads, k) + } + } + } + listed[app] = rows + return rows + } + logs := map[string][]events.Event{} + logNS := func(app string) []events.Event { + if evs, ok := logs[app]; ok { + return evs + } + evs, err := events.Read(ctx, exec, app, 0) + if err != nil { + st.Warnings = append(st.Warnings, fmt.Sprintf("could not read the %s event log: %v", app, err)) + } + logs[app] = evs + return evs + } + + stale := 2 * policy.IntervalDuration() + decide := func(e *StatusEntry, excluded bool, covered bool) { + switch { + case excluded: + e.Coverage = CoverageExcluded + case !covered: + e.Coverage = CoverageUncovered + case e.LastFailure != nil && (e.LastSuccess == nil || e.LastFailure.After(*e.LastSuccess)): + e.Coverage = CoverageFailing + case e.LastSuccess == nil: + e.Coverage, e.Reason = CoverageStale, "no backup has succeeded yet" + default: + limit := stale + if e.Schedule == "own" || limit == 0 { + limit = 48 * time.Hour + } + if now.Sub(*e.LastSuccess) > limit { + e.Coverage, e.Reason = CoverageStale, fmt.Sprintf("the last success is older than %s", limit) + } else if e.Adopted { + e.Coverage = CoverageAdopted + } else { + e.Coverage = CoverageCovered + } + } + } + + for i := range facts { + f := &facts[i] + if isScratch(f.Name) { + continue + } + a, isAdopted := adopted[f.Name] + isAccessory := f.Role == "accessory" && f.App != "" && f.Accessory != "" && f.Name == ContainerName(f.App, f.Accessory) + if !isAdopted && !isAccessory && (f.EngineErr != nil || !f.Engine.IsDB()) { + continue + } + e := StatusEntry{Name: f.Name, Engine: f.Engine.Engine, Detection: f.Engine.Source, State: f.State, Schedule: "none", Unmanaged: f.Unmanaged(), Adopted: isAdopted} + var app, name string + switch { + case isAdopted: + e.Kind, app, name = TargetAdopted, AdoptedApp, f.Name + if a.Engine != "" { + e.Engine, e.Detection = a.Engine, "policy" + } + e.Accessory = f.Name + case isAccessory: + e.Kind, app, name = TargetAccessory, f.App, f.Accessory + e.App, e.Accessory = f.App, f.Accessory + default: + e.Kind = "unmanaged" + } + excluded := policy.Excluded(app, name, f.Name) + covered := false + switch { + case isAdopted: + covered, e.Schedule = policy.Configured(), "policy" + case isAccessory && ownCron[f.App+"/"+f.Accessory]: + covered, e.Schedule = true, "own" + case isAccessory && policy.Configured() && policy.Interval != "": + covered, e.Schedule = true, "policy" + } + if app != "" { + fillEntryEvents(&e, logNS(app), name) + fillEntryObjects(&e, listNS(app), name) + } + decide(&e, excluded, covered) + if e.Coverage == CoverageUncovered && e.Reason == "" { + if e.Kind == "unmanaged" { + e.Reason = "no policy entry covers this container (adopt it or exclude it)" + } else { + e.Reason = "no backup policy or schedule covers this accessory" + } + } + if e.Coverage == CoverageUncovered { + st.UncoveredDatabases = append(st.UncoveredDatabases, f.Name) + } + // What the server holds beyond what this entry's backups cover. + if !excluded && covered && f.State == "running" && app != "" { + opts := BackupOptions{AllDatabases: e.Schedule == "policy"} + if isAdopted && !a.Databases.All { + opts = BackupOptions{Databases: a.Databases.Names} + } + e.UncoveredDatabases = c.statusUncovered(ctx, app, name, f, e.Engine, opts, &st) + for _, db := range e.UncoveredDatabases { + st.UncoveredDatabases = append(st.UncoveredDatabases, f.Name+"/"+db) + } + } + st.Entries = append(st.Entries, e) + } + for _, a := range policy.Adopted { + found := false + for _, f := range facts { + found = found || f.Name == a.Container + } + if !found { + e := StatusEntry{Name: a.Container, Kind: TargetAdopted, Accessory: a.Container, Engine: a.Engine, Adopted: true, Unmanaged: true, Schedule: "policy", + Coverage: CoverageFailing, Reason: "adopted container does not exist on this server", State: "missing"} + fillEntryEvents(&e, logNS(AdoptedApp), a.Container) + fillEntryObjects(&e, listNS(AdoptedApp), a.Container) + st.Entries = append(st.Entries, e) + } + } + if trees, err := c.listVolumeTrees(ctx); err != nil { + st.Warnings = append(st.Warnings, err.Error()) + } else { + for _, tr := range trees { + e := StatusEntry{Name: tr.App + "/volumes", Kind: TargetVolumes, App: tr.App, Schedule: "none", SizeBytes: tr.KB * 1024} + guard := policy.VolumeMaxGB > 0 && float64(e.SizeBytes) > policy.VolumeMaxGB*(1<<30) + excluded := policy.Excluded(tr.App, "volumes", "") || guard + covered := false + switch { + case ownCron[tr.App+"/"]: + covered, e.Schedule = true, "own" + case policy.Volumes && policy.Interval != "": + covered, e.Schedule = true, "policy" + } + fillEntryEvents(&e, logNS(tr.App), "") + fillEntryObjects(&e, listNS(tr.App), "") + decide(&e, excluded, covered) + if guard { + e.Reason = fmt.Sprintf("%.2f GB is over the policy's volume_max_gb %.2f", float64(e.SizeBytes)/(1<<30), policy.VolumeMaxGB) + } + st.Entries = append(st.Entries, e) + } + } + // The last fleet run. + for _, ev := range logNS(FleetApp) { + if ev.Kind != events.KindBackupAll { + continue + } + run := &StatusLastRun{At: ev.TS, Status: detailString(ev.Detail, "status"), TargetsOK: detailInt(ev.Detail, "targets_ok"), TargetsFailed: detailInt(ev.Detail, "targets_failed"), Error: detailString(ev.Detail, "error")} + if list, ok := ev.Detail["omissions"].([]any); ok { + for _, o := range list { + if s, ok := o.(string); ok { + run.Omissions = append(run.Omissions, s) + } + } + } + st.LastRun = run + } + if st.LastRun != nil && stale > 0 && now.Sub(st.LastRun.At) > stale { + st.LastRun.Stale = true + } + if out, err := exec.Run(ctx, LeftoverWorkspacesProbe()); err == nil { + st.LeftoverWorkspaces = splitLines(out) + } + sort.Strings(st.UncoveredDatabases) + return st, nil +} + +// statusUncovered reads what a running database server holds beyond this +// entry's coverage. Read-only (coverageWith's mongo cleanup is not reached). +func (c *Client) statusUncovered(ctx context.Context, app, name string, f *ContainerFact, engine string, opts BackupOptions, st *FleetStatus) []string { + if engine != EnginePostgres && engine != EngineMySQL && engine != EngineMariaDB { + return nil + } + qctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + var t pgTarget + if engine == EnginePostgres { + db, user := postgresDBAndUser(app, f.Env) + if app == AdoptedApp { + db = "" + } + t = newPGTarget(f.Name, db, user, f.Env) + } + cov, err := c.coverageWith(qctx, app, name, EngineDetection{Engine: engine}, t, f.Env, opts) + if err != nil { + st.Warnings = append(st.Warnings, fmt.Sprintf("%s: %v", f.Name, err)) + return nil + } + return append(cov.Uncovered, cov.Missing...) +} + +// fillEntryEvents sets the success, failure and verify facts from an event log. +// name is the accessory ("" for an app's volumes). Events are oldest first. +func fillEntryEvents(e *StatusEntry, evs []events.Event, name string) { + for _, ev := range evs { + acc := detailString(ev.Detail, "accessory") + if acc != name { + continue + } + switch ev.Kind { + case events.KindBackupRun: + if name == "" && detailString(ev.Detail, "kind") != KindVolumes { + continue + } + ts := ev.TS + switch ev.Outcome { + case events.Succeeded: + e.LastSuccess = &ts + // An older failure's error is no longer the current story, but + // the timestamp stays: it says the target has failed before. + case events.Failed: + e.LastFailure = &ts + e.LastError = detailString(ev.Detail, "error") + } + case events.KindBackupVerify: + e.LastVerify = &StatusVerify{At: ev.TS, OK: ev.Outcome == events.Succeeded, ID: detailString(ev.Detail, "date"), Metric: detailString(ev.Detail, "metric"), Database: detailString(ev.Detail, "database")} + } + } +} + +// fillEntryObjects sets the newest object of each series from a listing (newest +// first). Below-floor objects are not Restorable and never count. +func fillEntryObjects(e *StatusEntry, rows []ListedBackup, name string) { + seen := map[string]bool{} + for _, r := range rows { + switch { + case name == "" && r.Kind != KindVolumes: + continue + case name != "" && (r.Kind != KindAccessory || r.Accessory != name): + continue + case !r.Restorable: + continue + } + if seen[r.Database] { + continue + } + seen[r.Database] = true + obj := &StatusObject{ID: r.ID, Key: r.Key, SizeBytes: r.SizeBytes, CreatedAt: r.CreatedAt, Encrypted: r.Encrypted, Previous: r.Previous} + if r.Database == "" { + e.Newest = obj + } else { + e.Databases = append(e.Databases, StatusDatabase{Name: r.Database, Newest: obj}) + if e.Newest == nil || r.CreatedAt > e.Newest.CreatedAt { + e.Newest = obj + } + } + } + sort.Slice(e.Databases, func(i, j int) bool { return e.Databases[i].Name < e.Databases[j].Name }) + // Without any event yet, a good object is still evidence of a success: the + // history may predate the event log. + if e.LastSuccess == nil && e.Newest != nil { + if t, err := time.Parse(time.RFC3339, e.Newest.CreatedAt); err == nil { + e.LastSuccess = &t + } + } +} + +func detailString(d map[string]any, k string) string { + if s, ok := d[k].(string); ok { + return s + } + return "" +} + +func detailInt(d map[string]any, k string) int { + if n, ok := d[k].(float64); ok { + return int(n) + } + return 0 +} diff --git a/internal/events/events.go b/internal/events/events.go index ef107b6..1c0d1d4 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -114,6 +114,8 @@ const ( KindBackupRestore = "backup.restore" KindBackupVerify = "backup.verify" KindBackupTarget = "backup_target.set" + KindBackupAll = "backup.all" + KindBackupPolicy = "backup_policy.set" ) // detailAllow is the per-kind allowlist of detail keys. Env and secret @@ -144,10 +146,12 @@ var detailAllow = map[string][]string{ KindMaintenanceOff: {"error"}, KindGateRepair: {"case", "message"}, KindWatermarkReset: {"ref"}, - KindBackupRun: {"id", "kind", "accessory", "size_bytes", "key", "duration_ms", "engine", "consistency", "encrypted", "uncovered_databases", "warnings", "error"}, - KindBackupVerify: {"accessory", "date", "ok", "engine", "kind", "metric", "key", "size_bytes", "duration_ms", "error"}, - KindBackupRestore: {"id", "kind", "accessory", "action", "database", "released", "error"}, + KindBackupRun: {"id", "kind", "accessory", "size_bytes", "key", "duration_ms", "engine", "consistency", "encrypted", "uncovered_databases", "failed_databases", "databases", "warnings", "error", "policy_managed", "trigger", "pruned", "reason"}, + KindBackupVerify: {"accessory", "database", "date", "ok", "engine", "kind", "metric", "key", "size_bytes", "duration_ms", "encrypted", "error"}, + KindBackupRestore: {"id", "kind", "accessory", "action", "database", "released", "pre_backup_id", "error"}, KindBackupTarget: {"id", "kind", "accessory", "error"}, + KindBackupAll: {"status", "targets_ok", "targets_failed", "omissions", "encrypted", "duration_ms", "heartbeats_ok", "heartbeats_failed", "warnings", "error"}, + KindBackupPolicy: {"interval", "encrypted", "volumes", "adopted", "error"}, } // maxErrorDetail bounds the "error" detail so a command's full stderr From 092cc65e7f6352d41700cef77e15de7c2629e039 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Wed, 4 Nov 2026 22:34:35 -0700 Subject: [PATCH 03/17] feat(cli): backup-policy, run --all, status, fetch, keygen, --adopted, backup before restore - `teploy server backup-policy set/show/unset`: exactly one cron line, marker [teploy-backup-all]; flags change only what they name; --stdin is the machine path; heartbeat URLs arrive on stdin only and show back as hosts. - `teploy backup run --all`, with a nonzero exit for anything but a clean run. - `teploy backup status --host S --json`. - `teploy backup keygen`, printed once; `teploy backup fetch --identity`, local download and decrypt with nothing else, no partial result left behind. - --adopted and --database on restore, verify-backup, list, create, prune. - backup restore and accessory restore/upgrade store a fresh backup first and refuse if that fails (--no-backup); a database that is already gone has nothing to protect and is not a refusal. - backup-target set keeps the prefix history. - dr create wraps --include-age-key; dr restore --identity unwraps it. Contracts rev 17: backup-status, backup-run and backup-policy envelopes, additive fields on backup-list/create/restore/target, capability backup-fleet-v1. Co-Authored-By: Claude Opus 5.5 --- contracts/MANIFEST.md | 4 + .../valid/accessory.json | 3 +- .../backup-create-envelope/valid/volumes.json | 3 +- .../valid/configured.json | 39 + .../valid/unconfigured.json | 9 + .../backup-run-envelope/valid/partial.json | 70 + .../backup-run-envelope/valid/skipped.json | 9 + .../backup-status-envelope/valid/fleet.json | 172 +++ .../valid/unconfigured.json | 22 + .../fixtures/version-handshake/valid/mi2.json | 1 + .../schema/backup-create-envelope.schema.json | 101 +- .../schema/backup-policy-envelope.schema.json | 124 ++ .../backup-restore-envelope.schema.json | 83 +- .../schema/backup-run-envelope.schema.json | 204 +++ .../schema/backup-status-envelope.schema.json | 464 +++++++ .../schema/backup-target-envelope.schema.json | 71 +- internal/cli/accessory.go | 69 +- internal/cli/backup.go | 180 ++- internal/cli/backup_cron_test.go | 1 + internal/cli/backup_fetch_aws.go | 40 + internal/cli/backup_fleet.go | 1176 +++++++++++++++++ internal/cli/backup_fleet_test.go | 502 +++++++ internal/cli/backup_run.go | 29 +- internal/cli/backup_target.go | 75 +- internal/cli/contracts_golden_test.go | 60 + internal/cli/dr.go | 83 +- internal/cli/machineinterface.go | 8 + internal/cli/machineinterface_test.go | 2 +- internal/cli/root.go | 5 +- internal/cli/server.go | 1 + 30 files changed, 3501 insertions(+), 109 deletions(-) create mode 100644 contracts/fixtures/backup-policy-envelope/valid/configured.json create mode 100644 contracts/fixtures/backup-policy-envelope/valid/unconfigured.json create mode 100644 contracts/fixtures/backup-run-envelope/valid/partial.json create mode 100644 contracts/fixtures/backup-run-envelope/valid/skipped.json create mode 100644 contracts/fixtures/backup-status-envelope/valid/fleet.json create mode 100644 contracts/fixtures/backup-status-envelope/valid/unconfigured.json create mode 100644 contracts/schema/backup-policy-envelope.schema.json create mode 100644 contracts/schema/backup-run-envelope.schema.json create mode 100644 contracts/schema/backup-status-envelope.schema.json create mode 100644 internal/cli/backup_fetch_aws.go create mode 100644 internal/cli/backup_fleet.go create mode 100644 internal/cli/backup_fleet_test.go diff --git a/contracts/MANIFEST.md b/contracts/MANIFEST.md index 686bd34..4a07e7c 100644 --- a/contracts/MANIFEST.md +++ b/contracts/MANIFEST.md @@ -10,6 +10,7 @@ Neutron/Nucleus dependency and a public mirror. | Corpus rev | Emitting CLI | Machine Interface | Notes | |---|---|---|---| +| 17 | main (Dash auto-config Slice 4: fleet backup engine) | 2 | Additive. New artifacts backup-status-envelope (`backup status --json`: every DB-like container and app volume with coverage state, last success/failure, newest object, last verify, plus target and policy summaries; no secrets), backup-run-envelope (`backup run --all --json`) and backup-policy-envelope (`server backup-policy show/set --json`), fixtures GENERATED from the real DTOs. backup-list rows gain optional `database`, `encrypted`, `previous_prefix`; backup-create gains `encrypted` and `objects`; backup-restore gains `pre_backup_id`; backup-target gains `previous_prefixes`. Event kinds `backup.all` and `backup_policy.set` under the pseudo-app `_fleet`; adopted containers log under `_adopted`. version-handshake advertises `backup-fleet-v1` (regenerated mi2.json). No MI bump. | | 16 | main (server-authority spec P3 / plan D4, git sources) | 2 | Additive. New artifacts source-show-envelope (`config source show --json`: repository/branch/poll, public credential facts — type, deploy-key public half + fingerprint, token username, pinned host-key fingerprints — and the per-ref admission state), source-key-envelope (`config source key --json` for ssh/token and `host-key --json`) and trigger-poll-envelope (`trigger poll --json`: one result object per app — ref, head, admitted, reason vocabulary incl. force-push/retry-limit/not-due, deployed_release), all fixtures GENERATED from the real DTOs. version-handshake advertises `git-source-v1` (git: code source, per-app credentials outside /deployments, build_secrets) and `trigger-poll-v1` (the one admitter + reset/retry/install-timer) (regenerated mi2.json). No MI bump. | | 15 | main (server-authority spec D3: backup target) | 2 | Additive. One backup target per server, stored outside /deployments with age-encrypted credentials and read at run time (`server backup-target set/show/unset`; `backup-target-v1`). New artifacts backup-target-envelope, backup-list-envelope, backup-create-envelope, backup-restore-envelope — fixtures GENERATED from the real DTO encoders (`backup list --json` now reports parsed rows {id, kind, accessory, size_bytes, created_at, key} plus schedules discovered from teploy cron markers; the old bare-array-of-raw-`aws s3 ls`-lines `backup list --json` shape is gone — it was never a corpus artifact and pinned no consumer). `backup create/restore --json` envelopes; restore quiesces the app under its fenced lock (backup-json-v1). version-handshake advertises `backup-target-v1` and `backup-json-v1` (regenerated mi2.json). No MI bump. | | 14 | main (server-authority plan D2, folderless reads) | 2 | Additive. New artifacts releases-envelope (`teploy releases --app X --host S --json`: container release records newest first with current/rollback_available (+reason); static apps list retained releases — both via the envelope, the folder-static legacy bare array is unchanged), rollback-result (`teploy rollback [--to X] --json`: {app, server, from, to, ok}; failures keep the error envelope) and redeploy-result (`teploy redeploy [--json]`: {app, server, home, release, image, ok}). Fixtures GENERATED from the real DTOs. version-handshake advertises `secret-stdin-v1` (secret set KEY --stdin; folderless secret set/get/list/rm/rotate), `releases-json-v1`, `rollback-json-v1`, `redeploy-v1` (fresh deploy of the current release's recorded image, env re-resolved) and `env-config-guard-v1` (env set/unset refuse keys a Teploy-home config declares) (regenerated mi2.json). No MI bump. || 13 | main (server-authority spec P2 support) | 2 | Additive. New artifact config-merge-envelope (`teploy config merge --base --file - --json`: merged config + conflicting keys, read-only), fixture GENERATED from the real configMergeDTO. version-handshake advertises `config-merge-v1`, `config-unprivileged-v1` (`config ... --unprivileged` refuses host-reaching changes) and `deploy-home-v1` (`deploy --app X --image T --home teploy` creates a new Teploy-home app) (regenerated mi2.json). No MI bump. | @@ -54,6 +55,9 @@ Neutron/Nucleus dependency and a public mirror. | backup-list-envelope | yes (rev 15) | valid x2 (app, legacy-schedule; real backupListDTO) | teploy-cli | | backup-create-envelope | yes (rev 15) | valid x2 (volumes, accessory; real backupCreateDTO) | teploy-cli | | backup-restore-envelope | yes (rev 15) | valid x2 (ok, unhealthy; real backupRestoreDTO) | teploy-cli | +| backup-status-envelope | yes (rev 17) | valid x2 (fleet, unconfigured; real backupStatusDTO) | teploy-cli | +| backup-run-envelope | yes (rev 17) | valid x2 (partial, skipped; real backupRunEnvelope) | teploy-cli | +| backup-policy-envelope | yes (rev 17) | valid x2 (configured, unconfigured; real backupPolicyDTO) | teploy-cli | | releases-envelope | yes (rev 14) | valid x2 (container app, static app; real releasesDTO) | teploy-cli | | rollback-result | yes (rev 14) | valid (real rollbackResult) | teploy-cli | | redeploy-result | yes (rev 14) | valid x2 (teploy-home, folder-home; real redeployDTO) | teploy-cli | diff --git a/contracts/fixtures/backup-create-envelope/valid/accessory.json b/contracts/fixtures/backup-create-envelope/valid/accessory.json index e480c95..b15c633 100644 --- a/contracts/fixtures/backup-create-envelope/valid/accessory.json +++ b/contracts/fixtures/backup-create-envelope/valid/accessory.json @@ -8,5 +8,6 @@ "size_bytes": 2097152, "created_at": "2026-10-20T03:17:04Z", "key": "s3://teploy-backups/blog/accessories/db/20261020-031704-1f2e3d4c5b6a7988.sql.gz", - "pruned": 0 + "pruned": 0, + "encrypted": false } diff --git a/contracts/fixtures/backup-create-envelope/valid/volumes.json b/contracts/fixtures/backup-create-envelope/valid/volumes.json index 59c9a0b..091437e 100644 --- a/contracts/fixtures/backup-create-envelope/valid/volumes.json +++ b/contracts/fixtures/backup-create-envelope/valid/volumes.json @@ -8,5 +8,6 @@ "created_at": "2026-10-20T03:17:04Z", "key": "s3://teploy-backups/blog/volumes/20261020-031704-1f2e3d4c5b6a7988.tar.gz", "teploy_key": "s3://teploy-backups/blog/teploy/20261020-031704-1f2e3d4c5b6a7988.tar.gz", - "pruned": 1 + "pruned": 1, + "encrypted": false } diff --git a/contracts/fixtures/backup-policy-envelope/valid/configured.json b/contracts/fixtures/backup-policy-envelope/valid/configured.json new file mode 100644 index 0000000..e5fb2a9 --- /dev/null +++ b/contracts/fixtures/backup-policy-envelope/valid/configured.json @@ -0,0 +1,39 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "configured": true, + "path": "/etc/teploy/backup-policy.json", + "interval": "1h", + "cron": "37 * * * *", + "retention": { + "keep_last": 3, + "keep_daily": 7, + "keep_weekly": 4, + "keep_monthly": 12, + "max_age_days": 2 + }, + "excludes": [ + "blog/volumes" + ], + "adopted": [ + { + "container": "wordpress-shared-mysql", + "engine": "mysql", + "databases": "all" + }, + { + "container": "omni_postgres", + "databases": [ + "omni" + ] + } + ], + "volumes": true, + "volume_max_gb": 20, + "heartbeat_count": 1, + "heartbeat_hosts": [ + "https://dash.example.internal" + ], + "recipient": "age1lggyhqrw2nlhcxprm67z43rta597azn8gknawjehu9d9dl0jq3yqqvfafg", + "encrypted": true +} diff --git a/contracts/fixtures/backup-policy-envelope/valid/unconfigured.json b/contracts/fixtures/backup-policy-envelope/valid/unconfigured.json new file mode 100644 index 0000000..162c8bc --- /dev/null +++ b/contracts/fixtures/backup-policy-envelope/valid/unconfigured.json @@ -0,0 +1,9 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "configured": false, + "retention": {}, + "volumes": false, + "heartbeat_count": 0, + "encrypted": false +} diff --git a/contracts/fixtures/backup-run-envelope/valid/partial.json b/contracts/fixtures/backup-run-envelope/valid/partial.json new file mode 100644 index 0000000..c07cbfe --- /dev/null +++ b/contracts/fixtures/backup-run-envelope/valid/partial.json @@ -0,0 +1,70 @@ +{ + "machine_interface": 2, + "status": "partial", + "started_at": "2026-11-04T03:17:04Z", + "duration_ms": 8123, + "encrypted": true, + "targets": [ + { + "kind": "accessory", + "app": "blog", + "accessory": "db", + "container": "blog-db", + "engine": "postgres", + "status": "ok", + "id": "20261104-031704-1f2e3d4c5b6a7988", + "size_bytes": 1048576, + "duration_ms": 900, + "encrypted": true, + "objects": [ + { + "key": "s3://teploy-backups/prod/blog/accessories/db/20261104-031704-1f2e3d4c5b6a7988.pgdump.age", + "size_bytes": 1048576, + "encrypted": true + } + ], + "pruned": 2 + }, + { + "kind": "adopted", + "app": "_adopted", + "accessory": "omni_postgres", + "container": "omni_postgres", + "engine": "postgres", + "status": "failed", + "reason": "dumping omni: exit status 1", + "size_bytes": 0, + "duration_ms": 0, + "encrypted": true, + "pruned": 0, + "failed_databases": { + "omni": "exit status 1" + } + }, + { + "kind": "volumes", + "app": "blog", + "status": "excluded", + "reason": "volumes are 50.00 GB, over the policy's volume_max_gb 20.00; not backed up", + "size_bytes": 53687091200, + "duration_ms": 0, + "encrypted": true, + "pruned": 0, + "size_guard": true + } + ], + "uncovered_containers": [ + "redis-cache" + ], + "omissions": [ + "blog/volumes: volumes are 50.00 GB, over the policy's volume_max_gb 20.00; not backed up", + "redis-cache: a database container no policy entry covers (adopt or exclude it)" + ], + "heartbeats": [ + { + "host": "dash.example.internal", + "ok": true, + "status": 200 + } + ] +} diff --git a/contracts/fixtures/backup-run-envelope/valid/skipped.json b/contracts/fixtures/backup-run-envelope/valid/skipped.json new file mode 100644 index 0000000..0987994 --- /dev/null +++ b/contracts/fixtures/backup-run-envelope/valid/skipped.json @@ -0,0 +1,9 @@ +{ + "machine_interface": 2, + "status": "skipped", + "started_at": "2026-11-04T03:17:04Z", + "duration_ms": 0, + "encrypted": false, + "targets": [], + "error": "another run holds the lock" +} diff --git a/contracts/fixtures/backup-status-envelope/valid/fleet.json b/contracts/fixtures/backup-status-envelope/valid/fleet.json new file mode 100644 index 0000000..90c6589 --- /dev/null +++ b/contracts/fixtures/backup-status-envelope/valid/fleet.json @@ -0,0 +1,172 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "observed_at": "2026-11-04T03:17:04Z", + "target": { + "configured": true, + "bucket": "teploy-backups", + "region": "us-east-1", + "prefix": "prod", + "has_credentials": true, + "previous_prefixes": [ + { + "prefix": "" + } + ], + "reachable": true + }, + "policy": { + "configured": true, + "path": "/etc/teploy/backup-policy.json", + "interval": "1h", + "cron_installed": true, + "retention": { + "keep_last": 3, + "keep_daily": 7, + "keep_weekly": 4, + "keep_monthly": 12, + "max_age_days": 2 + }, + "excludes": [ + "blog/volumes" + ], + "adopted": [ + { + "container": "wordpress-shared-mysql", + "engine": "mysql", + "databases": "all" + }, + { + "container": "omni_postgres", + "databases": [ + "omni" + ] + } + ], + "volumes": true, + "volume_max_gb": 20, + "heartbeat_count": 1, + "heartbeat_hosts": [ + "https://dash.example.internal" + ], + "recipient": "age1lggyhqrw2nlhcxprm67z43rta597azn8gknawjehu9d9dl0jq3yqqvfafg" + }, + "encrypted": true, + "aws_cli": true, + "age_cli": true, + "last_run": { + "at": "2026-11-04T03:17:04Z", + "status": "partial", + "targets_ok": 3, + "targets_failed": 1, + "omissions": [ + "redis-cache: a database container no policy entry covers (adopt or exclude it)" + ], + "stale": false + }, + "entries": [ + { + "name": "blog-db", + "kind": "accessory", + "app": "blog", + "accessory": "db", + "engine": "postgres", + "detection": "env", + "state": "running", + "unmanaged": false, + "adopted": false, + "coverage": "covered", + "schedule": "policy", + "last_success": "2026-11-04T03:17:04Z", + "newest": { + "id": "20261104-031704-1f2e3d4c5b6a7988", + "key": "s3://teploy-backups/prod/blog/accessories/db/20261104-031704-1f2e3d4c5b6a7988.pgdump.age", + "size_bytes": 1048576, + "created_at": "2026-11-04T03:17:04Z", + "encrypted": true + }, + "last_verify": { + "at": "2026-11-04T03:17:04Z", + "ok": true, + "id": "20261104-031704-1f2e3d4c5b6a7988", + "metric": "tables=42" + } + }, + { + "name": "wordpress-shared-mysql", + "kind": "adopted", + "accessory": "wordpress-shared-mysql", + "engine": "mysql", + "detection": "policy", + "state": "running", + "unmanaged": true, + "adopted": true, + "coverage": "adopted", + "schedule": "policy", + "last_success": "2026-11-04T03:17:04Z", + "newest": { + "id": "20261104-031704-1f2e3d4c5b6a7988", + "key": "s3://teploy-backups/prod/_adopted/accessories/wordpress-shared-mysql/databases/wp_blog/20261104-031704-1f2e3d4c5b6a7988.sql.gz.age", + "size_bytes": 2097152, + "created_at": "2026-11-04T03:17:04Z", + "encrypted": true + }, + "databases": [ + { + "name": "wp_blog", + "newest": { + "id": "20261104-031704-1f2e3d4c5b6a7988", + "key": "s3://teploy-backups/prod/_adopted/accessories/wordpress-shared-mysql/databases/wp_blog/20261104-031704-1f2e3d4c5b6a7988.sql.gz.age", + "size_bytes": 2097152, + "created_at": "2026-11-04T03:17:04Z", + "encrypted": true + } + } + ] + }, + { + "name": "omni_postgres", + "kind": "adopted", + "accessory": "omni_postgres", + "engine": "postgres", + "detection": "env", + "state": "running", + "unmanaged": true, + "adopted": true, + "coverage": "failing", + "schedule": "policy", + "last_failure": "2026-11-04T02:17:04Z", + "last_error": "dumping omni: exit status 1", + "uncovered_databases": [ + "reports" + ] + }, + { + "name": "redis-cache", + "kind": "unmanaged", + "engine": "redis", + "detection": "env", + "state": "running", + "unmanaged": true, + "adopted": false, + "coverage": "uncovered", + "reason": "no policy entry covers this container (adopt or exclude it)", + "schedule": "none" + }, + { + "name": "blog/volumes", + "kind": "volumes", + "app": "blog", + "unmanaged": false, + "adopted": false, + "coverage": "excluded", + "reason": "50.00 GB is over the policy's volume_max_gb 20.00", + "schedule": "policy", + "size_bytes": 53687091200 + } + ], + "uncovered_databases": [ + "omni_postgres/reports", + "redis-cache" + ] +} diff --git a/contracts/fixtures/backup-status-envelope/valid/unconfigured.json b/contracts/fixtures/backup-status-envelope/valid/unconfigured.json new file mode 100644 index 0000000..70d48b2 --- /dev/null +++ b/contracts/fixtures/backup-status-envelope/valid/unconfigured.json @@ -0,0 +1,22 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "observed_at": "2026-11-04T03:17:04Z", + "target": { + "configured": false, + "has_credentials": false, + "reachable": false + }, + "policy": { + "configured": false, + "cron_installed": false, + "retention": {}, + "volumes": false, + "heartbeat_count": 0 + }, + "encrypted": false, + "aws_cli": false, + "age_cli": false, + "entries": [], + "uncovered_databases": [] +} diff --git a/contracts/fixtures/version-handshake/valid/mi2.json b/contracts/fixtures/version-handshake/valid/mi2.json index 9d1b5ab..dcaa822 100644 --- a/contracts/fixtures/version-handshake/valid/mi2.json +++ b/contracts/fixtures/version-handshake/valid/mi2.json @@ -3,6 +3,7 @@ "actor-flag-v1", "app-list-machine", "autodeploy-redeploy", + "backup-fleet-v1", "backup-json-v1", "backup-target-v1", "config-home-v1", diff --git a/contracts/schema/backup-create-envelope.schema.json b/contracts/schema/backup-create-envelope.schema.json index 86990b0..38455f5 100644 --- a/contracts/schema/backup-create-envelope.schema.json +++ b/contracts/schema/backup-create-envelope.schema.json @@ -3,18 +3,95 @@ "$id": "https://teploy.github.io/contracts/schema/backup-create-envelope.schema.json", "title": "`teploy backup create --json`: the created backup's identity (backup-json-v1, rev 14).", "type": "object", - "required": ["machine_interface", "host", "app", "id", "kind", "size_bytes", "created_at", "key", "pruned"], + "required": [ + "machine_interface", + "host", + "app", + "id", + "kind", + "size_bytes", + "created_at", + "key", + "pruned" + ], "properties": { - "machine_interface": {"type": "integer", "minimum": 1}, - "host": {"type": "string", "minLength": 1}, - "app": {"type": "string", "minLength": 1}, - "id": {"type": "string"}, - "kind": {"enum": ["volumes", "accessory"]}, - "accessory": {"type": "string"}, - "size_bytes": {"type": "integer", "minimum": 0, "description": "Size of the uploaded volumes archive or engine dump. 0 when the size could not be stated (the backup itself succeeded)."}, - "created_at": {"type": "string", "description": "RFC3339 UTC derived from the backup id's timestamp."}, - "key": {"type": "string", "pattern": "^s3://"}, - "teploy_key": {"type": "string", "pattern": "^s3://", "description": "Sibling object holding the app's server config/history (volume backups only, when present)."}, - "pruned": {"type": "integer", "minimum": 0, "description": "Backups removed by the requested retention policy after the upload."} + "machine_interface": { + "type": "integer", + "minimum": 1 + }, + "host": { + "type": "string", + "minLength": 1 + }, + "app": { + "type": "string", + "minLength": 1 + }, + "id": { + "type": "string" + }, + "kind": { + "enum": [ + "volumes", + "accessory" + ] + }, + "accessory": { + "type": "string" + }, + "size_bytes": { + "type": "integer", + "minimum": 0, + "description": "Size of the uploaded volumes archive or engine dump. 0 when the size could not be stated (the backup itself succeeded)." + }, + "created_at": { + "type": "string", + "description": "RFC3339 UTC derived from the backup id's timestamp." + }, + "key": { + "type": "string", + "pattern": "^s3://" + }, + "teploy_key": { + "type": "string", + "pattern": "^s3://", + "description": "Sibling object holding the app's server config/history (volume backups only, when present)." + }, + "pruned": { + "type": "integer", + "minimum": 0, + "description": "Backups removed by the requested retention policy after the upload." + }, + "encrypted": { + "type": "boolean", + "description": "age-encrypted (rev 17)." + }, + "objects": { + "type": "array", + "description": "Every primary and per-database object stored (rev 17).", + "items": { + "type": "object", + "required": [ + "key", + "size_bytes", + "encrypted" + ], + "properties": { + "database": { + "type": "string" + }, + "key": { + "type": "string" + }, + "size_bytes": { + "type": "integer", + "minimum": 0 + }, + "encrypted": { + "type": "boolean" + } + } + } + } } } diff --git a/contracts/schema/backup-policy-envelope.schema.json b/contracts/schema/backup-policy-envelope.schema.json new file mode 100644 index 0000000..4e8d3c5 --- /dev/null +++ b/contracts/schema/backup-policy-envelope.schema.json @@ -0,0 +1,124 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/backup-policy-envelope.schema.json", + "title": "`teploy server backup-policy show|set --json` (backup-fleet-v1, rev 17). Heartbeat URLs appear only as hosts.", + "type": "object", + "required": [ + "machine_interface", + "host", + "configured", + "retention", + "volumes", + "heartbeat_count", + "encrypted" + ], + "properties": { + "machine_interface": { + "type": "integer", + "minimum": 1 + }, + "host": { + "type": "string", + "minLength": 1 + }, + "configured": { + "type": "boolean" + }, + "path": { + "type": "string" + }, + "interval": { + "type": "string" + }, + "cron": { + "type": "string", + "description": "The installed cron line's five fields; empty when there is none." + }, + "retention": { + "type": "object", + "properties": { + "keep_last": { + "type": "integer", + "minimum": 0 + }, + "keep_daily": { + "type": "integer", + "minimum": 0 + }, + "keep_weekly": { + "type": "integer", + "minimum": 0 + }, + "keep_monthly": { + "type": "integer", + "minimum": 0 + }, + "max_age_days": { + "type": "integer", + "minimum": 0 + } + }, + "description": "Union of rules over the good backups (restic forget semantics). keep_daily/weekly/monthly: newest backup of the last N UTC days / ISO weeks / months that have one. max_age_days: keep everything younger (keep-within). The newest good backup always stays." + }, + "excludes": { + "type": "array", + "items": { + "type": "string" + } + }, + "adopted": { + "type": "array", + "items": { + "type": "object", + "required": [ + "container", + "databases" + ], + "properties": { + "container": { + "type": "string" + }, + "engine": { + "type": "string" + }, + "databases": { + "oneOf": [ + { + "const": "all" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + } + } + }, + "volumes": { + "type": "boolean" + }, + "volume_max_gb": { + "type": "number", + "minimum": 0 + }, + "heartbeat_count": { + "type": "integer", + "minimum": 0 + }, + "heartbeat_hosts": { + "type": "array", + "items": { + "type": "string" + } + }, + "recipient": { + "type": "string" + }, + "encrypted": { + "type": "boolean" + } + } +} diff --git a/contracts/schema/backup-restore-envelope.schema.json b/contracts/schema/backup-restore-envelope.schema.json index 62ef245..8c29136 100644 --- a/contracts/schema/backup-restore-envelope.schema.json +++ b/contracts/schema/backup-restore-envelope.schema.json @@ -3,19 +3,76 @@ "$id": "https://teploy.github.io/contracts/schema/backup-restore-envelope.schema.json", "title": "`teploy backup restore --json`: the quiesced restore receipt (backup-json-v1, rev 14).", "type": "object", - "required": ["machine_interface", "host", "app", "id", "kind", "ok", "stopped", "started", "health"], + "required": [ + "machine_interface", + "host", + "app", + "id", + "kind", + "ok", + "stopped", + "started", + "health" + ], "properties": { - "machine_interface": {"type": "integer", "minimum": 1}, - "host": {"type": "string", "minLength": 1}, - "app": {"type": "string", "minLength": 1}, - "id": {"type": "string"}, - "kind": {"enum": ["volumes", "accessory"]}, - "accessory": {"type": "string"}, - "ok": {"type": "boolean"}, - "stopped": {"type": "array", "items": {"type": "string"}, "description": "Containers stopped for the restore (the app's processes, plus a generic accessory's container for accessory restores)."}, - "started": {"type": "array", "items": {"type": "string"}}, - "health": {"enum": ["healthy", "unhealthy", "skipped"], "description": "Post-restore health check of the app's web process; skipped when nothing was stopped (undeployed app)."}, - "recovery_dir": {"type": "string", "description": "Present when the pre-restore data copy was KEPT (health failed after promotion, or a recovery-incomplete failure): the operator's way back."}, - "error": {"type": "string"} + "machine_interface": { + "type": "integer", + "minimum": 1 + }, + "host": { + "type": "string", + "minLength": 1 + }, + "app": { + "type": "string", + "minLength": 1 + }, + "id": { + "type": "string" + }, + "kind": { + "enum": [ + "volumes", + "accessory" + ] + }, + "accessory": { + "type": "string" + }, + "ok": { + "type": "boolean" + }, + "stopped": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Containers stopped for the restore (the app's processes, plus a generic accessory's container for accessory restores)." + }, + "started": { + "type": "array", + "items": { + "type": "string" + } + }, + "health": { + "enum": [ + "healthy", + "unhealthy", + "skipped" + ], + "description": "Post-restore health check of the app's web process; skipped when nothing was stopped (undeployed app)." + }, + "recovery_dir": { + "type": "string", + "description": "Present when the pre-restore data copy was KEPT (health failed after promotion, or a recovery-incomplete failure): the operator's way back." + }, + "error": { + "type": "string" + }, + "pre_backup_id": { + "type": "string", + "description": "The automatic backup taken just before this restore (rev 17)." + } } } diff --git a/contracts/schema/backup-run-envelope.schema.json b/contracts/schema/backup-run-envelope.schema.json new file mode 100644 index 0000000..d1f6d22 --- /dev/null +++ b/contracts/schema/backup-run-envelope.schema.json @@ -0,0 +1,204 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/backup-run-envelope.schema.json", + "title": "`teploy backup run --all --json` (backup-fleet-v1, rev 17). Exit status is nonzero unless status is ok or skipped. omissions lists everything not backed up: a status of ok with omissions is not a complete backup.", + "type": "object", + "required": [ + "machine_interface", + "status", + "started_at", + "duration_ms", + "encrypted", + "targets" + ], + "properties": { + "machine_interface": { + "type": "integer", + "minimum": 1 + }, + "status": { + "enum": [ + "ok", + "partial", + "failed", + "skipped" + ], + "description": "skipped: another run holds the overlap lock." + }, + "started_at": { + "type": "string" + }, + "duration_ms": { + "type": "integer", + "minimum": 0 + }, + "encrypted": { + "type": "boolean" + }, + "targets": { + "type": "array", + "items": { + "type": "object", + "required": [ + "kind", + "status", + "size_bytes", + "duration_ms", + "encrypted", + "pruned" + ], + "properties": { + "kind": { + "enum": [ + "accessory", + "volumes", + "adopted" + ] + }, + "app": { + "type": "string" + }, + "accessory": { + "type": "string" + }, + "container": { + "type": "string" + }, + "engine": { + "type": "string" + }, + "status": { + "enum": [ + "ok", + "failed", + "excluded", + "own-schedule" + ] + }, + "reason": { + "type": "string" + }, + "id": { + "type": "string" + }, + "size_bytes": { + "type": "integer", + "minimum": 0 + }, + "duration_ms": { + "type": "integer", + "minimum": 0 + }, + "encrypted": { + "type": "boolean" + }, + "objects": { + "type": "array", + "items": { + "type": "object", + "required": [ + "key", + "size_bytes", + "encrypted" + ], + "properties": { + "database": { + "type": "string" + }, + "key": { + "type": "string" + }, + "size_bytes": { + "type": "integer", + "minimum": 0 + }, + "encrypted": { + "type": "boolean" + } + } + } + }, + "pruned": { + "type": "integer", + "minimum": 0 + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "failed_databases": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "uncovered_databases": { + "type": "array", + "items": { + "type": "string" + } + }, + "size_guard": { + "type": "boolean" + } + } + } + }, + "uncovered_containers": { + "type": "array", + "items": { + "type": "string" + } + }, + "omissions": { + "type": "array", + "items": { + "type": "string" + } + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "heartbeats": { + "type": "array", + "items": { + "type": "object", + "required": [ + "host", + "ok" + ], + "properties": { + "host": { + "type": "string" + }, + "ok": { + "type": "boolean" + }, + "status": { + "type": "integer", + "minimum": 0 + }, + "error": { + "type": "string" + } + } + } + }, + "cleaned_workspaces": { + "type": "integer", + "minimum": 0 + }, + "aborted_uploads": { + "type": "integer", + "minimum": 0 + }, + "error": { + "type": "string" + } + } +} diff --git a/contracts/schema/backup-status-envelope.schema.json b/contracts/schema/backup-status-envelope.schema.json new file mode 100644 index 0000000..bfc87a0 --- /dev/null +++ b/contracts/schema/backup-status-envelope.schema.json @@ -0,0 +1,464 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://teploy.github.io/contracts/schema/backup-status-envelope.schema.json", + "title": "`teploy backup status --host S --json`: one read-only report per server (backup-fleet-v1, rev 17). Never carries credentials, env values or heartbeat tokens.", + "type": "object", + "required": [ + "machine_interface", + "host", + "observed_at", + "target", + "policy", + "encrypted", + "aws_cli", + "age_cli", + "entries", + "uncovered_databases" + ], + "properties": { + "machine_interface": { + "type": "integer", + "minimum": 1 + }, + "host": { + "type": "string", + "minLength": 1 + }, + "observed_at": { + "type": "string" + }, + "target": { + "type": "object", + "required": [ + "configured", + "has_credentials", + "reachable" + ], + "properties": { + "configured": { + "type": "boolean" + }, + "bucket": { + "type": "string" + }, + "region": { + "type": "string" + }, + "endpoint": { + "type": "string" + }, + "prefix": { + "type": "string" + }, + "has_credentials": { + "type": "boolean" + }, + "previous_prefixes": { + "type": "array", + "items": { + "type": "object", + "required": [ + "prefix" + ], + "properties": { + "prefix": { + "type": "string" + }, + "owned": { + "type": "boolean" + } + } + } + }, + "reachable": { + "type": "boolean", + "description": "The listing this report needed worked." + }, + "error": { + "type": "string" + } + } + }, + "policy": { + "type": "object", + "required": [ + "configured", + "cron_installed", + "retention", + "volumes", + "heartbeat_count" + ], + "properties": { + "configured": { + "type": "boolean" + }, + "path": { + "type": "string" + }, + "interval": { + "type": "string" + }, + "cron_installed": { + "type": "boolean" + }, + "retention": { + "type": "object", + "properties": { + "keep_last": { + "type": "integer", + "minimum": 0 + }, + "keep_daily": { + "type": "integer", + "minimum": 0 + }, + "keep_weekly": { + "type": "integer", + "minimum": 0 + }, + "keep_monthly": { + "type": "integer", + "minimum": 0 + }, + "max_age_days": { + "type": "integer", + "minimum": 0 + } + }, + "description": "Union of rules over the good backups (restic forget semantics). keep_daily/weekly/monthly: newest backup of the last N UTC days / ISO weeks / months that have one. max_age_days: keep everything younger (keep-within). The newest good backup always stays." + }, + "excludes": { + "type": "array", + "items": { + "type": "string" + } + }, + "adopted": { + "type": "array", + "items": { + "type": "object", + "required": [ + "container", + "databases" + ], + "properties": { + "container": { + "type": "string" + }, + "engine": { + "type": "string" + }, + "databases": { + "oneOf": [ + { + "const": "all" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + } + } + } + }, + "volumes": { + "type": "boolean" + }, + "volume_max_gb": { + "type": "number", + "minimum": 0 + }, + "heartbeat_count": { + "type": "integer", + "minimum": 0 + }, + "heartbeat_hosts": { + "type": "array", + "items": { + "type": "string" + }, + "description": "scheme://host only: the URLs carry tokens." + }, + "recipient": { + "type": "string", + "description": "The owner's recovery PUBLIC key." + } + } + }, + "encrypted": { + "type": "boolean", + "description": "New objects will be encrypted. false: Dash should warn." + }, + "aws_cli": { + "type": "boolean" + }, + "age_cli": { + "type": "boolean" + }, + "last_run": { + "type": "object", + "required": [ + "at", + "status", + "targets_ok", + "targets_failed", + "stale" + ], + "properties": { + "at": { + "type": "string" + }, + "status": { + "enum": [ + "ok", + "partial", + "failed" + ] + }, + "targets_ok": { + "type": "integer", + "minimum": 0 + }, + "targets_failed": { + "type": "integer", + "minimum": 0 + }, + "omissions": { + "type": "array", + "items": { + "type": "string" + } + }, + "error": { + "type": "string" + }, + "stale": { + "type": "boolean", + "description": "Older than twice the policy interval." + } + } + }, + "entries": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "kind", + "unmanaged", + "adopted", + "coverage", + "schedule" + ], + "properties": { + "name": { + "type": "string", + "description": "Container name, or /volumes." + }, + "kind": { + "enum": [ + "accessory", + "adopted", + "unmanaged", + "volumes" + ] + }, + "app": { + "type": "string" + }, + "accessory": { + "type": "string" + }, + "engine": { + "type": "string" + }, + "detection": { + "type": "string" + }, + "state": { + "type": "string", + "description": "Docker state, or missing for an adopted container that is gone." + }, + "unmanaged": { + "type": "boolean", + "description": "No teploy labels." + }, + "adopted": { + "type": "boolean" + }, + "coverage": { + "enum": [ + "covered", + "stale", + "failing", + "uncovered", + "excluded", + "adopted" + ], + "description": "adopted = covered, via a policy entry. A failing or stale adopted container reports failing or stale, with adopted:true." + }, + "reason": { + "type": "string" + }, + "schedule": { + "enum": [ + "policy", + "own", + "none" + ] + }, + "last_success": { + "type": "string" + }, + "last_failure": { + "type": "string" + }, + "last_error": { + "type": "string" + }, + "newest": { + "type": "object", + "required": [ + "id", + "key", + "size_bytes", + "created_at", + "encrypted" + ], + "properties": { + "id": { + "type": "string" + }, + "key": { + "type": "string", + "pattern": "^s3://" + }, + "size_bytes": { + "type": "integer", + "minimum": 0 + }, + "created_at": { + "type": "string" + }, + "encrypted": { + "type": "boolean" + }, + "previous_prefix": { + "type": "boolean" + } + } + }, + "databases": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string" + }, + "newest": { + "type": "object", + "required": [ + "id", + "key", + "size_bytes", + "created_at", + "encrypted" + ], + "properties": { + "id": { + "type": "string" + }, + "key": { + "type": "string", + "pattern": "^s3://" + }, + "size_bytes": { + "type": "integer", + "minimum": 0 + }, + "created_at": { + "type": "string" + }, + "encrypted": { + "type": "boolean" + }, + "previous_prefix": { + "type": "boolean" + } + } + } + } + } + }, + "last_verify": { + "type": "object", + "required": [ + "at", + "ok" + ], + "properties": { + "at": { + "type": "string" + }, + "ok": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "metric": { + "type": "string" + }, + "database": { + "type": "string" + } + } + }, + "uncovered_databases": { + "type": "array", + "items": { + "type": "string" + } + }, + "size_bytes": { + "type": "integer", + "minimum": 0 + } + } + } + }, + "uncovered_databases": { + "type": "array", + "items": { + "type": "string" + }, + "description": " when nothing covers it, / when one database of a covered container is left out." + }, + "leftover_workspaces": { + "type": "array", + "items": { + "type": "string" + } + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "leftover_uploads": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Bucket keys of uploads a killed run never published." + } + } +} diff --git a/contracts/schema/backup-target-envelope.schema.json b/contracts/schema/backup-target-envelope.schema.json index 5da3557..808d269 100644 --- a/contracts/schema/backup-target-envelope.schema.json +++ b/contracts/schema/backup-target-envelope.schema.json @@ -1,18 +1,67 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://teploy.github.io/contracts/schema/backup-target-envelope.schema.json", - "title": "`teploy server backup-target show --json`: the server's stored backup target (backup-target-v1, rev 14). Never carries secret material — only whether any is stored.", + "title": "`teploy server backup-target show --json`: the server's stored backup target (backup-target-v1, rev 14). Never carries secret material \u2014 only whether any is stored.", "type": "object", - "required": ["machine_interface", "host", "configured", "has_credentials"], + "required": [ + "machine_interface", + "host", + "configured", + "has_credentials" + ], "properties": { - "machine_interface": {"type": "integer", "minimum": 1}, - "host": {"type": "string", "minLength": 1}, - "configured": {"type": "boolean"}, - "bucket": {"type": "string", "minLength": 1}, - "region": {"type": "string", "minLength": 1}, - "endpoint": {"type": "string", "pattern": "^https?://"}, - "prefix": {"type": "string", "minLength": 1}, - "has_credentials": {"type": "boolean", "description": "Whether age-encrypted credentials are stored on the server. The envelope never carries credential values."}, - "path": {"type": "string", "description": "Server path the target was read from (/etc/teploy/backup-target.json or the ~/.config/teploy fallback)."} + "machine_interface": { + "type": "integer", + "minimum": 1 + }, + "host": { + "type": "string", + "minLength": 1 + }, + "configured": { + "type": "boolean" + }, + "bucket": { + "type": "string", + "minLength": 1 + }, + "region": { + "type": "string", + "minLength": 1 + }, + "endpoint": { + "type": "string", + "pattern": "^https?://" + }, + "prefix": { + "type": "string", + "minLength": 1 + }, + "has_credentials": { + "type": "boolean", + "description": "Whether age-encrypted credentials are stored on the server. The envelope never carries credential values." + }, + "path": { + "type": "string", + "description": "Server path the target was read from (/etc/teploy/backup-target.json or the ~/.config/teploy fallback)." + }, + "previous_prefixes": { + "type": "array", + "description": "Old prefixes of this bucket list and restore still search; only owned ones are pruned (rev 17).", + "items": { + "type": "object", + "required": [ + "prefix" + ], + "properties": { + "prefix": { + "type": "string" + }, + "owned": { + "type": "boolean" + } + } + } + } } } diff --git a/internal/cli/accessory.go b/internal/cli/accessory.go index 8042141..cd151a6 100644 --- a/internal/cli/accessory.go +++ b/internal/cli/accessory.go @@ -327,17 +327,23 @@ func runAccessoryLogs(flags *Flags, appName, name string, lines int) error { } func newAccessoryUpgradeCmd(flags *Flags) *cobra.Command { - return &cobra.Command{ + var noBackup bool + cmd := &cobra.Command{ Use: "upgrade ", Short: "Upgrade an accessory to a new image", - Args: cobra.ExactArgs(2), + Long: "Recreates the accessory on a new image. A new major version can rewrite its\n" + + "data in place with no way down, so a fresh backup is taken first; if that\n" + + "fails the upgrade is refused, unless --no-backup.", + Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - return runAccessoryUpgrade(flags, args[0], args[1]) + return runAccessoryUpgrade(flags, args[0], args[1], noBackup) }, } + cmd.Flags().BoolVar(&noBackup, "no-backup", false, "skip the automatic backup taken before the upgrade") + return cmd } -func runAccessoryUpgrade(flags *Flags, name, newImage string) error { +func runAccessoryUpgrade(flags *Flags, name, newImage string, noBackup bool) error { appCfg, err := loadAppCfgForAccessory() if err != nil { return err @@ -357,6 +363,20 @@ func runAccessoryUpgrade(flags *Flags, name, newImage string) error { } defer executor.Close() + if !noBackup { + cfg, err := resolvePreBackupTarget(ctx, executor, "accessory upgrade") + if err != nil { + return err + } + id, err := preOperationBackup(ctx, backup.NewClient(executor, os.Stdout), executor, appCfg.App, name, "", cfg, "this upgrade") + if err != nil { + return err + } + if id != "" { + fmt.Printf("Backed up the current data first: %s\n", id) + } + } + mgr := accessories.NewManager(executor, os.Stdout) return mgr.Upgrade(ctx, appCfg.App, name, newImage, accCfg) } @@ -559,6 +579,7 @@ func newAccessoryRestoreCmd(flags *Flags) *cobra.Command { prefix string yes bool ignore bool + noBackup bool ) cmd := &cobra.Command{ @@ -566,7 +587,7 @@ func newAccessoryRestoreCmd(flags *Flags) *cobra.Command { Short: "Restore an accessory from backup (stops the app while it runs)", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - return runAccessoryRestore(flags, args[0], args[1], bucket, region, endpoint, prefix, yes, ignore) + return runAccessoryRestore(flags, args[0], args[1], bucket, region, endpoint, prefix, yes, ignore, noBackup) }, } @@ -576,11 +597,12 @@ func newAccessoryRestoreCmd(flags *Flags) *cobra.Command { cmd.Flags().StringVar(&prefix, "prefix", "", "key prefix under the bucket (default: the stored target's prefix)") cmd.Flags().BoolVar(&yes, "yes", false, "proceed without interactive confirmation (the app is stopped while the restore runs)") cmd.Flags().BoolVar(&ignore, "ignore-space-check", false, "skip the free-space preflight") + cmd.Flags().BoolVar(&noBackup, "no-backup", false, "skip the automatic backup taken before the restore") return cmd } -func runAccessoryRestore(flags *Flags, name, date, bucket, region, endpoint, prefix string, yes, ignoreSpace bool) error { +func runAccessoryRestore(flags *Flags, name, date, bucket, region, endpoint, prefix string, yes, ignoreSpace, noBackup bool) error { appCfg, err := loadAppCfgForAccessory() if err != nil { return err @@ -594,7 +616,7 @@ func runAccessoryRestore(flags *Flags, name, date, bucket, region, endpoint, pre // the live database with the app still writing). Stopping the app is // an effect it never had, so it now asks first (--yes skips it, and is // required without a terminal) — the same contract as backup restore. - return runBackupRestoreWith(flags, "", name, date, &backupTargetFlags{bucket: bucket, region: region, endpoint: endpoint, prefix: prefix}, yes, backup.RestoreOptions{IgnoreSpaceCheck: ignoreSpace}) + return runBackupRestoreWith(flags, "", name, false, date, &backupTargetFlags{bucket: bucket, region: region, endpoint: endpoint, prefix: prefix}, yes, backup.RestoreOptions{IgnoreSpaceCheck: ignoreSpace, NoPreBackup: noBackup}) } // verifyEventErr is the backup.verify outcome: an operational error, or a @@ -621,6 +643,10 @@ func verifyEventDetail(name, date string, res *backup.VerifyResult, elapsed time detail["key"] = res.S3Key detail["size_bytes"] = res.SizeBytes detail["duration_ms"] = res.DurationMs + detail["encrypted"] = res.Encrypted + if res.Database != "" { + detail["database"] = res.Database + } } return detail } @@ -648,6 +674,8 @@ func newAccessoryVerifyBackupCmd(flags *Flags) *cobra.Command { date string endpoint string local bool + database string + adopted bool ) cmd := &cobra.Command{ @@ -663,12 +691,15 @@ func newAccessoryVerifyBackupCmd(flags *Flags) *cobra.Command { "Examples:\n" + " teploy accessory verify-backup postgres --bucket my-backups\n" + " teploy accessory verify-backup db --app myapp --host 1.2.3.4 --bucket b --json\n" + - " teploy accessory verify-backup nucleus --local --app ship --bucket b --endpoint http://127.0.0.1:9100", + " teploy accessory verify-backup nucleus --local --app ship --bucket b --endpoint http://127.0.0.1:9100\n" + + " teploy accessory verify-backup wordpress-shared-mysql --adopted --database wp_blog --host prod --json", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runAccessoryVerifyBackup(flags, appName, args[0], bucket, region, date, endpoint, local) + return runAccessoryVerifyBackup(flags, appName, args[0], bucket, region, date, endpoint, database, local, adopted) }, } + cmd.Flags().StringVar(&database, "database", "", "verify this database's own backups instead of the primary") + cmd.Flags().BoolVar(&adopted, "adopted", false, " is an adopted container's docker name (see `server backup-policy`)") cmd.Flags().StringVar(&appName, "app", "", "app name — act on server state instead of teploy.yml (requires --host)") cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name") cmd.Flags().StringVar(®ion, "region", "us-east-1", "AWS region") @@ -678,7 +709,7 @@ func newAccessoryVerifyBackupCmd(flags *Flags) *cobra.Command { return cmd } -func runAccessoryVerifyBackup(flags *Flags, appName, name, bucket, region, date, endpoint string, local bool) error { +func runAccessoryVerifyBackup(flags *Flags, appName, name, bucket, region, date, endpoint, database string, local, adopted bool) error { if date != "" { if err := backup.ValidateDate(date); err != nil { return err @@ -697,13 +728,22 @@ func runAccessoryVerifyBackup(flags *Flags, appName, name, bucket, region, date, var appCfg *config.AppConfig var executor ssh.Executor var err error - if local { + switch { + case local && adopted: + appCfg = &config.AppConfig{App: backup.AdoptedApp} + executor = ssh.NewLocalExecutor() + case local: if appName == "" { return fmt.Errorf("--local requires --app (no teploy.yml exists on the server)") } appCfg = &config.AppConfig{App: appName} executor = ssh.NewLocalExecutor() - } else { + case adopted: + appCfg, executor, err = resolveBackupApp(ctx, flags, appName, true) + if err != nil { + return err + } + default: appCfg, executor, err = resolveAppForAccessory(ctx, flags, appName) if err != nil { return err @@ -722,8 +762,11 @@ func runAccessoryVerifyBackup(flags *Flags, appName, name, bucket, region, date, out = os.Stderr } client := backup.NewClient(executor, out) + if err := applyAdoptedEngine(ctx, client, executor, appCfg.App, name); err != nil { + return err + } start := time.Now() - res, err := client.VerifyBackup(ctx, appCfg.App, name, date, s3Cfg) + res, err := client.VerifyBackupDatabase(ctx, appCfg.App, name, date, database, s3Cfg) recordAppEvent(ctx, executor, appCfg.App, events.KindBackupVerify, verifyEventErr(res, err), verifyEventDetail(name, date, res, time.Since(start))) if err != nil { return err diff --git a/internal/cli/backup.go b/internal/cli/backup.go index 368cad0..3f78c93 100644 --- a/internal/cli/backup.go +++ b/internal/cli/backup.go @@ -43,6 +43,9 @@ func newBackupCmd(flags *Flags) *cobra.Command { cmd.AddCommand(newBackupScheduleCmd(flags)) cmd.AddCommand(newBackupUnscheduleCmd(flags)) cmd.AddCommand(newBackupRunCmd(flags)) + cmd.AddCommand(newBackupStatusCmd(flags)) + cmd.AddCommand(newBackupFetchCmd(flags)) + cmd.AddCommand(newBackupKeygenCmd(flags)) return cmd } @@ -91,6 +94,9 @@ func resolveBackupS3(ctx context.Context, exec ssh.Executor, f *backupTargetFlag } cfg := s3Config(f.bucket, region, f.endpoint) cfg.Prefix = f.prefix + if err := backup.ApplyPolicyEncryption(ctx, exec, &cfg); err != nil { + return backup.S3Config{}, err + } return cfg, nil } target, err := backup.ReadTarget(ctx, exec) @@ -100,7 +106,11 @@ func resolveBackupS3(ctx context.Context, exec ssh.Executor, f *backupTargetFlag if !TargetConfigured(target) { return backup.S3Config{}, fmt.Errorf("no backup target is configured for this server — set one with `teploy server backup-target set --host S --bucket B` (or pass the legacy --bucket flag)") } - return TargetS3(target), nil + cfg := TargetS3(target) + if err := backup.ApplyPolicyEncryption(ctx, exec, &cfg); err != nil { + return backup.S3Config{}, err + } + return cfg, nil } // TargetConfigured reports whether a stored target exists. @@ -122,6 +132,8 @@ func TargetS3(t backup.Target) backup.S3Config { Endpoint: t.Endpoint, Prefix: t.Prefix, StoredTarget: t.Path, + + PreviousPrefixes: t.PreviousPrefixes, } } @@ -187,6 +199,10 @@ type backupCreateDTO struct { Key string `json:"key"` TeployKey string `json:"teploy_key,omitempty"` Pruned int `json:"pruned"` + // Encrypted says the objects are age-encrypted; false means Dash should warn. + Encrypted bool `json:"encrypted"` + // Objects are every primary and per-database object this backup stored. + Objects []backup.BackupObject `json:"objects,omitempty"` // Warnings are coverage problems the backup met; UncoveredDatabases are // databases in the cluster this backup does not contain. Warnings []string `json:"warnings,omitempty"` @@ -197,6 +213,8 @@ func newBackupCreateCmd(flags *Flags) *cobra.Command { var ( appName string accessory string + database string + adopted bool tf backupTargetFlags keepLast int maxAgeDays int @@ -207,23 +225,31 @@ func newBackupCreateCmd(flags *Flags) *cobra.Command { Short: "Create a volume or accessory backup", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - return runBackupCreate(flags, appName, accessory, &tf, keepLast, maxAgeDays) + return runBackupCreate(flags, appName, accessory, database, adopted, &tf, keepLast, maxAgeDays) }, } cmd.Flags().StringVar(&appName, "app", "", "app name — act on server state instead of teploy.yml (requires --host)") cmd.Flags().StringVar(&accessory, "accessory", "", "back up this accessory instead of the app's volumes") + cmd.Flags().StringVar(&database, "database", "", "back up only this database of the accessory") + cmd.Flags().BoolVar(&adopted, "adopted", false, "--accessory names an adopted container (see `server backup-policy`), addressed by its docker name (requires --host)") addBackupTargetFlags(cmd, &tf) addRetentionFlags(cmd, &keepLast, &maxAgeDays) return cmd } -func runBackupCreate(flags *Flags, appName, accessory string, tf *backupTargetFlags, keepLast, maxAgeDays int) error { +func runBackupCreate(flags *Flags, appName, accessory, database string, adopted bool, tf *backupTargetFlags, keepLast, maxAgeDays int) error { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() + if adopted && accessory == "" { + return fmt.Errorf("--adopted needs --accessory ") + } + if database != "" && accessory == "" { + return fmt.Errorf("--database needs --accessory") + } - appCfg, executor, err := resolveApp(ctx, flags, appName) + appCfg, executor, err := resolveBackupApp(ctx, flags, appName, adopted) if err != nil { return err } @@ -242,11 +268,18 @@ func runBackupCreate(flags *Flags, appName, accessory string, tf *backupTargetFl if err := backup.ValidateBucket(accessory); err != nil { return fmt.Errorf("invalid accessory name %q: %w", accessory, err) } + if aErr := applyAdoptedEngine(ctx, client, executor, appCfg.App, accessory); aErr != nil { + return aErr + } image, env, accErr := client.InspectAccessory(ctx, appCfg.App, accessory) if accErr != nil { return fmt.Errorf("inspecting accessory %s: %w", accessory, accErr) } - res, err = client.AccessoryBackup(ctx, appCfg.App, accessory, image, env, cfg) + opts := backup.BackupOptions{SingleDatabase: database} + if adopted && database == "" { + opts.AllDatabases = true + } + res, err = client.AccessoryBackupWith(ctx, appCfg.App, accessory, image, env, cfg, opts) kind = backup.KindAccessory } else { res, err = client.BackupVolumes(ctx, appCfg.App, cfg) @@ -318,6 +351,8 @@ func runBackupCreate(flags *Flags, appName, accessory string, tf *backupTargetFl Pruned: pruned, Warnings: res.Warnings, UncoveredDatabases: res.UncoveredDatabases, + Encrypted: res.Encrypted, + Objects: res.Objects, }) } fmt.Printf("Backup %s created (%s, %d bytes)\n", res.ID, kind, res.SizeBytes) @@ -327,6 +362,14 @@ func runBackupCreate(flags *Flags, appName, accessory string, tf *backupTargetFl if pruned > 0 { fmt.Printf(" Pruned %d old backup(s)\n", pruned) } + for _, o := range res.Objects { + if o.Database != "" { + fmt.Printf(" Database %s: %s\n", o.Database, o.Key) + } + } + if !res.Encrypted { + fmt.Println(" Not encrypted: no recovery key is configured (teploy backup keygen; teploy server backup-policy set --recipient)") + } return nil } @@ -362,6 +405,7 @@ type backupScheduleDTO struct { func newBackupListCmd(flags *Flags) *cobra.Command { var ( appName string + adopted bool tf backupTargetFlags ) @@ -370,21 +414,22 @@ func newBackupListCmd(flags *Flags) *cobra.Command { Short: "List available backups and schedules", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - return runBackupList(flags, appName, &tf) + return runBackupList(flags, appName, adopted, &tf) }, } cmd.Flags().StringVar(&appName, "app", "", "app name — act on server state instead of teploy.yml (requires --host)") + cmd.Flags().BoolVar(&adopted, "adopted", false, "list adopted containers' backups instead of an app's (requires --host)") addBackupTargetFlags(cmd, &tf) return cmd } -func runBackupList(flags *Flags, appName string, tf *backupTargetFlags) error { +func runBackupList(flags *Flags, appName string, adopted bool, tf *backupTargetFlags) error { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() - appCfg, executor, err := resolveApp(ctx, flags, appName) + appCfg, executor, err := resolveBackupApp(ctx, flags, appName, adopted) if err != nil { return err } @@ -456,6 +501,9 @@ func runBackupList(flags *Flags, appName string, tf *backupTargetFlags) error { if b.Accessory != "" { label += "/" + b.Accessory } + if b.Database != "" { + label += "/" + b.Database + } fmt.Printf("%s %-18s %10d bytes %s\n", b.CreatedAt, label, b.SizeBytes, b.Key) } return nil @@ -512,6 +560,8 @@ type backupRestoreDTO struct { Warnings []string `json:"warnings,omitempty"` // Database names a non-primary database (--database). Database string `json:"database,omitempty"` + // PreBackupID is the automatic backup taken just before this restore. + PreBackupID string `json:"pre_backup_id,omitempty"` // Action and Released report --release / --rollback. Action string `json:"action,omitempty"` Released []string `json:"released,omitempty"` @@ -522,6 +572,7 @@ func newBackupRestoreCmd(flags *Flags) *cobra.Command { var ( appName string accessory string + adopted bool tf backupTargetFlags yes bool opts restoreFlags @@ -538,7 +589,11 @@ func newBackupRestoreCmd(flags *Flags) *cobra.Command { "Non-interactive runs must pass --yes.\n\n" + "A restore whose health check failed, or one that was killed, leaves its\n" + "recovery copies: --release confirms the restore and discards them, and\n" + - "--rollback swaps a kept previous postgres database back (no ).", + "--rollback swaps a kept previous postgres database back (no ).\n\n" + + "Before anything changes a fresh backup of what is about to be replaced is\n" + + "taken; if that fails the restore is refused, unless --no-backup.\n\n" + + "--adopted addresses a container by its docker name (no app, so nothing is\n" + + "stopped for you: stop what uses it first), one database at a time.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { date := "" @@ -548,10 +603,13 @@ func newBackupRestoreCmd(flags *Flags) *cobra.Command { if err := opts.validate(date, accessory); err != nil { return err } + if adopted && accessory == "" { + return fmt.Errorf("--adopted needs --accessory ") + } if opts.release || opts.rollback { - return runBackupRecoveryAction(flags, appName, accessory, &opts, yes) + return runBackupRecoveryAction(flags, appName, accessory, adopted, &opts, yes) } - return runBackupRestoreWith(flags, appName, accessory, date, &tf, yes, opts.restoreOptions()) + return runBackupRestoreWith(flags, appName, accessory, adopted, date, &tf, yes, opts.restoreOptions()) }, } @@ -563,20 +621,22 @@ func newBackupRestoreCmd(flags *Flags) *cobra.Command { cmd.Flags().BoolVar(&opts.force, "force", false, "with --release: discard safety copies whose restore never recorded finishing") cmd.Flags().BoolVar(&opts.rollback, "rollback", false, "swap a kept previous postgres database back into place (needs --accessory, no )") cmd.Flags().StringVar(&opts.recoveryDatabase, "recovery-database", "", "with --rollback: which kept database, when several are") - cmd.Flags().StringVar(&opts.database, "database", "", "restore this database from a backup_databases: dump instead of the accessory's primary; with --release or --rollback, act on its copies") + cmd.Flags().StringVar(&opts.database, "database", "", "restore this database from its own series instead of the accessory's primary; with --release or --rollback, act on its copies") cmd.Flags().BoolVar(&opts.ignoreSpace, "ignore-space-check", false, "skip the free-space preflight") + cmd.Flags().BoolVar(&opts.noBackup, "no-backup", false, "skip the automatic backup taken before the restore") + cmd.Flags().BoolVar(&adopted, "adopted", false, "--accessory names an adopted container (requires --host)") return cmd } // restoreFlags are the accessory-restore options of `backup restore`. type restoreFlags struct { - release, rollback, ignoreSpace, force bool - recoveryDatabase, database string + release, rollback, ignoreSpace, force, noBackup bool + recoveryDatabase, database string } func (o *restoreFlags) restoreOptions() backup.RestoreOptions { - return backup.RestoreOptions{IgnoreSpaceCheck: o.ignoreSpace, Database: o.database} + return backup.RestoreOptions{IgnoreSpaceCheck: o.ignoreSpace, Database: o.database, NoPreBackup: o.noBackup} } func (o *restoreFlags) validate(date, accessory string) error { @@ -591,6 +651,8 @@ func (o *restoreFlags) validate(date, accessory string) error { return fmt.Errorf("--release and --rollback act on kept copies, not on a backup id") case action && o.ignoreSpace: return fmt.Errorf("--ignore-space-check applies to a restore, not to --release/--rollback") + case action && o.noBackup: + return fmt.Errorf("--no-backup applies to a restore, not to --release/--rollback") case o.force && !o.release: return fmt.Errorf("--force applies to --release") case !action && date == "": @@ -605,11 +667,11 @@ func (o *restoreFlags) validate(date, accessory string) error { return nil } -func runBackupRestoreWith(flags *Flags, appName, accessory, date string, tf *backupTargetFlags, yes bool, restoreOpts backup.RestoreOptions) error { +func runBackupRestoreWith(flags *Flags, appName, accessory string, adopted bool, date string, tf *backupTargetFlags, yes bool, restoreOpts backup.RestoreOptions) error { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() - appCfg, executor, err := resolveApp(ctx, flags, appName) + appCfg, executor, err := resolveBackupApp(ctx, flags, appName, adopted) if err != nil { return err } @@ -632,11 +694,18 @@ func runBackupRestoreWith(flags *Flags, appName, accessory, date string, tf *bac if !stdinIsTerminal() { return fmt.Errorf("--yes is required to restore without an interactive terminal") } - if !confirm(os.Stdout, fmt.Sprintf("Restore %s backup %s for %s (stops the app while it runs)?", restoreKindLabel(accessory), date, appCfg.App)) { + if !confirm(os.Stdout, restorePrompt(appCfg.App, accessory, restoreOpts.Database, date)) { return fmt.Errorf("aborted") } } + // The adopted namespace has no app directory for the lock to live in. + if adopted { + if _, err := executor.Run(ctx, "mkdir -p "+ssh.ShellQuote("/deployments/"+backup.AdoptedApp)); err != nil { + return fmt.Errorf("preparing the adopted namespace's lock directory: %w", err) + } + } + // The fenced lock serializes the restore against deploys and every // other mutating command; every stop/start below runs guarded by it. lk, err := state.AcquireLockFenced(ctx, executor, appCfg.App) @@ -648,6 +717,9 @@ func runBackupRestoreWith(flags *Flags, appName, accessory, date string, tf *bac progress := backupProgressOut(flags) client := backup.NewClient(executor, progress) + if err := applyAdoptedEngine(ctx, client, executor, appCfg.App, accessory); err != nil { + return err + } lc := deploy.NewLifecycle(executor, progress) dto := backupRestoreDTO{ @@ -685,6 +757,19 @@ func runBackupRestoreWith(flags *Flags, appName, accessory, date string, tf *bac if aErr != nil { return finishRestoreFailure(flags, ctx, executor, lc, lk, dto, false, nil, aErr) } + } else if _, _, _, aErr := client.FindVolumeBackup(ctx, appCfg.App, date, cfg); aErr != nil { + return finishRestoreFailure(flags, ctx, executor, lc, lk, dto, false, nil, aErr) + } + // A fresh copy of what is about to be replaced, before anything stops. + if !restoreOpts.NoPreBackup { + id, pErr := preOperationBackup(ctx, client, executor, appCfg.App, accessory, restoreOpts.Database, cfg, "this restore") + if pErr != nil { + return finishRestoreFailure(flags, ctx, executor, lc, lk, dto, false, nil, pErr) + } + dto.PreBackupID = id + if id != "" { + fmt.Fprintf(progress, "Backed up the current data first: %s\n", id) + } } stopped, qErr := lc.QuiesceApp(ctx, lk, appCfg.App, stopTimeout) for _, c := range stopped { @@ -693,7 +778,7 @@ func runBackupRestoreWith(flags *Flags, appName, accessory, date string, tf *bac accessoryStopped := false if qErr == nil && accessory != "" { if art.FileRestore { - container := appCfg.App + "-" + accessory + container := backup.ContainerName(appCfg.App, accessory) if _, sErr := lk.Guarded(ctx, executor, fmt.Sprintf("docker stop --time %d %s", stopTimeout, ssh.ShellQuote(container))); sErr == nil { accessoryStopped = true dto.Stopped = append(dto.Stopped, container) @@ -735,7 +820,7 @@ func runBackupRestoreWith(flags *Flags, appName, accessory, date string, tf *bac dto.Started = []string{} } if accessoryStopped { - container := appCfg.App + "-" + accessory + container := backup.ContainerName(appCfg.App, accessory) if _, sErr := lk.Guarded(ctx, executor, "docker start "+ssh.ShellQuote(container)); sErr == nil { dto.Started = append(dto.Started, container) } else if resume.HealthErr == nil { @@ -779,11 +864,11 @@ func runBackupRestoreWith(flags *Flags, appName, accessory, date string, tf *bac // the way out of a restore that was kept (failed health check) or left behind // (killed). --release drops the copies with the app running. --rollback stops // the app, swaps the previous database back and resumes. -func runBackupRecoveryAction(flags *Flags, appName, accessory string, opts *restoreFlags, yes bool) error { +func runBackupRecoveryAction(flags *Flags, appName, accessory string, adopted bool, opts *restoreFlags, yes bool) error { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() - appCfg, executor, err := resolveApp(ctx, flags, appName) + appCfg, executor, err := resolveBackupApp(ctx, flags, appName, adopted) if err != nil { return err } @@ -797,6 +882,9 @@ func runBackupRecoveryAction(flags *Flags, appName, accessory string, opts *rest } progress := backupProgressOut(flags) client := backup.NewClient(executor, progress) + if err := applyAdoptedEngine(ctx, client, executor, appCfg.App, accessory); err != nil { + return err + } image, env := inspectAccessoryForRestore(ctx, client, appCfg.App, accessory, appCfg) rc, err := client.FindRecoveryCopies(ctx, appCfg.App, accessory, image, opts.database, env) if err != nil { @@ -924,7 +1012,7 @@ func inspectAccessoryForRestore(ctx context.Context, client *backup.Client, app, // event. func finishRestoreFailure(flags *Flags, ctx context.Context, executor ssh.Executor, lc *deploy.Lifecycle, lk *state.Lock, dto backupRestoreDTO, accessoryStopped bool, stopped []docker.Container, rerr error) error { if accessoryStopped { - container := dto.App + "-" + dto.Accessory + container := backup.ContainerName(dto.App, dto.Accessory) if _, err := lk.Guarded(ctx, executor, "docker start "+ssh.ShellQuote(container)); err == nil { dto.Started = append(dto.Started, container) } @@ -944,6 +1032,9 @@ func finishRestoreFailure(flags *Flags, ctx context.Context, executor ssh.Execut func restoreEventDetail(dto backupRestoreDTO) map[string]any { detail := map[string]any{"id": dto.ID, "kind": dto.Kind} + if dto.PreBackupID != "" { + detail["pre_backup_id"] = dto.PreBackupID + } if dto.Accessory != "" { detail["accessory"] = dto.Accessory } @@ -953,33 +1044,53 @@ func restoreEventDetail(dto backupRestoreDTO) map[string]any { return detail } -func restoreKindLabel(accessory string) string { - if accessory != "" { - return "accessory " + accessory - } - return "volume" +// restorePrompt says exactly what a restore will replace, and what it will not +// protect: an adopted container's users are not stopped for it. +func restorePrompt(app, accessory, database, date string) string { + switch { + case app == backup.AdoptedApp && database != "": + return fmt.Sprintf("Replace database %s in adopted container %s with backup %s? Nothing that uses it is stopped for you: stop those first.", database, accessory, date) + case app == backup.AdoptedApp: + return fmt.Sprintf("Replace the data in adopted container %s with backup %s? Nothing that uses it is stopped for you: stop those first.", accessory, date) + case accessory != "" && database != "": + return fmt.Sprintf("Replace database %s in %s with backup %s (stops %s's containers while it runs)?", database, backup.ContainerName(app, accessory), date, app) + case accessory != "": + return fmt.Sprintf("Replace the data in %s with backup %s (stops %s's containers while it runs)?", backup.ContainerName(app, accessory), date, app) + } + return fmt.Sprintf("Replace everything in %s's volumes (and its .env, if the backup holds one) with backup %s (stops the app while it runs)?", app, date) } func newBackupPruneCmd(flags *Flags) *cobra.Command { var ( appName string + adopted bool tf backupTargetFlags accessory string keepLast int maxAgeDays int + gfs backup.FleetRetention ) cmd := &cobra.Command{ Use: "prune", Short: "Delete old backups per a retention policy", - Long: "Removes backups outside the retention policy.\nExample: teploy backup prune --keep-last 7\n teploy backup prune --max-age-days 30 --keep-last 3", - Args: cobra.NoArgs, + Long: "Removes backups outside the retention policy, with every object they keep\n" + + "beside them. --keep-daily/--keep-weekly/--keep-monthly keep the newest backup\n" + + "of the last N days, ISO weeks and months that have one; --max-age-days keeps\n" + + "everything younger than that. The rules are a union, and the newest good\n" + + "backup always stays.\nExample: teploy backup prune --keep-last 7\n teploy backup prune --max-age-days 30 --keep-last 3\n teploy backup prune --keep-daily 7 --keep-weekly 4 --keep-monthly 12", + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - return runBackupPrune(flags, appName, &tf, accessory, keepLast, maxAgeDays) + gfs.KeepLast, gfs.MaxAgeDays = keepLast, maxAgeDays + return runBackupPrune(flags, appName, adopted, &tf, accessory, gfs.Policy()) }, } cmd.Flags().StringVar(&appName, "app", "", "app name — act on server state instead of teploy.yml (requires --host)") + cmd.Flags().BoolVar(&adopted, "adopted", false, "--accessory names an adopted container (requires --host)") + cmd.Flags().IntVar(&gfs.KeepDaily, "keep-daily", 0, "keep the newest backup of the last N days that have one") + cmd.Flags().IntVar(&gfs.KeepWeekly, "keep-weekly", 0, "keep the newest backup of the last N ISO weeks that have one") + cmd.Flags().IntVar(&gfs.KeepMonthly, "keep-monthly", 0, "keep the newest backup of the last N months that have one") addBackupTargetFlags(cmd, &tf) cmd.Flags().StringVar(&accessory, "accessory", "", "Prune an accessory's backups instead of volumes (accessory name)") addRetentionFlags(cmd, &keepLast, &maxAgeDays) @@ -987,11 +1098,11 @@ func newBackupPruneCmd(flags *Flags) *cobra.Command { return cmd } -func runBackupPrune(flags *Flags, appName string, tf *backupTargetFlags, accessory string, keepLast, maxAgeDays int) error { +func runBackupPrune(flags *Flags, appName string, adopted bool, tf *backupTargetFlags, accessory string, policy backup.RetentionPolicy) error { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() - appCfg, executor, err := resolveApp(ctx, flags, appName) + appCfg, executor, err := resolveBackupApp(ctx, flags, appName, adopted) if err != nil { return err } @@ -1002,9 +1113,8 @@ func runBackupPrune(flags *Flags, appName string, tf *backupTargetFlags, accesso return err } - policy := backup.RetentionPolicy{KeepLast: keepLast, MaxAgeDays: maxAgeDays} if policy.IsZero() { - return fmt.Errorf("nothing to prune: set --keep-last and/or --max-age-days") + return fmt.Errorf("nothing to prune: set --keep-last, --keep-daily, --keep-weekly, --keep-monthly and/or --max-age-days") } prefix := "volumes" diff --git a/internal/cli/backup_cron_test.go b/internal/cli/backup_cron_test.go index a0e1abc..39f8d0d 100644 --- a/internal/cli/backup_cron_test.go +++ b/internal/cli/backup_cron_test.go @@ -151,6 +151,7 @@ func TestDiscoverBackupSchedules(t *testing.T) { func TestResolveBackupS3(t *testing.T) { ctx := context.Background() stored := ssh.NewMockExecutor("srv", + ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "absent\n"}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"kept","region":"eu-west-1","endpoint":"http://127.0.0.1:9100","credentials":"AGE"}`}, ) cfg, err := resolveBackupS3(ctx, stored, &backupTargetFlags{}) diff --git a/internal/cli/backup_fetch_aws.go b/internal/cli/backup_fetch_aws.go new file mode 100644 index 0000000..faba70a --- /dev/null +++ b/internal/cli/backup_fetch_aws.go @@ -0,0 +1,40 @@ +package cli + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "strings" + + "github.com/useteploy/teploy/internal/backup" +) + +// localAWSCopy downloads one object with this machine's aws CLI, the same +// transport the servers use. Credentials come from the environment and go to the +// child process only: TEPLOY_S3_* are the names the rest of the CLI reads. +func localAWSCopy(ctx context.Context, key, dest, region, endpoint string) error { + if _, err := exec.LookPath("aws"); err != nil { + return fmt.Errorf("the aws CLI is not on this machine — %s", backup.AWSCLIRemedy) + } + args := []string{"s3", "cp", key, dest, "--region", region, "--only-show-errors"} + if endpoint != "" { + args = append(args, "--endpoint-url", endpoint) + } + c := exec.CommandContext(ctx, "aws", args...) + env := os.Environ() + if v := os.Getenv("TEPLOY_S3_ACCESS_KEY"); v != "" { + env = append(env, "AWS_ACCESS_KEY_ID="+v) + } + if v := os.Getenv("TEPLOY_S3_SECRET_KEY"); v != "" { + env = append(env, "AWS_SECRET_ACCESS_KEY="+v) + } + c.Env = env + var stderr bytes.Buffer + c.Stderr = &stderr + if err := c.Run(); err != nil { + return fmt.Errorf("downloading %s: %w: %s", key, err, strings.TrimSpace(stderr.String())) + } + return nil +} diff --git a/internal/cli/backup_fleet.go b/internal/cli/backup_fleet.go new file mode 100644 index 0000000..618746b --- /dev/null +++ b/internal/cli/backup_fleet.go @@ -0,0 +1,1176 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/signal" + "path/filepath" + "strings" + "syscall" + "time" + + "filippo.io/age" + "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/backup" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/events" + "github.com/useteploy/teploy/internal/secret" + "github.com/useteploy/teploy/internal/ssh" +) + +// The fleet engine's commands: `backup run --all` (the one policy cron line), +// `server backup-policy`, `backup status`, `backup fetch`, `backup keygen`. + +// fleetRunDeadline bounds a whole `backup run --all`; each target has its own +// backup.BackupDeadline inside it. A holder older than this plus an hour is stuck. +const fleetRunDeadline = 23 * time.Hour + +// serverToolPaths are directories cron's minimal PATH leaves out, where aws +// (the AWS installer's symlink) and age may live. +var serverToolPaths = []string{"/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin", "/snap/bin"} + +// ensureServerPath completes cron's PATH: the job finding no aws would fail every +// night with a misleading message, on a server where the command works by hand. +func ensureServerPath() { + have := map[string]bool{} + cur := os.Getenv("PATH") + for _, d := range filepath.SplitList(cur) { + have[d] = true + } + for _, d := range serverToolPaths { + if !have[d] { + cur += string(os.PathListSeparator) + d + } + } + os.Setenv("PATH", cur) +} + +// backupRunEnvelope is `backup run --all --json` (backup-fleet-v1). +type backupRunEnvelope struct { + MachineInterface int `json:"machine_interface"` + backup.FleetRunResult +} + +func runBackupRunAll(flags *Flags, ignoreSpace bool) error { + ensureServerPath() + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + ctx, cancel := context.WithTimeout(ctx, fleetRunDeadline) + defer cancel() + executor := ssh.NewLocalExecutor() + defer executor.Close() + + progress := io.Writer(os.Stdout) + if flags.JSON { + progress = os.Stderr + } + res, err := runFleet(ctx, executor, progress, ignoreSpace, nil) + if flags.JSON { + encErr := json.NewEncoder(os.Stdout).Encode(backupRunEnvelope{MachineInterface: MachineInterface, FleetRunResult: res}) + return errors.Join(err, encErr) + } + return err +} + +// fleetFailed is the run's error once a result exists: nonzero exit for anything +// but a clean run. +func fleetFailed(res backup.FleetRunResult) error { + switch res.Status { + case backup.RunOK, "skipped": + return nil + case backup.RunPartial: + n := 0 + for _, t := range res.Targets { + if t.Status == backup.TargetFailed { + n++ + } + } + return fmt.Errorf("backup run --all: %d target(s) failed", n) + } + if res.Error != "" { + return fmt.Errorf("backup run --all failed: %s", res.Error) + } + return fmt.Errorf("backup run --all failed: no target succeeded") +} + +// runFleet is the whole run, against a given executor (the seam tests use). +func runFleet(ctx context.Context, exec ssh.Executor, out io.Writer, ignoreSpace bool, hc *http.Client) (backup.FleetRunResult, error) { + started := time.Now().UTC() + res := backup.FleetRunResult{StartedAt: started, Targets: []backup.FleetTargetResult{}} + + policy, err := backup.ReadFleetPolicy(ctx, exec) + if err != nil { + res.Status, res.Error = backup.RunFailed, err.Error() + return res, fleetFailed(res) + } + if !policy.Configured() { + res.Status, res.Error = backup.RunFailed, "no backup policy on this server: set one with `teploy server backup-policy set`" + return res, fleetFailed(res) + } + res.Encrypted = policy.Recipient != "" + + lock, holder, lerr := backup.AcquireRunLock(backup.RunLockPath(policy.Path), started, fleetRunDeadline+time.Hour) + if lerr != nil { + if errors.Is(lerr, backup.ErrRunLocked) { + // The running one reports; a stuck one stops reporting, and the + // missing heartbeat is the alert. + fmt.Fprintf(out, "Another backup run --all is running (pid %d, since %s); this one stands down\n", holder.PID, holder.StartedAt.Format(time.RFC3339)) + res.Status, res.Error = "skipped", "another run holds the lock" + return res, nil + } + return finishFleet(ctx, exec, out, res, policy, started, lerr, hc) + } + defer lock.Release() + + target, err := backup.ReadTarget(ctx, exec) + if err == nil && !TargetConfigured(target) { + err = fmt.Errorf("the policy has no backup target to write to: set one with `teploy server backup-target set`") + } + if err != nil { + return finishFleet(ctx, exec, out, res, policy, started, err, hc) + } + cfg := TargetS3(target) + cfg.OwnerRecipient = policy.Recipient + client := backup.NewClient(exec, out) + if err := client.EnsureAWSCLIPublic(ctx); err != nil { + return finishFleet(ctx, exec, out, res, policy, started, err, hc) + } + if err := client.PrepareEncryption(ctx, cfg); err != nil { + return finishFleet(ctx, exec, out, res, policy, started, err, hc) + } + if !cfg.Encrypted() { + res.Warnings = append(res.Warnings, "no recovery key is configured, so objects are stored unencrypted and app .env files are left out of volume backups (teploy backup keygen; teploy server backup-policy set --recipient)") + } + + // A killed run's leftovers, now that this run owns the lock. Bounded: a wedged + // janitor must not cost the backups. + jctx, jcancel := context.WithTimeout(ctx, 5*time.Minute) + res.CleanedWorkspaces = len(client.CleanStaleWorkspaces(jctx)) + res.AbortedUploads = client.AbortStaleUploads(jctx, cfg, started) + jcancel() + + ectx, ecancel := context.WithTimeout(ctx, 10*time.Minute) + targets, uncovered, err := client.EnumerateFleet(ectx, policy) + ecancel() + if err != nil { + return finishFleet(ctx, exec, out, res, policy, started, err, hc) + } + res.UncoveredContainers = uncovered + + for _, t := range targets { + if ctx.Err() != nil { + r := backup.FleetTargetResult{Kind: t.Kind, App: t.App, Accessory: t.Name, Container: t.Container, Status: backup.TargetFailed, Reason: "the run was stopped before this target: " + ctx.Err().Error()} + res.Targets = append(res.Targets, r) + recordFleetTargetEvent(context.WithoutCancel(ctx), exec, r) + continue + } + r := client.RunFleetTarget(ctx, t, cfg, backup.BackupOptions{IgnoreSpace: ignoreSpace}) + res.Targets = append(res.Targets, r) + recordFleetTargetEvent(ctx, exec, r) + switch r.Status { + case backup.TargetOK: + fmt.Fprintf(out, "OK %s (%d bytes)\n", fleetLabel(r), r.SizeBytes) + case backup.TargetFailed: + fmt.Fprintf(out, "FAILED %s: %s\n", fleetLabel(r), r.Reason) + default: + fmt.Fprintf(out, "SKIPPED %s: %s\n", fleetLabel(r), r.Reason) + } + } + return finishFleet(ctx, exec, out, res, policy, started, nil, hc) +} + +func fleetLabel(r backup.FleetTargetResult) string { + if r.Container != "" { + return r.Container + } + return r.App + "/volumes" +} + +// recordFleetTargetEvent writes the target's backup.run: in the app's history, or +// the adopted namespace's. Retired per-app alert script: a policy run signals +// through its heartbeat (plan 2.3), and legacy `backup run --app` keeps the script. +func recordFleetTargetEvent(ctx context.Context, exec ssh.Executor, r backup.FleetTargetResult) { + if r.Status == backup.TargetOwnCron || (r.Status == backup.TargetExcluded && !r.SizeGuard) { + return + } + kind := backup.KindAccessory + if r.Kind == backup.TargetVolumes { + kind = backup.KindVolumes + } + detail := backupRunDetail(r.Result, kind, r.Accessory) + detail["policy_managed"] = true + detail["pruned"] = r.Pruned + var dbs []string + for _, o := range r.Objects { + if o.Database != "" { + dbs = append(dbs, o.Database) + } + } + if len(dbs) > 0 { + detail["databases"] = dbs + } + if len(r.FailedDatabases) > 0 { + names := make([]string, 0, len(r.FailedDatabases)) + for db := range r.FailedDatabases { + names = append(names, db) + } + detail["failed_databases"] = names + } + ev := events.Event{App: r.App, Kind: events.KindBackupRun, Outcome: events.Succeeded, Detail: detail} + switch r.Status { + case backup.TargetFailed: + ev.Outcome, ev.Detail = events.Failed, events.ErrDetail(detail, errors.New(r.Reason)) + case backup.TargetExcluded: + ev.Outcome = events.Refused + detail["reason"] = r.Reason + detail["size_bytes"] = r.SizeBytes + } + events.Record(ctx, exec, ev) +} + +// finishFleet settles the status, sends the heartbeat and writes the summary +// event. It runs on a fresh deadline: a run that hit its own must still report. +func finishFleet(ctx context.Context, exec ssh.Executor, out io.Writer, res backup.FleetRunResult, policy backup.FleetPolicy, started time.Time, fatal error, hc *http.Client) (backup.FleetRunResult, error) { + rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) + defer cancel() + res.DurationMs = time.Since(started).Milliseconds() + backup.SummarizeFleet(&res) + if fatal != nil { + res.Status, res.Error = backup.RunFailed, fatal.Error() + } + for _, o := range res.Omissions { + fmt.Fprintf(out, "NOT BACKED UP: %s\n", o) + } + for _, w := range res.Warnings { + fmt.Fprintf(out, "Warning: %s\n", w) + } + // up = every target that should have run did: an omission is reported, but a + // deliberate exclusion is not an outage. + res.Heartbeats = backup.SendHeartbeats(rctx, policy.HeartbeatURLs, res.Status == backup.RunOK, hc) + hbOK, hbFailed, tOK, tFailed := 0, 0, 0, 0 + for _, h := range res.Heartbeats { + if h.OK { + hbOK++ + } else { + hbFailed++ + // A dead receiver is worth saying, but never fails the backup. + fmt.Fprintf(out, "Warning: heartbeat to %s failed: %s\n", h.Host, h.Error) + } + } + for _, t := range res.Targets { + switch t.Status { + case backup.TargetOK: + tOK++ + case backup.TargetFailed: + tFailed++ + } + } + detail := map[string]any{ + "status": res.Status, "targets_ok": tOK, "targets_failed": tFailed, "encrypted": res.Encrypted, + "duration_ms": res.DurationMs, "heartbeats_ok": hbOK, "heartbeats_failed": hbFailed, + } + if len(res.Omissions) > 0 { + detail["omissions"] = res.Omissions + } + if len(res.Warnings) > 0 { + detail["warnings"] = res.Warnings + } + outcome := events.Succeeded + if res.Status != backup.RunOK { + outcome = events.Failed + if res.Error != "" { + detail["error"] = res.Error + } else { + detail["error"] = fmt.Sprintf("%d target(s) failed", tFailed) + } + } + events.Record(rctx, exec, events.Event{App: backup.FleetApp, Kind: events.KindBackupAll, Outcome: outcome, Detail: detail}) + fmt.Fprintf(out, "backup run --all: %s (%d ok, %d failed, %d not backed up)\n", res.Status, tOK, tFailed, len(res.Omissions)) + return res, fleetFailed(res) +} + +// ---- server backup-policy ---- + +// backupPolicyDTO is `server backup-policy show --json`: heartbeat URLs carry +// tokens, so they appear only as hosts. +type backupPolicyDTO struct { + MachineInterface int `json:"machine_interface"` + Host string `json:"host"` + Configured bool `json:"configured"` + Path string `json:"path,omitempty"` + Interval string `json:"interval,omitempty"` + Cron string `json:"cron,omitempty"` + Retention backup.FleetRetention `json:"retention"` + Excludes []string `json:"excludes,omitempty"` + Adopted []backup.AdoptedEntry `json:"adopted,omitempty"` + Volumes bool `json:"volumes"` + VolumeMaxGB float64 `json:"volume_max_gb,omitempty"` + HeartbeatCount int `json:"heartbeat_count"` + HeartbeatHosts []string `json:"heartbeat_hosts,omitempty"` + Recipient string `json:"recipient,omitempty"` + Encrypted bool `json:"encrypted"` +} + +func newServerBackupPolicyCmd(flags *Flags) *cobra.Command { + cmd := &cobra.Command{ + Use: "backup-policy", + Short: "Manage the server's backup policy (the fleet engine's one cron line)", + Long: "One policy per server, at /etc/teploy/backup-policy.json (0600; a non-root\n" + + "user who cannot write /etc/teploy gets ~/.config/teploy/backup-policy.json).\n" + + "A single cron line runs `teploy backup run --all`, which works out at run\n" + + "time what to back up: every labelled accessory (each of its databases), app\n" + + "volumes when enabled, and adopted containers. A per-app schedule stays as an\n" + + "override for that target.", + } + cmd.AddCommand(newServerBackupPolicySetCmd(flags)) + cmd.AddCommand(newServerBackupPolicyShowCmd(flags)) + cmd.AddCommand(newServerBackupPolicyUnsetCmd(flags)) + return cmd +} + +type policySetFlags struct { + stdin, heartbeatStdin, noRecipient, clearHeartbeats, volumes bool + interval, recipient string + keepLast, keepDaily, keepWeekly, keepMonthly, maxAgeDays int + volumeMaxGB float64 + excludes, adopts, removeAdopted []string +} + +func newServerBackupPolicySetCmd(flags *Flags) *cobra.Command { + var f policySetFlags + cmd := &cobra.Command{ + Use: "set", + Short: "Store the policy and install its one cron line", + Long: "Flags change only what they name; everything else keeps its stored value.\n" + + "--stdin instead reads the whole policy as JSON on stdin (the machine path).\n" + + "Heartbeat URLs carry tokens, so they only ever arrive on stdin: with\n" + + "--heartbeat-stdin, one URL per line.\n\n" + + "--max-age-days keeps everything younger than that (restic keep-within);\n" + + "--keep-daily/--keep-weekly/--keep-monthly keep the newest of the last N days,\n" + + "ISO weeks and months that have a backup. The rules are a union, and the\n" + + "newest good backup always stays.\n\n" + + "Examples:\n" + + " teploy server backup-policy set --host prod --interval 1h --max-age-days 2 \\\n" + + " --keep-daily 7 --keep-weekly 4 --keep-monthly 12 --recipient age1...\n" + + " teploy server backup-policy set --host prod --adopt omni_postgres:postgres:all \\\n" + + " --adopt wordpress-shared-mysql:mysql:all\n" + + " printf 'https://dash/api/push/TOKEN\\n' | teploy server backup-policy set --host prod --heartbeat-stdin", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return runServerBackupPolicySet(cmd, flags, &f) + }, + } + cmd.Flags().BoolVar(&f.stdin, "stdin", false, "read the whole policy as JSON on stdin") + cmd.Flags().StringVar(&f.interval, "interval", "", "15m, 30m, 1h, 2h, 3h, 4h, 6h, 8h, 12h, 24h, 7d (or hourly, daily, weekly); none removes the cron line") + cmd.Flags().IntVar(&f.keepLast, "keep-last", 0, "keep the N most recent backups") + cmd.Flags().IntVar(&f.keepDaily, "keep-daily", 0, "keep the newest backup of the last N days that have one") + cmd.Flags().IntVar(&f.keepWeekly, "keep-weekly", 0, "keep the newest backup of the last N ISO weeks that have one") + cmd.Flags().IntVar(&f.keepMonthly, "keep-monthly", 0, "keep the newest backup of the last N months that have one") + cmd.Flags().IntVar(&f.maxAgeDays, "max-age-days", 0, "keep everything younger than N days") + cmd.Flags().StringSliceVar(&f.excludes, "exclude", nil, "leave out: , /, /volumes or a container name (replaces the list)") + cmd.Flags().StringArrayVar(&f.adopts, "adopt", nil, "adopt an unmanaged container: [:[:all|db1,db2]] (default: detected engine, all databases)") + cmd.Flags().StringSliceVar(&f.removeAdopted, "remove-adopted", nil, "stop adopting these containers (their backups stay)") + cmd.Flags().BoolVar(&f.volumes, "volumes", false, "back up app volumes (each archive is a full copy)") + cmd.Flags().Float64Var(&f.volumeMaxGB, "volume-max-gb", 0, "warn and exclude a volume archive over this many GB") + cmd.Flags().StringVar(&f.recipient, "recipient", "", "owner recovery public key (`teploy backup keygen`): encrypts every object from now on") + cmd.Flags().BoolVar(&f.noRecipient, "no-recipient", false, "stop encrypting new objects") + cmd.Flags().BoolVar(&f.heartbeatStdin, "heartbeat-stdin", false, "read heartbeat URLs from stdin, one per line (replaces the list)") + cmd.Flags().BoolVar(&f.clearHeartbeats, "clear-heartbeats", false, "remove every heartbeat URL") + return cmd +} + +// parseAdoptFlag parses [:[:all|db1,db2]]. +func parseAdoptFlag(s string) (backup.AdoptedEntry, error) { + parts := strings.SplitN(s, ":", 3) + e := backup.AdoptedEntry{Container: parts[0], Databases: backup.DatabaseSelector{All: true}} + if len(parts) > 1 { + e.Engine = parts[1] + } + if len(parts) > 2 && parts[2] != "all" { + e.Databases = backup.DatabaseSelector{Names: strings.Split(parts[2], ",")} + } + return e, nil +} + +// applyPolicyFlags merges the changed flags over a policy. +func applyPolicyFlags(cmd *cobra.Command, p *backup.FleetPolicy, f *policySetFlags) error { + ch := cmd.Flags().Changed + if ch("interval") { + p.Interval = f.interval + if f.interval == "none" { + p.Interval = "" + } + } + if ch("keep-last") { + p.Retention.KeepLast = f.keepLast + } + if ch("keep-daily") { + p.Retention.KeepDaily = f.keepDaily + } + if ch("keep-weekly") { + p.Retention.KeepWeekly = f.keepWeekly + } + if ch("keep-monthly") { + p.Retention.KeepMonthly = f.keepMonthly + } + if ch("max-age-days") { + p.Retention.MaxAgeDays = f.maxAgeDays + } + if ch("exclude") { + p.Excludes = f.excludes + } + if ch("volumes") { + p.Volumes = f.volumes + } + if ch("volume-max-gb") { + p.VolumeMaxGB = f.volumeMaxGB + } + if ch("recipient") && ch("no-recipient") { + return fmt.Errorf("--recipient and --no-recipient are mutually exclusive") + } + if ch("recipient") { + p.Recipient = f.recipient + } + if f.noRecipient { + p.Recipient = "" + } + for _, name := range f.removeAdopted { + kept := p.Adopted[:0] + for _, a := range p.Adopted { + if a.Container != name { + kept = append(kept, a) + } + } + p.Adopted = kept + } + for _, s := range f.adopts { + e, err := parseAdoptFlag(s) + if err != nil { + return err + } + replaced := false + for i := range p.Adopted { + if p.Adopted[i].Container == e.Container { + p.Adopted[i], replaced = e, true + } + } + if !replaced { + p.Adopted = append(p.Adopted, e) + } + } + if f.clearHeartbeats { + p.HeartbeatURLs = nil + } + return nil +} + +func readHeartbeatLines(r io.Reader) ([]string, error) { + b, err := io.ReadAll(io.LimitReader(r, 1<<16)) + if err != nil { + return nil, err + } + var urls []string + for _, line := range strings.Split(string(b), "\n") { + if line = strings.TrimSpace(line); line != "" { + urls = append(urls, line) + } + } + return urls, nil +} + +func runServerBackupPolicySet(cmd *cobra.Command, flags *Flags, f *policySetFlags) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + if f.stdin && f.heartbeatStdin { + return fmt.Errorf("--stdin and --heartbeat-stdin both read stdin: put heartbeat_urls in the JSON") + } + if f.stdin && cmd.Flags().NFlag() > 1 { + return fmt.Errorf("--stdin takes the whole policy: combine it with no other policy flags") + } + var incoming *backup.FleetPolicy + var heartbeats []string + if f.stdin { + dec := json.NewDecoder(io.LimitReader(os.Stdin, 1<<20)) + dec.DisallowUnknownFields() + var p backup.FleetPolicy + if err := dec.Decode(&p); err != nil { + return fmt.Errorf("reading the policy from stdin: %w", err) + } + incoming = &p + } else if f.heartbeatStdin { + var err error + if heartbeats, err = readHeartbeatLines(os.Stdin); err != nil { + return fmt.Errorf("reading heartbeat URLs from stdin: %w", err) + } + if len(heartbeats) == 0 { + return fmt.Errorf("--heartbeat-stdin: no URLs on stdin") + } + } + + host, user, key, err := configResolveServerForTarget(flags) + if err != nil { + return err + } + if !flags.JSON { + fmt.Printf("Connecting to %s@%s...\n", user, host) + } + executor, err := ssh.Connect(ctx, ssh.ConnectConfig{Host: host, User: user, KeyPath: key}) + if err != nil { + return err + } + defer executor.Close() + return setBackupPolicyOn(ctx, cmd, executor, flags, f, incoming, heartbeats) +} + +func setBackupPolicyOn(ctx context.Context, cmd *cobra.Command, executor ssh.Executor, flags *Flags, f *policySetFlags, incoming *backup.FleetPolicy, heartbeats []string) error { + stored, err := backup.ReadFleetPolicy(ctx, executor) + if err != nil { + return err + } + p := stored + if incoming != nil { + p = *incoming + } else { + if err := applyPolicyFlags(cmd, &p, f); err != nil { + return err + } + if heartbeats != nil { + p.HeartbeatURLs = heartbeats + } + } + p.V = "1" + if err := p.Validate(); err != nil { + return err + } + if p.Retention.Policy().IsZero() && p.Interval != "" { + fmt.Fprintln(os.Stderr, "Warning: this policy has no retention rule, so backups are never pruned") + } + + // Everything that can refuse does so before the policy is stored. + if p.Interval != "" { + if err := backup.RequireScheduler(ctx, executor); err != nil { + return err + } + } + if p.Recipient != "" { + if err := secret.NewManager(executor).EnsureAge(ctx); err != nil { + return err + } + } + bin := "" + if p.Interval != "" { + if bin, err = ensureFleetBinary(ctx, executor); err != nil { + return err + } + } + target, err := backup.ReadTarget(ctx, executor) + if err != nil { + return err + } + if !TargetConfigured(target) { + fmt.Fprintln(os.Stderr, "Warning: this server has no backup target yet, so runs will fail until `teploy server backup-target set` stores one") + } + + path, fallback, err := backup.WriteFleetPolicy(ctx, executor, p) + if err != nil { + return err + } + p.Path = path + + client := backup.NewClient(executor, backupProgressOut(flags)) + cronSpec := "" + if p.Interval == "" { + if err := client.RemoveSchedule(ctx, backup.FleetCronMarker); err != nil { + return err + } + } else { + cronSpec, err = backup.IntervalCron(p.Interval, executor.Host()) + if err != nil { + return err + } + if err := client.SetSchedule(ctx, cronSpec, bin+" backup run --all", backup.FleetCronMarker); err != nil { + return err + } + } + events.Record(ctx, executor, events.Event{App: backup.FleetApp, Kind: events.KindBackupPolicy, Outcome: events.Succeeded, Detail: map[string]any{ + "interval": p.Interval, "encrypted": p.Recipient != "", "volumes": p.Volumes, "adopted": len(p.Adopted), + }}) + if flags.JSON { + return json.NewEncoder(os.Stdout).Encode(policyDTO(executor.Host(), p, cronSpec)) + } + fmt.Printf("Backup policy stored at %s\n", path) + if fallback { + fmt.Println(" /etc/teploy is not writable by this user — the policy lives in ~/.config/teploy instead") + } + printPolicy(p, cronSpec) + return nil +} + +func policyDTO(host string, p backup.FleetPolicy, cronSpec string) backupPolicyDTO { + d := backupPolicyDTO{ + MachineInterface: MachineInterface, Host: host, Configured: p.Configured(), Path: p.Path, + Interval: p.Interval, Cron: cronSpec, Retention: p.Retention, Excludes: p.Excludes, Adopted: p.Adopted, + Volumes: p.Volumes, VolumeMaxGB: p.VolumeMaxGB, HeartbeatCount: len(p.HeartbeatURLs), + Recipient: p.Recipient, Encrypted: p.Recipient != "", + } + for _, h := range backup.MaskedHeartbeats(p.HeartbeatURLs) { + d.HeartbeatHosts = append(d.HeartbeatHosts, strings.TrimSuffix(h, "/...")) + } + return d +} + +func printPolicy(p backup.FleetPolicy, cronSpec string) { + if p.Interval != "" { + fmt.Printf(" Interval: %s (cron %q: teploy backup run --all)\n", p.Interval, cronSpec) + } else { + fmt.Println(" Interval: none (no cron line; manual runs only)") + } + r := p.Retention + fmt.Printf(" Retention: keep last %d, daily %d, weekly %d, monthly %d, everything younger than %d days\n", r.KeepLast, r.KeepDaily, r.KeepWeekly, r.KeepMonthly, r.MaxAgeDays) + if p.Recipient != "" { + fmt.Printf(" Encryption: on, to %s and the server's own identity\n", p.Recipient) + } else { + fmt.Println(" Encryption: OFF (no recovery key); app .env files are left out of volume backups") + } + fmt.Printf(" Volumes: %v", p.Volumes) + if p.VolumeMaxGB > 0 { + fmt.Printf(" (cap %.2f GB per archive)", p.VolumeMaxGB) + } + fmt.Println() + for _, a := range p.Adopted { + dbs := "all" + if !a.Databases.All { + dbs = strings.Join(a.Databases.Names, ", ") + } + fmt.Printf(" Adopted: %s (%s; databases: %s)\n", a.Container, orDetected(a.Engine), dbs) + } + if len(p.Excludes) > 0 { + fmt.Printf(" Excludes: %s\n", strings.Join(p.Excludes, ", ")) + } + for _, h := range backup.MaskedHeartbeats(p.HeartbeatURLs) { + fmt.Printf(" Heartbeat: %s\n", h) + } +} + +func orDetected(engine string) string { + if engine == "" { + return "engine detected" + } + return engine +} + +// ensureFleetBinary is ensureScheduledBackupBinary for the policy line: the +// server's teploy must speak `backup run --all`, and an older one is upgraded. +func ensureFleetBinary(ctx context.Context, executor ssh.Executor) (string, error) { + bin := scheduledTeployPath() + supports := func() bool { + return !ssh.RunDetailed(ctx, executor, ssh.ShellQuote(bin)+" backup run --help 2>/dev/null | grep -q -- --all").Failed() + } + if supports() { + return bin, nil + } + if bin != serverTeployPath { + return "", fmt.Errorf("%s does not support `backup run --all` (check TEPLOY_SERVER_BINARY)", bin) + } + if _, err := deployTeployBinaryToServer(ctx, executor, serverTeployPath, "backup run"); err != nil { + return "", fmt.Errorf("the server's teploy binary does not support `backup run --all` and installing one failed: %w", err) + } + if !supports() { + return "", fmt.Errorf("the installed teploy binary still does not support `backup run --all` — the latest release may predate it") + } + return bin, nil +} + +func newServerBackupPolicyShowCmd(flags *Flags) *cobra.Command { + return &cobra.Command{ + Use: "show", + Short: "Show the stored policy (heartbeat URLs only as hosts)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + host, user, key, err := configResolveServerForTarget(flags) + if err != nil { + return err + } + executor, err := ssh.Connect(ctx, ssh.ConnectConfig{Host: host, User: user, KeyPath: key}) + if err != nil { + return err + } + defer executor.Close() + p, err := backup.ReadFleetPolicy(ctx, executor) + if err != nil { + return err + } + cronSpec := "" + if lines, err := backup.ReadCrontab(ctx, executor); err == nil { + for _, line := range lines { + if m := backup.FindCronMarker(line); m != nil && m.Kind == backup.KindFleet { + cronSpec = m.Cron + } + } + } + if flags.JSON { + return json.NewEncoder(os.Stdout).Encode(policyDTO(executor.Host(), p, cronSpec)) + } + if !p.Configured() { + fmt.Println("No backup policy configured — set one with `teploy server backup-policy set`") + return nil + } + fmt.Printf("Backup policy (%s)\n", p.Path) + printPolicy(p, cronSpec) + return nil + }, + } +} + +func newServerBackupPolicyUnsetCmd(flags *Flags) *cobra.Command { + return &cobra.Command{ + Use: "unset", + Short: "Remove the policy and its cron line (stored backups are kept)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + host, user, key, err := configResolveServerForTarget(flags) + if err != nil { + return err + } + executor, err := ssh.Connect(ctx, ssh.ConnectConfig{Host: host, User: user, KeyPath: key}) + if err != nil { + return err + } + defer executor.Close() + // Cron first: a line with no policy behind it would fail every interval. + if err := backup.NewClient(executor, io.Discard).RemoveSchedule(ctx, backup.FleetCronMarker); err != nil { + return err + } + removed, err := backup.UnsetFleetPolicy(ctx, executor) + if err != nil { + return err + } + if len(removed) == 0 { + fmt.Println("No backup policy configured") + return nil + } + for _, p := range removed { + fmt.Printf("Removed %s\n", p) + } + fmt.Println("The cron line is gone and new backups stop. Stored backups were not touched; nothing else on this server encrypts them any more.") + return nil + }, + } +} + +// ---- backup status ---- + +// backupStatusDTO is `backup status --json` (backup-status-envelope). +type backupStatusDTO struct { + MachineInterface int `json:"machine_interface"` + Host string `json:"host"` + ObservedAt time.Time `json:"observed_at"` + backup.FleetStatus +} + +func newBackupStatusCmd(flags *Flags) *cobra.Command { + return &cobra.Command{ + Use: "status", + Short: "Every database on the server: covered, stale, failing, uncovered, excluded or adopted", + Long: "One read-only call for a whole server: every DB-like container (labelled or\n" + + "not), every app's volumes, what covers each, its last success and failure,\n" + + "its newest object and last verify, every database nothing backs up, and the\n" + + "target and policy. No secrets. --host names the server.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + host, user, key, err := configResolveServerForTarget(flags) + if err != nil { + return err + } + executor, err := ssh.Connect(ctx, ssh.ConnectConfig{Host: host, User: user, KeyPath: key}) + if err != nil { + return err + } + defer executor.Close() + now := time.Now().UTC() + sctx, scancel := context.WithTimeout(ctx, 10*time.Minute) + defer scancel() + st, err := backup.ReadFleetStatus(sctx, executor, now) + if err != nil { + return err + } + if flags.JSON { + return json.NewEncoder(os.Stdout).Encode(backupStatusDTO{MachineInterface: MachineInterface, Host: executor.Host(), ObservedAt: now, FleetStatus: st}) + } + printBackupStatus(st) + return nil + }, + } +} + +func printBackupStatus(st backup.FleetStatus) { + if st.Target.Configured { + fmt.Printf("Target: s3://%s", st.Target.Bucket) + if st.Target.Prefix != "" { + fmt.Printf("/%s", st.Target.Prefix) + } + if !st.Target.Reachable { + fmt.Printf(" (NOT REACHABLE: %s)", st.Target.Error) + } + fmt.Println() + } else { + fmt.Println("Target: none configured") + } + if st.Policy.Configured { + fmt.Printf("Policy: every %s, encryption %v, %d heartbeat(s)\n", st.Policy.Interval, st.Encrypted, st.Policy.HeartbeatCount) + } else { + fmt.Println("Policy: none configured") + } + if st.LastRun != nil { + fmt.Printf("Last run: %s at %s (%d ok, %d failed)\n", st.LastRun.Status, st.LastRun.At.Format(time.RFC3339), st.LastRun.TargetsOK, st.LastRun.TargetsFailed) + } + for _, e := range st.Entries { + newest := "-" + if e.Newest != nil { + newest = fmt.Sprintf("%s %d bytes", e.Newest.CreatedAt, e.Newest.SizeBytes) + if e.Newest.Encrypted { + newest += " encrypted" + } + } + fmt.Printf("%-36s %-10s %-10s %s", e.Name, e.Engine, e.Coverage, newest) + if e.Reason != "" { + fmt.Printf(" (%s)", e.Reason) + } + fmt.Println() + } + if len(st.UncoveredDatabases) > 0 { + fmt.Printf("NOT BACKED UP: %s\n", strings.Join(st.UncoveredDatabases, ", ")) + } + for _, w := range st.Warnings { + fmt.Printf("Warning: %s\n", w) + } +} + +// ---- backup keygen ---- + +func newBackupKeygenCmd(flags *Flags) *cobra.Command { + return &cobra.Command{ + Use: "keygen", + Short: "Generate a recovery keypair, printed once and stored nowhere", + Long: "Generates the owner recovery key backups are encrypted to. The private key\n" + + "is printed once, here, and is not written anywhere: store it offline. Without\n" + + "it, a lost server's backups cannot be read. Give the public key to\n" + + "`teploy server backup-policy set --recipient`, then prove the private key\n" + + "works: `teploy backup fetch --identity `.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return writeBackupKeygen(os.Stdout, flags.JSON) + }, + } +} + +func writeBackupKeygen(w io.Writer, jsonOut bool) error { + id, err := age.GenerateX25519Identity() + if err != nil { + return fmt.Errorf("generating a keypair: %w", err) + } + if jsonOut { + return json.NewEncoder(w).Encode(map[string]any{"machine_interface": MachineInterface, "public_key": id.Recipient().String(), "private_key": id.String()}) + } + fmt.Fprintln(w, "# Teploy backup recovery key. Shown once; stored nowhere. Keep it offline.") + fmt.Fprintf(w, "# public key: %s\n", id.Recipient()) + fmt.Fprintln(w, id.String()) + return nil +} + +// ---- backup fetch ---- + +// backupFetchDTO is `backup fetch --json`. +type backupFetchDTO struct { + MachineInterface int `json:"machine_interface"` + Key string `json:"key"` + Output string `json:"output"` + SizeBytes int64 `json:"size_bytes"` + Encrypted bool `json:"encrypted"` +} + +func newBackupFetchCmd(flags *Flags) *cobra.Command { + var identity, output, region, endpoint string + cmd := &cobra.Command{ + Use: "fetch ", + Short: "Download and decrypt one backup object here, with no server involved", + Long: "Runs on THIS machine: the owner's recovery drill, and the way back when the\n" + + "server is gone. It needs the aws CLI, the bucket's credentials\n" + + "(TEPLOY_S3_ACCESS_KEY/SECRET_KEY or AWS_* in the environment) and, for a .age\n" + + "object, the recovery key (--identity, the file `teploy backup keygen`'s output\n" + + "was saved to). The key is the s3:// URI from `teploy backup list`. A partial\n" + + "or wrong result is never left behind: the output appears only once it is\n" + + "complete and authenticated.\n\n" + + "Example:\n" + + " teploy backup fetch s3://b/prod/app/accessories/db/20261104-030000-ab.pgdump.age \\\n" + + " --identity recovery.txt --endpoint https://s3.example.com", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + dto, err := runBackupFetch(ctx, args[0], identity, output, region, endpoint) + if err != nil { + return err + } + if flags.JSON { + return json.NewEncoder(os.Stdout).Encode(dto) + } + fmt.Printf("Fetched %s (%d bytes)%s\n", dto.Output, dto.SizeBytes, map[bool]string{true: ", decrypted with the given identity", false: ""}[dto.Encrypted]) + return nil + }, + } + cmd.Flags().StringVar(&identity, "identity", "", "age identity file (required for .age objects)") + cmd.Flags().StringVarP(&output, "output", "o", "", "where to write the result (default: the object's name, minus .age, in this directory)") + cmd.Flags().StringVar(®ion, "region", "", "AWS region (default: us-east-1)") + cmd.Flags().StringVar(&endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2)") + return cmd +} + +func runBackupFetch(ctx context.Context, key, identity, output, region, endpoint string) (backupFetchDTO, error) { + var dto backupFetchDTO + if !strings.HasPrefix(key, "s3://") || strings.ContainsAny(key, "\r\n\x00") { + return dto, fmt.Errorf("the key must be an s3:// URI (see `teploy backup list`)") + } + encrypted := strings.HasSuffix(key, ".age") + var ids []age.Identity + if encrypted { + if identity == "" { + return dto, fmt.Errorf("%s is encrypted: pass --identity ", filepath.Base(key)) + } + f, err := os.Open(identity) + if err != nil { + return dto, fmt.Errorf("opening the identity file: %w", err) + } + ids, err = age.ParseIdentities(f) + f.Close() + if err != nil { + return dto, fmt.Errorf("reading the identity file: %w", err) + } + } + if endpoint != "" { + if err := backup.ValidateEndpoint(endpoint); err != nil { + return dto, err + } + } + if region == "" { + region = "us-east-1" + } + if err := backup.ValidateRegion(region); err != nil { + return dto, err + } + if output == "" { + output = strings.TrimSuffix(filepath.Base(key), ".age") + } + if _, err := os.Lstat(output); err == nil { + return dto, fmt.Errorf("%s already exists: refusing to overwrite it (choose another --output)", output) + } + dir := filepath.Dir(output) + tmp, err := os.MkdirTemp(dir, ".teploy-fetch-") + if err != nil { + return dto, fmt.Errorf("creating a private work directory beside %s: %w", output, err) + } + defer os.RemoveAll(tmp) + raw := filepath.Join(tmp, "object") + if err := localAWSCopy(ctx, key, raw, region, endpoint); err != nil { + return dto, err + } + result := raw + if encrypted { + result = filepath.Join(tmp, "plain") + if err := decryptFile(raw, result, ids); err != nil { + return dto, fmt.Errorf("decrypting %s with %s: %w (a recovery key from before a rotation is needed for older objects)", filepath.Base(key), identity, err) + } + } + if err := os.Chmod(result, 0o600); err != nil { + return dto, err + } + st, err := os.Stat(result) + if err != nil { + return dto, err + } + if err := os.Rename(result, output); err != nil { + return dto, err + } + return backupFetchDTO{MachineInterface: MachineInterface, Key: key, Output: output, SizeBytes: st.Size(), Encrypted: encrypted}, nil +} + +func decryptFile(src, dst string, ids []age.Identity) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + r, err := age.Decrypt(in, ids...) + if err != nil { + return err + } + out, err := os.OpenFile(dst, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + if _, err := io.Copy(out, r); err != nil { + out.Close() + return err + } + return out.Close() +} + +// ---- adopted addressing ---- + +// resolveBackupApp is resolveApp plus the adopted namespace: with adopted, the +// target is the server named by --host, and the app is the namespace. +func resolveBackupApp(ctx context.Context, flags *Flags, appName string, adopted bool) (*config.AppConfig, ssh.Executor, error) { + if !adopted { + return resolveApp(ctx, flags, appName) + } + if appName != "" { + return nil, nil, fmt.Errorf("--adopted and --app are mutually exclusive: an adopted container belongs to no app") + } + host, user, key, err := configResolveServerForTarget(flags) + if err != nil { + return nil, nil, err + } + executor, err := ssh.Connect(ctx, ssh.ConnectConfig{Host: host, User: user, KeyPath: key}) + if err != nil { + return nil, nil, err + } + return &config.AppConfig{App: backup.AdoptedApp}, executor, nil +} + +// applyAdoptedEngine gives the client the policy entry's engine: for an adopted +// container that override has no label to live on. +func applyAdoptedEngine(ctx context.Context, client *backup.Client, exec ssh.Executor, app, container string) error { + if app != backup.AdoptedApp { + return nil + } + if err := backup.ValidateContainerName(container); err != nil { + return err + } + p, err := backup.ReadFleetPolicy(ctx, exec) + if err != nil { + return err + } + for _, a := range p.Adopted { + if a.Container == container { + return client.SetEngineOverride(container, a.Engine) + } + } + return nil +} + +// ---- backup before destructive operations ---- + +// preOperationBackup stores a copy of what op is about to destroy. A failure +// refuses the operation and says exactly what is at stake; --no-backup is the +// operator's explicit way past it (a restore is sometimes the fix for what is +// breaking the backup, and the local safety copy still exists). Nothing to +// protect (no such container) is not a failure. +func preOperationBackup(ctx context.Context, client *backup.Client, exec ssh.Executor, app, accessory, database string, cfg backup.S3Config, op string) (string, error) { + var res backup.BackupResult + var err error + if accessory == "" { + // An empty or missing tree has nothing to protect, and restoring into one + // is exactly what must not be refused. + d := ssh.ShellQuote("/deployments/" + app + "/volumes") + if out, lsErr := exec.Run(ctx, "if [ -n \"$(ls -A "+d+" 2>/dev/null)\" ]; then echo yes; fi"); lsErr == nil && strings.TrimSpace(out) != "yes" { + return "", nil + } + res, err = client.BackupVolumesWith(ctx, app, cfg, backup.BackupOptions{}) + recordPreBackupEvent(ctx, exec, app, accessory, res, err, op) + if err != nil { + return "", fmt.Errorf("the automatic backup before %s failed, so nothing was changed: %w\nFix it, or pass --no-backup to go ahead without one", op, err) + } + return res.ID, nil + } + image, env, inspErr := client.InspectAccessory(ctx, app, accessory) + if inspErr != nil { + return "", nil + } + // Only what op will replace, by name: a database that is already gone has + // nothing to protect, and restoring it is exactly what must not be refused. + if database == "" { + database = client.PrimaryDatabase(ctx, app, accessory, image, env) + } + res, err = client.AccessoryBackupWith(ctx, app, accessory, image, env, cfg, backup.BackupOptions{SingleDatabase: database}) + if err != nil || len(res.Objects) > 0 { + recordPreBackupEvent(ctx, exec, app, accessory, res, err, op) + } + if err != nil { + return "", fmt.Errorf("the automatic backup of %s before %s failed, so nothing was changed: %w\nFix it, or pass --no-backup to go ahead without one", backup.ContainerName(app, accessory), op, err) + } + if len(res.Objects) == 0 { + return "", nil + } + return res.ID, nil +} + +func recordPreBackupEvent(ctx context.Context, exec ssh.Executor, app, accessory string, res backup.BackupResult, err error, op string) { + kind := backup.KindAccessory + if accessory == "" { + kind = backup.KindVolumes + } + detail := backupRunDetail(res, kind, accessory) + detail["trigger"] = op + recordAppEvent(ctx, exec, app, events.KindBackupRun, err, detail) +} + +// resolvePreBackupTarget is the S3 config the safety backups use: the server's +// stored target. No target is a refusal naming the way out. +func resolvePreBackupTarget(ctx context.Context, exec ssh.Executor, op string) (backup.S3Config, error) { + target, err := backup.ReadTarget(ctx, exec) + if err != nil { + return backup.S3Config{}, err + } + if !TargetConfigured(target) { + return backup.S3Config{}, fmt.Errorf("%s destroys data and this server has no backup target to save a copy to: set one with `teploy server backup-target set`, or pass --no-backup to go ahead without one", op) + } + cfg := TargetS3(target) + if err := backup.ApplyPolicyEncryption(ctx, exec, &cfg); err != nil { + return backup.S3Config{}, err + } + return cfg, nil +} + +// deployNoBackup is `deploy --no-backup`. +var deployNoBackup bool + +// backupBeforeDeployHook backs up each running database accessory before a deploy +// whose pre_deploy hook is about to run. A first deploy has nothing to protect. +func backupBeforeDeployHook(ctx context.Context, exec ssh.Executor, appCfg *config.AppConfig, out io.Writer) error { + client := backup.NewClient(exec, out) + var cfg backup.S3Config + haveCfg := false + for _, name := range sortedAccessoryNames(appCfg.Accessories) { + image, env, err := client.InspectAccessory(ctx, appCfg.App, name) + if err != nil { + continue + } + det, err := client.ResolveEngine(ctx, appCfg.App, name, image, env) + if err != nil || !det.IsDB() { + continue + } + if !haveCfg { + if cfg, err = resolvePreBackupTarget(ctx, exec, "a deploy with backup_before_hooks"); err != nil { + return err + } + haveCfg = true + } + id, err := preOperationBackup(ctx, client, exec, appCfg.App, name, "", cfg, "this deploy's pre_deploy hook") + if err != nil { + return err + } + fmt.Fprintf(out, "Backed up %s before the pre_deploy hook: %s\n", backup.ContainerName(appCfg.App, name), id) + } + return nil +} diff --git a/internal/cli/backup_fleet_test.go b/internal/cli/backup_fleet_test.go new file mode 100644 index 0000000..e1d9910 --- /dev/null +++ b/internal/cli/backup_fleet_test.go @@ -0,0 +1,502 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "filippo.io/age" + "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/backup" + "github.com/useteploy/teploy/internal/ssh" +) + +const testRecipient = "age1lggyhqrw2nlhcxprm67z43rta597azn8gknawjehu9d9dl0jq3yqqvfafg" + +func TestParseAdoptFlag(t *testing.T) { + e, err := parseAdoptFlag("omni_postgres") + if err != nil || e.Container != "omni_postgres" || !e.Databases.All || e.Engine != "" { + t.Fatalf("%+v %v", e, err) + } + e, _ = parseAdoptFlag("m:mysql:wp_a,wp_b") + if e.Engine != "mysql" || e.Databases.All || strings.Join(e.Databases.Names, ",") != "wp_a,wp_b" { + t.Fatalf("%+v", e) + } + e, _ = parseAdoptFlag("m:mysql:all") + if !e.Databases.All { + t.Fatalf("%+v", e) + } +} + +// Flags change only what they name. +func TestApplyPolicyFlagsMerges(t *testing.T) { + cmd := &cobra.Command{} + var f policySetFlags + cmd.Flags().StringVar(&f.interval, "interval", "", "") + cmd.Flags().IntVar(&f.keepLast, "keep-last", 0, "") + cmd.Flags().IntVar(&f.keepDaily, "keep-daily", 0, "") + cmd.Flags().IntVar(&f.keepWeekly, "keep-weekly", 0, "") + cmd.Flags().IntVar(&f.keepMonthly, "keep-monthly", 0, "") + cmd.Flags().IntVar(&f.maxAgeDays, "max-age-days", 0, "") + cmd.Flags().StringSliceVar(&f.excludes, "exclude", nil, "") + cmd.Flags().BoolVar(&f.volumes, "volumes", false, "") + cmd.Flags().Float64Var(&f.volumeMaxGB, "volume-max-gb", 0, "") + cmd.Flags().StringVar(&f.recipient, "recipient", "", "") + cmd.Flags().BoolVar(&f.noRecipient, "no-recipient", false, "") + if err := cmd.ParseFlags([]string{"--keep-daily", "7"}); err != nil { + t.Fatal(err) + } + f.adopts = []string{"a:postgres:all", "b"} + f.removeAdopted = []string{"gone"} + p := backup.FleetPolicy{Interval: "1h", Retention: backup.FleetRetention{KeepLast: 3, KeepDaily: 1}, Recipient: testRecipient, Volumes: true, + Adopted: []backup.AdoptedEntry{{Container: "gone", Databases: backup.DatabaseSelector{All: true}}, {Container: "a", Databases: backup.DatabaseSelector{Names: []string{"x"}}}}, + HeartbeatURLs: []string{"https://h/t"}, + } + if err := applyPolicyFlags(cmd, &p, &f); err != nil { + t.Fatal(err) + } + if p.Interval != "1h" || p.Retention.KeepLast != 3 || p.Retention.KeepDaily != 7 || p.Recipient != testRecipient || !p.Volumes || len(p.HeartbeatURLs) != 1 { + t.Fatalf("%+v", p) + } + if len(p.Adopted) != 2 || p.Adopted[0].Container != "a" || !p.Adopted[0].Databases.All || p.Adopted[1].Container != "b" { + t.Fatalf("adopted: %+v", p.Adopted) + } +} + +func TestRestorePromptNamesWhatIsReplaced(t *testing.T) { + for _, tc := range []struct{ app, acc, db, want string }{ + {"blog", "db", "", "Replace the data in blog-db"}, + {"blog", "db", "reports", "Replace database reports in blog-db"}, + {"blog", "", "", "Replace everything in blog's volumes"}, + {backup.AdoptedApp, "shared", "wp_a", "Replace database wp_a in adopted container shared"}, + } { + got := restorePrompt(tc.app, tc.acc, tc.db, "20260101-000000") + if !strings.Contains(got, tc.want) || !strings.Contains(got, "20260101-000000") { + t.Errorf("%q does not name %q", got, tc.want) + } + } + if !strings.Contains(restorePrompt(backup.AdoptedApp, "s", "d", "x"), "Nothing that uses it is stopped") { + t.Error("an adopted restore must say nothing is stopped for you") + } +} + +func TestMergePreviousPrefixes(t *testing.T) { + old := backup.Target{Bucket: "b", Endpoint: "e", Prefix: "", PreviousPrefixes: []backup.PreviousPrefix{{Prefix: "older"}}} + prev, lost := mergePreviousPrefixes(old, "b", "e", "srv1", false, false) + if lost || len(prev) != 2 || prev[0].Prefix != "older" || prev[1].Prefix != "" || prev[1].Owned { + t.Fatalf("%+v %v", prev, lost) + } + // Moving back to an old prefix takes it off the list. + prev, _ = mergePreviousPrefixes(backup.Target{Bucket: "b", Endpoint: "e", Prefix: "srv1", PreviousPrefixes: []backup.PreviousPrefix{{Prefix: ""}}}, "b", "e", "", false, false) + if len(prev) != 1 || prev[0].Prefix != "srv1" { + t.Fatalf("%+v", prev) + } + // Same prefix: no new history, and ownership can be declared. + prev, _ = mergePreviousPrefixes(backup.Target{Bucket: "b", Endpoint: "e", Prefix: "srv1", PreviousPrefixes: []backup.PreviousPrefix{{Prefix: ""}}}, "b", "e", "srv1", true, false) + if len(prev) != 1 || !prev[0].Owned { + t.Fatalf("%+v", prev) + } + if prev, lost := mergePreviousPrefixes(old, "other", "e", "x", false, false); !lost || prev != nil { + t.Fatalf("a new bucket starts fresh: %+v %v", prev, lost) + } + if prev, lost := mergePreviousPrefixes(old, "b", "e", "x", false, true); lost || prev != nil { + t.Fatalf("forget: %+v %v", prev, lost) + } + if prev, lost := mergePreviousPrefixes(backup.Target{}, "b", "e", "x", false, false); lost || prev != nil { + t.Fatalf("first target: %+v %v", prev, lost) + } +} + +func TestBackupKeygenPrintsAUsablePair(t *testing.T) { + var b bytes.Buffer + if err := writeBackupKeygen(&b, false); err != nil { + t.Fatal(err) + } + ids, err := age.ParseIdentities(strings.NewReader(b.String())) + if err != nil || len(ids) != 1 { + t.Fatalf("the printed output must parse as an identity file: %v", err) + } + if !strings.Contains(b.String(), "# public key: age1") { + t.Fatalf("%s", b.String()) + } + var j bytes.Buffer + if err := writeBackupKeygen(&j, true); err != nil { + t.Fatal(err) + } + var dto struct { + Public string `json:"public_key"` + Private string `json:"private_key"` + } + if err := json.Unmarshal(j.Bytes(), &dto); err != nil { + t.Fatal(err) + } + if backup.ValidateRecipient(dto.Public) != nil || !strings.HasPrefix(dto.Private, "AGE-SECRET-KEY-") { + t.Fatalf("%+v", dto) + } +} + +// fetchFixture puts a fake aws on PATH that copies a local file as "the object". +func fetchFixture(t *testing.T, recipient age.Recipient, plain string) string { + t.Helper() + dir := t.TempDir() + obj := filepath.Join(dir, "object") + f, err := os.Create(obj) + if err != nil { + t.Fatal(err) + } + w, err := age.Encrypt(f, recipient) + if err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(w, plain); err != nil { + t.Fatal(err) + } + w.Close() + f.Close() + bin := filepath.Join(dir, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + script := "#!/bin/sh\n[ \"$1\" = s3 ] && [ \"$2\" = cp ] || exit 2\ncp " + obj + " \"$4\"\n" + if err := os.WriteFile(filepath.Join(bin, "aws"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + return obj +} + +func TestBackupFetchRecoversWithOnlyTheOwnerIdentity(t *testing.T) { + id, err := age.GenerateX25519Identity() + if err != nil { + t.Fatal(err) + } + fetchFixture(t, id.Recipient(), "the dump") + dir := t.TempDir() + idFile := filepath.Join(dir, "recovery.txt") + if err := os.WriteFile(idFile, []byte("# comment\n"+id.String()+"\n"), 0o600); err != nil { + t.Fatal(err) + } + out := filepath.Join(dir, "restored.pgdump") + ctx := context.Background() + dto, err := runBackupFetch(ctx, "s3://b/a/accessories/db/1.pgdump.age", idFile, out, "", "") + if err != nil { + t.Fatal(err) + } + if !dto.Encrypted || dto.SizeBytes != 8 { + t.Fatalf("%+v", dto) + } + if b, _ := os.ReadFile(out); string(b) != "the dump" { + t.Fatalf("content %q", b) + } + st, _ := os.Stat(out) + if st.Mode().Perm() != 0o600 { + t.Fatalf("mode %v", st.Mode()) + } + // Refuses to overwrite. + if _, err := runBackupFetch(ctx, "s3://b/k.pgdump.age", idFile, out, "", ""); err == nil || !strings.Contains(err.Error(), "already exists") { + t.Fatalf("overwrite: %v", err) + } + // A wrong identity leaves nothing behind. + wrong, _ := age.GenerateX25519Identity() + wrongFile := filepath.Join(dir, "wrong.txt") + os.WriteFile(wrongFile, []byte(wrong.String()+"\n"), 0o600) + out2 := filepath.Join(dir, "second.pgdump") + if _, err := runBackupFetch(ctx, "s3://b/k.pgdump.age", wrongFile, out2, "", ""); err == nil { + t.Fatal("a wrong identity must fail") + } + entries, _ := os.ReadDir(dir) + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".teploy-fetch-") || e.Name() == "second.pgdump" { + t.Fatalf("left behind: %s", e.Name()) + } + } + // An encrypted object needs an identity; a key must be a key. + if _, err := runBackupFetch(ctx, "s3://b/k.pgdump.age", "", filepath.Join(dir, "x"), "", ""); err == nil || !strings.Contains(err.Error(), "--identity") { + t.Fatalf("no identity: %v", err) + } + if _, err := runBackupFetch(ctx, "/etc/passwd", "", filepath.Join(dir, "x"), "", ""); err == nil { + t.Fatal("not an s3 key") + } +} + +// A truncated ciphertext fails authentication: never a short "success". +func TestBackupFetchRefusesATruncatedObject(t *testing.T) { + id, _ := age.GenerateX25519Identity() + obj := fetchFixture(t, id.Recipient(), strings.Repeat("x", 200000)) + b, _ := os.ReadFile(obj) + os.WriteFile(obj, b[:len(b)/2], 0o600) + dir := t.TempDir() + idFile := filepath.Join(dir, "id") + os.WriteFile(idFile, []byte(id.String()+"\n"), 0o600) + out := filepath.Join(dir, "out") + if _, err := runBackupFetch(context.Background(), "s3://b/k.age", idFile, out, "", ""); err == nil { + t.Fatal("a truncated object must fail") + } + if _, err := os.Stat(out); err == nil { + t.Fatal("no output may appear") + } +} + +func TestFleetFailedExitStatus(t *testing.T) { + if fleetFailed(backup.FleetRunResult{Status: backup.RunOK}) != nil || fleetFailed(backup.FleetRunResult{Status: "skipped"}) != nil { + t.Fatal("ok and skipped exit zero") + } + if err := fleetFailed(backup.FleetRunResult{Status: backup.RunPartial, Targets: []backup.FleetTargetResult{{Status: backup.TargetFailed}}}); err == nil || !strings.Contains(err.Error(), "1 target") { + t.Fatalf("%v", err) + } + if err := fleetFailed(backup.FleetRunResult{Status: backup.RunFailed, Error: "boom"}); err == nil || !strings.Contains(err.Error(), "boom") { + t.Fatalf("%v", err) + } +} + +// runFleet with no target: fails, heartbeats down, names what to do, and the +// token reaches neither output nor event. +func TestRunFleetNoTargetHeartbeatsDown(t *testing.T) { + var got []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { got = append(got, r.URL.RequestURI()) })) + defer srv.Close() + dir := t.TempDir() + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","interval":"1h","heartbeat_urls":["` + srv.URL + `/api/push/SECRET"]}`}, + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "absent\n"}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq":1,"ts":"2026-01-01T00:00:00Z"}`}, + ) + var out bytes.Buffer + res, err := runFleet(context.Background(), mock, &out, false, nil) + if err == nil || res.Status != backup.RunFailed || !strings.Contains(res.Error, "backup-target set") { + t.Fatalf("%v %+v", err, res) + } + if len(got) != 1 || !strings.Contains(got[0], "status=down") { + t.Fatalf("heartbeats %v", got) + } + if strings.Contains(out.String(), "SECRET") { + t.Fatalf("output leaks the token: %s", out.String()) + } + for _, input := range mock.Inputs { + if strings.Contains(input, "SECRET") { + t.Fatalf("event leaks the token: %s", input) + } + } +} + +func TestRunFleetWithNoPolicyFails(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "absent\n"}) + res, err := runFleet(context.Background(), mock, io.Discard, false, nil) + if err == nil || res.Status != backup.RunFailed || !strings.Contains(res.Error, "backup-policy set") { + t.Fatalf("%v %+v", err, res) + } +} + +// A run that finds the lock held stands down: exit zero, no heartbeat, no event. +func TestRunFleetStandsDownOnOverlap(t *testing.T) { + var got []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { got = append(got, r.URL.RequestURI()) })) + defer srv.Close() + dir := t.TempDir() + lock, _, err := backup.AcquireRunLock(backup.RunLockPath(dir+"/backup-policy.json"), time.Now(), time.Hour) + if err != nil { + t.Fatal(err) + } + defer lock.Release() + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","heartbeat_urls":["` + srv.URL + `/hb"]}`}, + ) + res, err := runFleet(context.Background(), mock, io.Discard, false, nil) + if err != nil || res.Status != "skipped" { + t.Fatalf("%v %+v", err, res) + } + if len(got) != 0 { + t.Fatalf("a skipped run must not heartbeat: %v", got) + } +} + +// One target failing does not stop the next; exit, event and heartbeat say so. +func TestRunFleetPartialFailure(t *testing.T) { + var got []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { got = append(got, r.URL.RequestURI()) })) + defer srv.Close() + dir := t.TempDir() + line := func(name, app, acc string) string { + b, _ := json.Marshal(map[string]any{"name": "/" + name, "image": "redis:7", "state": "running", + "labels": map[string]string{"teploy.role": "accessory", "teploy.app": app, "teploy.accessory": acc}, + "env": []string{"REDIS_VERSION=7"}, "entrypoint": []string{}, "cmd": []string{}}) + return string(b) + } + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","interval":"1h","retention":{"keep_last":1},"heartbeat_urls":["` + srv.URL + `/hb"]}`}, + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"b","region":"r"}`}, + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "for d in", Output: ""}, + ssh.MockCommand{Match: "( c=$(python3", Output: ""}, + ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: line("a-cache", "a", "cache") + "\n" + line("b-cache", "b", "cache") + "\n"}, + ssh.MockCommand{Match: "crontab -l", Output: ""}, + ssh.MockCommand{Match: "cat", Output: ""}, + ssh.MockCommand{Match: "docker inspect -f", Output: "{}\n[]\n[]\n"}, + ssh.MockCommand{Match: "umask 077", Output: "/var/tmp/teploy-backup.t\n"}, + ssh.MockCommand{Match: "docker exec 'a-cache'", Err: errors.New("exit status 1: no such container")}, + ssh.MockCommand{Match: "set -eu", Err: errors.New("exit status 1: bgsave refused")}, + ssh.MockCommand{Match: "docker exec", Output: ""}, + ssh.MockCommand{Match: "rm -rf", Output: ""}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq":1,"ts":"2026-01-01T00:00:00Z"}`}, + ) + var out bytes.Buffer + res, err := runFleet(context.Background(), mock, &out, false, nil) + if err == nil { + t.Fatal("a run with failures must exit nonzero") + } + if len(res.Targets) != 2 { + t.Fatalf("the second target must still run: %+v", res.Targets) + } + if res.Status != backup.RunFailed { + t.Fatalf("both failed here: %s", res.Status) + } + if len(got) != 1 || !strings.Contains(got[0], "status=down") { + t.Fatalf("heartbeats %v", got) + } + for _, c := range mock.Calls { + if strings.Contains(c, "s3 rm") { + t.Fatalf("pruned after failures: %s", c) + } + } +} + +// ---- backup before destructive operations ---- + +// The safety backup failing refuses the operation and names the way past it. +func TestPreOperationBackupRefuses(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "which aws", Err: errors.New("exit status 1")}, + ssh.MockCommand{Match: "docker inspect -f '{{.Config.Image}}'", Output: "postgres:17\n"}, + ssh.MockCommand{Match: "docker inspect -f '{{range .Config.Env}}", Output: "PG_MAJOR=17\n"}, + ssh.MockCommand{Match: "docker inspect -f", Output: "{}\n[]\n[]\n"}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq":1,"ts":"2026-01-01T00:00:00Z"}`}, + ) + client := backup.NewClient(mock, io.Discard) + _, err := preOperationBackup(context.Background(), client, mock, "a", "db", "", backup.S3Config{Bucket: "b", Region: "r"}, "this restore") + if err == nil || !strings.Contains(err.Error(), "--no-backup") || !strings.Contains(err.Error(), "nothing was changed") { + t.Fatalf("err = %v", err) + } +} + +// No such container: nothing to protect, not a refusal. +func TestPreOperationBackupNothingToProtect(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "docker inspect -f", Err: errors.New("exit status 1: no such object")}) + id, err := preOperationBackup(context.Background(), backup.NewClient(mock, io.Discard), mock, "a", "db", "", backup.S3Config{Bucket: "b", Region: "r"}, "this restore") + if err != nil || id != "" { + t.Fatalf("%q %v", id, err) + } +} + +// A database that is already gone has nothing to protect: restoring it after it +// was dropped must not fail the same way the database did. +func TestPreOperationBackupOfADroppedDatabaseIsNotARefusal(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "docker inspect -f '{{.Config.Image}}'", Output: "postgres:17\n"}, + ssh.MockCommand{Match: "docker inspect -f '{{range .Config.Env}}", Output: "PG_MAJOR=17\nPOSTGRES_DB=app\n"}, + ssh.MockCommand{Match: "docker inspect -f", Output: "{}\n[]\n[]\n"}, + ssh.MockCommand{Match: "umask 077", Output: "/var/tmp/teploy-backup.t\n"}, + ssh.MockCommand{Match: "docker exec 'a-db' psql -X -qtA -v ON_ERROR_STOP=1 -U 'postgres' -d 'postgres' -c 'SELECT datname", Output: "postgres\n"}, + ssh.MockCommand{Match: "docker exec", Output: "0\n"}, + ssh.MockCommand{Match: "rm -rf", Output: ""}, + ssh.MockCommand{Match: "{ docker", Output: ""}, + ssh.MockCommand{Match: "aws s3", Output: ""}, + ssh.MockCommand{Match: "sha256sum", Output: "abc f\n4096\n"}, + ssh.MockCommand{Match: "aws s3api", Output: `{"ContentLength":4096,"Metadata":{"sha256":"abc"}}`}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq":1,"ts":"2026-01-01T00:00:00Z"}`}, + ) + id, err := preOperationBackup(context.Background(), backup.NewClient(mock, io.Discard), mock, "a", "db", "", backup.S3Config{Bucket: "b", Region: "r"}, "this restore") + if err != nil || id != "" { + t.Fatalf("a dropped database must not refuse the restore that brings it back: %q %v", id, err) + } +} + +func TestCronMarkerRecognisesThePolicyLine(t *testing.T) { + m := backup.FindCronMarker("5 * * * * /usr/local/bin/teploy backup run --all # [teploy-backup-all]") + if m == nil || m.Kind != backup.KindFleet || m.Form != backup.CronFormRun || m.Cron != "5 * * * *" { + t.Fatalf("%+v", m) + } +} + +func TestPolicyDTOCarriesNoToken(t *testing.T) { + d := policyDTO("h", backup.FleetPolicy{V: "1", HeartbeatURLs: []string{"https://dash.example/api/push/SECRET"}}, "") + b, _ := json.Marshal(d) + if strings.Contains(string(b), "SECRET") || d.HeartbeatCount != 1 || d.HeartbeatHosts[0] != "https://dash.example" { + t.Fatalf("%s", b) + } +} + +func TestEnsureServerPathCompletesCronPath(t *testing.T) { + t.Setenv("PATH", "/usr/bin:/bin") + ensureServerPath() + if !strings.Contains(os.Getenv("PATH"), "/usr/local/bin") { + t.Fatal(os.Getenv("PATH")) + } +} + +// The machine path: the whole policy on stdin. It is validated before anything is +// stored, gets exactly one marked cron line, and is reported back with no token. +func TestSetBackupPolicyFromStdin(t *testing.T) { + t.Setenv("TEPLOY_SERVER_BINARY", "/opt/teploy") + mock := ssh.NewMockExecutor("srv", + ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "absent\n"}, + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"b","region":"r"}`}, + ssh.MockCommand{Match: "sched_daemon=no", Output: "daemon=yes\nenabled=enabled\ncrontab=yes\nteploy_entries=0\n"}, + ssh.MockCommand{Match: "which age", Output: "/usr/bin/age"}, + ssh.MockCommand{Match: "'/opt/teploy' backup run --help", Output: ""}, + ssh.MockCommand{Match: "d=\nif [ \"$(id -u)\" = 0 ]", Output: "written /etc/teploy/backup-policy.json\n"}, + ssh.MockCommand{Match: "raw=$(crontab -l", Output: ""}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq":1,"ts":"2026-01-01T00:00:00Z"}`}, + ) + p := backup.FleetPolicy{Interval: "1h", Retention: backup.FleetRetention{KeepDaily: 7}, Recipient: testRecipient, + HeartbeatURLs: []string{"https://dash.example/api/push/SECRET"}, + Adopted: []backup.AdoptedEntry{{Container: "omni_postgres", Databases: backup.DatabaseSelector{All: true}}}} + stdout := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + err := setBackupPolicyOn(context.Background(), &cobra.Command{}, mock, &Flags{JSON: true}, &policySetFlags{stdin: true}, &p, nil) + w.Close() + os.Stdout = stdout + if err != nil { + t.Fatal(err) + } + printed, _ := io.ReadAll(r) + if strings.Contains(string(printed), "SECRET") || !strings.Contains(string(printed), `"heartbeat_count":1`) || !strings.Contains(string(printed), `"encrypted":true`) { + t.Fatalf("%s", printed) + } + stored := strings.Join(mock.Inputs, "\n") + if !strings.Contains(stored, "SECRET") || !strings.Contains(stored, `"v":"1"`) { + t.Fatalf("the policy must reach the server's file: %s", stored) + } + lines := 0 + for _, c := range mock.Calls { + if strings.Contains(c, "/opt/teploy backup run --all # [teploy-backup-all]") { + lines++ + } + if strings.Contains(c, "SECRET") { + t.Fatalf("a token reached a command line: %s", c) + } + } + if lines != 1 { + t.Fatalf("cron lines: %d\n%v", lines, mock.Calls) + } + + // Refused before anything is stored. + bad := backup.FleetPolicy{Adopted: []backup.AdoptedEntry{{Container: "x'; rm -rf /", Databases: backup.DatabaseSelector{All: true}}}} + refused := ssh.NewMockExecutor("srv", ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "absent\n"}) + if err := setBackupPolicyOn(context.Background(), &cobra.Command{}, refused, &Flags{}, &policySetFlags{stdin: true}, &bad, nil); err == nil { + t.Fatal("a hostile container name must be refused") + } + if len(refused.Calls) != 1 { + t.Fatalf("refused policy touched the server: %v", refused.Calls) + } +} diff --git a/internal/cli/backup_run.go b/internal/cli/backup_run.go index 639be6d..6623aa2 100644 --- a/internal/cli/backup_run.go +++ b/internal/cli/backup_run.go @@ -21,19 +21,36 @@ import ( func newBackupRunCmd(flags *Flags) *cobra.Command { var ( - appName string - accessory string + appName string + accessory string + all bool + ignoreSpace bool ) cmd := &cobra.Command{ Use: "run", Short: "Run a scheduled backup on the server (the cron entrypoint)", - Args: cobra.NoArgs, + Long: "Runs ON the server. --app backs up one app's volumes or one accessory (a\n" + + "per-app schedule). --all is the policy line: it reads the server's backup\n" + + "policy and works out at run time what to back up (every labelled accessory\n" + + "and each of its databases, app volumes when enabled, adopted containers), one\n" + + "target at a time. One target failing does not stop the others; the exit status\n" + + "is nonzero for anything but a clean run, and nothing of a failed target is\n" + + "pruned. A run that finds another one running stands down.", + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { + if all { + if appName != "" || accessory != "" { + return fmt.Errorf("--all takes no --app or --accessory: it works out its own targets") + } + return runBackupRunAll(flags, ignoreSpace) + } return runBackupRun(flags, appName, accessory) }, } - cmd.Flags().StringVar(&appName, "app", "", "app name (required)") + cmd.Flags().StringVar(&appName, "app", "", "app name (required without --all)") cmd.Flags().StringVar(&accessory, "accessory", "", "back up this accessory instead of the app's volumes") + cmd.Flags().BoolVar(&all, "all", false, "back up everything the server's backup policy covers") + cmd.Flags().BoolVar(&ignoreSpace, "ignore-space-check", false, "with --all: skip the free-space preflights") return cmd } @@ -48,6 +65,7 @@ func runBackupRun(flags *Flags, appName, accessory string) error { } // Runs on the server itself: no SSH hop, local docker and aws. + ensureServerPath() executor := ssh.NewLocalExecutor() defer executor.Close() return runBackupRunOn(context.Background(), executor, appName, accessory) @@ -65,6 +83,9 @@ func runBackupRunOn(ctx context.Context, executor ssh.Executor, appName, accesso return fmt.Errorf("no backup target is configured on this server — run `teploy server backup-target set` from a machine that manages it") } cfg := TargetS3(target) + if err := backup.ApplyPolicyEncryption(ctx, executor, &cfg); err != nil { + return err + } client := backup.NewClient(executor, os.Stdout) policy, err := backup.ReadPolicy(ctx, executor, appName) diff --git a/internal/cli/backup_target.go b/internal/cli/backup_target.go index bd8d833..6d6bce0 100644 --- a/internal/cli/backup_target.go +++ b/internal/cli/backup_target.go @@ -59,6 +59,8 @@ type backupTargetDTO struct { Prefix string `json:"prefix,omitempty"` HasCredentials bool `json:"has_credentials"` Path string `json:"path,omitempty"` + // PreviousPrefixes are old prefixes of this bucket list and restore search. + PreviousPrefixes []backup.PreviousPrefix `json:"previous_prefixes,omitempty"` } func newServerBackupTargetSetCmd(flags *Flags) *cobra.Command { @@ -68,6 +70,8 @@ func newServerBackupTargetSetCmd(flags *Flags) *cobra.Command { endpoint string prefix string noCredentials bool + prevOwned bool + dropPrevious bool ) cmd := &cobra.Command{ Use: "set", @@ -77,10 +81,15 @@ func newServerBackupTargetSetCmd(flags *Flags) *cobra.Command { "lines: the access key, then the secret key. With a terminal, both are\n" + "prompted with echo off.\n\n" + "Example:\n" + - " printf 'ACCESS\\nSECRET\\n' | teploy server backup-target set --host prod --bucket b --endpoint https://s3.example.com", + " printf 'ACCESS\\nSECRET\\n' | teploy server backup-target set --host prod --bucket b --endpoint https://s3.example.com\n\n" + + "Give each server its own --prefix when servers share a bucket. Changing the\n" + + "prefix keeps the old one on record: list, verify and restore still find the\n" + + "objects there. They are not pruned, because they may be another server's;\n" + + "--previous-prefix-owned says only this server ever wrote there, and lets them\n" + + "age out under the policy's retention.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - return runServerBackupTargetSet(flags, bucket, region, endpoint, prefix, noCredentials) + return runServerBackupTargetSet(flags, bucket, region, endpoint, prefix, noCredentials, prevOwned, dropPrevious) }, } cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name") @@ -88,10 +97,38 @@ func newServerBackupTargetSetCmd(flags *Flags) *cobra.Command { cmd.Flags().StringVar(&endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2)") cmd.Flags().StringVar(&prefix, "prefix", "", "key prefix under the bucket (optional)") cmd.Flags().BoolVar(&noCredentials, "no-credentials", false, "store no credentials — use the server's ambient AWS credential chain") + cmd.Flags().BoolVar(&prevOwned, "previous-prefix-owned", false, "the previous prefix holds only this server's objects: let retention age them out") + cmd.Flags().BoolVar(&dropPrevious, "forget-previous-prefixes", false, "stop looking under previous prefixes (their objects are left in the bucket)") return cmd } -func runServerBackupTargetSet(flags *Flags, bucket, region, endpoint, prefix string, noCredentials bool) error { +// mergePreviousPrefixes carries a target's prefix history across a `set`: the +// prefix being left joins the list, the prefix being entered leaves it. A new +// bucket or endpoint starts fresh, since the old location is not reachable from +// the new config. +func mergePreviousPrefixes(old backup.Target, bucket, endpoint, prefix string, owned, drop bool) (prev []backup.PreviousPrefix, lost bool) { + if !TargetConfigured(old) { + return nil, false + } + if old.Bucket != bucket || old.Endpoint != endpoint { + return nil, true + } + if drop { + return nil, false + } + for _, p := range old.PreviousPrefixes { + if p.Prefix != prefix && p.Prefix != old.Prefix { + p.Owned = p.Owned || owned + prev = append(prev, p) + } + } + if old.Prefix != prefix { + prev = append(prev, backup.PreviousPrefix{Prefix: old.Prefix, Owned: owned}) + } + return prev, false +} + +func runServerBackupTargetSet(flags *Flags, bucket, region, endpoint, prefix string, noCredentials, prevOwned, dropPrevious bool) error { if bucket == "" { return fmt.Errorf("--bucket is required") } @@ -142,7 +179,13 @@ func runServerBackupTargetSet(flags *Flags, bucket, region, endpoint, prefix str return err } } + old, err := backup.ReadTarget(ctx, executor) + if err != nil { + return err + } t := backup.Target{Bucket: bucket, Region: region, Endpoint: endpoint, Prefix: prefix, HasCredentials: credsArmor != ""} + var lostOld bool + t.PreviousPrefixes, lostOld = mergePreviousPrefixes(old, bucket, endpoint, prefix, prevOwned, dropPrevious) path, fallback, err := backup.WriteTarget(ctx, executor, t, credsArmor) if err != nil { return err @@ -151,6 +194,19 @@ func runServerBackupTargetSet(flags *Flags, bucket, region, endpoint, prefix str fmt.Printf("Backup target stored: s3://%s at %s\n", backupTargetLabel(t), path) fmt.Printf(" Credentials: %s\n", credentialMode(t)) + if lostOld { + fmt.Printf(" Note: backups under the old target s3://%s are not reachable from this one and are no longer listed; they were not touched\n", backupTargetLabel(old)) + } + for _, p := range t.PreviousPrefixes { + owned := "kept until you remove them (they may be another server's; --previous-prefix-owned lets them age out)" + if p.Owned { + owned = "ages out under the policy's retention" + } + fmt.Printf(" Previous prefix %q: still searched by list and restore; %s\n", p.Prefix, owned) + } + if prefix == "" { + fmt.Println(" No prefix: if other servers share this bucket, give each its own --prefix. A namespace another server has written to is never pruned") + } if fallback { fmt.Println(" /etc/teploy is not writable by this user — the target lives in ~/.config/teploy instead") } @@ -187,12 +243,13 @@ func checkReachability(ctx context.Context, executor ssh.Executor, t backup.Targ cfg := TargetS3(t) client := backup.NewClient(executor, io.Discard) if err := client.EnsureAWSCLIPublic(ctx); err != nil { - fmt.Fprintf(os.Stderr, "Warning: could not verify the target: %v\n", err) + // The target is stored, and every backup through it will fail until this + // is fixed: say so in the terms that fix it. + fmt.Fprintf(os.Stderr, "Warning: backups through this target will fail: %v\n", err) return } - res := ssh.RunDetailed(ctx, executor, cfg.AWS("s3 ls s3://"+cfg.Bucket+"/")) - if res.Failed() { - fmt.Fprintf(os.Stderr, "Warning: the stored target did not answer `aws s3 ls` (%s) — check the bucket, endpoint and credentials\n", oneLine(res.ExitErrorText())) + if err := client.ProbeTarget(ctx, cfg); err != nil { + fmt.Fprintf(os.Stderr, "Warning: the stored target did not answer `aws s3 ls` (%s) — check the bucket, endpoint and credentials\n", oneLine(err.Error())) return } fmt.Println(" Reachability: aws s3 ls answered") @@ -346,6 +403,7 @@ func writeBackupTargetJSON(w io.Writer, host string, target backup.Target) error Host: host, HasCredentials: target.HasCredentials, Path: target.Path, + PreviousPrefixes: target.PreviousPrefixes, } if TargetConfigured(target) { dto.Configured = true @@ -527,7 +585,8 @@ func serverAppNames(ctx context.Context, executor ssh.Executor) ([]string, error seen := map[string]bool{} var apps []string for _, line := range strings.Split(string(res.Stdout), "\n") { - if app := strings.TrimSpace(line); app != "" && !seen[app] { + // _adopted and _fleet are event-log namespaces, not apps. + if app := strings.TrimSpace(line); app != "" && !strings.HasPrefix(app, "_") && !seen[app] { seen[app] = true apps = append(apps, app) } diff --git a/internal/cli/contracts_golden_test.go b/internal/cli/contracts_golden_test.go index 965adf2..0bd82ba 100644 --- a/internal/cli/contracts_golden_test.go +++ b/internal/cli/contracts_golden_test.go @@ -611,3 +611,63 @@ func TestContractsGitSourceGolden(t *testing.T) { MachineInterface: MachineInterface, App: "blog", Host: "", Ref: "refs/heads/main", Admitted: false, Reason: "not-due", }) } + +// TestContractsBackupFleetGolden pins the fleet engine's envelopes (rev 17, +// backup-fleet-v1). Dash's backups page builds on backup-status-envelope. +func TestContractsBackupFleetGolden(t *testing.T) { + at := time.Date(2026, 11, 4, 3, 17, 4, 0, time.UTC) + failedAt := at.Add(-time.Hour) + key := "s3://teploy-backups/prod/_adopted/accessories/wordpress-shared-mysql/databases/wp_blog/20261104-031704-1f2e3d4c5b6a7988.sql.gz.age" + newest := &backup.StatusObject{ID: "20261104-031704-1f2e3d4c5b6a7988", Key: key, SizeBytes: 2097152, CreatedAt: "2026-11-04T03:17:04Z", Encrypted: true} + policy := backup.FleetPolicy{ + V: "1", Interval: "1h", Path: "/etc/teploy/backup-policy.json", + Retention: backup.FleetRetention{KeepLast: 3, KeepDaily: 7, KeepWeekly: 4, KeepMonthly: 12, MaxAgeDays: 2}, + Excludes: []string{"blog/volumes"}, + Adopted: []backup.AdoptedEntry{ + {Container: "wordpress-shared-mysql", Engine: "mysql", Databases: backup.DatabaseSelector{All: true}}, + {Container: "omni_postgres", Databases: backup.DatabaseSelector{Names: []string{"omni"}}}, + }, + Volumes: true, VolumeMaxGB: 20, + HeartbeatURLs: []string{"https://dash.example.internal/api/push/secret-token"}, + Recipient: "age1lggyhqrw2nlhcxprm67z43rta597azn8gknawjehu9d9dl0jq3yqqvfafg", + } + writeFixture(t, "backup-policy-envelope/valid/configured.json", policyDTO("192.0.2.10", policy, "37 * * * *")) + writeFixture(t, "backup-policy-envelope/valid/unconfigured.json", policyDTO("192.0.2.10", backup.FleetPolicy{}, "")) + + status := backupStatusDTO{MachineInterface: MachineInterface, Host: "192.0.2.10", ObservedAt: at, FleetStatus: backup.FleetStatus{ + Target: backup.StatusTarget{Configured: true, Bucket: "teploy-backups", Region: "us-east-1", Prefix: "prod", HasCredentials: true, Reachable: true, + PreviousPrefixes: []backup.PreviousPrefix{{Prefix: ""}}}, + Policy: backup.StatusPolicy{Configured: true, Path: policy.Path, Interval: "1h", CronInstalled: true, Retention: policy.Retention, Excludes: policy.Excludes, Adopted: policy.Adopted, + Volumes: true, VolumeMaxGB: 20, HeartbeatCount: 1, HeartbeatHosts: []string{"https://dash.example.internal"}, Recipient: policy.Recipient}, + Encrypted: true, AWSCLI: true, AgeCLI: true, + LastRun: &backup.StatusLastRun{At: at, Status: backup.RunPartial, TargetsOK: 3, TargetsFailed: 1, Omissions: []string{"redis-cache: a database container no policy entry covers (adopt or exclude it)"}}, + Entries: []backup.StatusEntry{ + {Name: "blog-db", Kind: backup.TargetAccessory, App: "blog", Accessory: "db", Engine: "postgres", Detection: "env", State: "running", Coverage: backup.CoverageCovered, Schedule: "policy", LastSuccess: &at, Newest: &backup.StatusObject{ID: "20261104-031704-1f2e3d4c5b6a7988", Key: "s3://teploy-backups/prod/blog/accessories/db/20261104-031704-1f2e3d4c5b6a7988.pgdump.age", SizeBytes: 1048576, CreatedAt: "2026-11-04T03:17:04Z", Encrypted: true}, LastVerify: &backup.StatusVerify{At: at, OK: true, ID: "20261104-031704-1f2e3d4c5b6a7988", Metric: "tables=42"}}, + {Name: "wordpress-shared-mysql", Kind: backup.TargetAdopted, Accessory: "wordpress-shared-mysql", Engine: "mysql", Detection: "policy", State: "running", Unmanaged: true, Adopted: true, Coverage: backup.CoverageAdopted, Schedule: "policy", LastSuccess: &at, Newest: newest, Databases: []backup.StatusDatabase{{Name: "wp_blog", Newest: newest}}}, + {Name: "omni_postgres", Kind: backup.TargetAdopted, Accessory: "omni_postgres", Engine: "postgres", Detection: "env", State: "running", Unmanaged: true, Adopted: true, Coverage: backup.CoverageFailing, Schedule: "policy", LastFailure: &failedAt, LastError: "dumping omni: exit status 1", UncoveredDatabases: []string{"reports"}}, + {Name: "redis-cache", Kind: "unmanaged", Engine: "redis", Detection: "env", State: "running", Unmanaged: true, Coverage: backup.CoverageUncovered, Reason: "no policy entry covers this container (adopt or exclude it)", Schedule: "none"}, + {Name: "blog/volumes", Kind: backup.TargetVolumes, App: "blog", Coverage: backup.CoverageExcluded, Schedule: "policy", SizeBytes: 53687091200, Reason: "50.00 GB is over the policy's volume_max_gb 20.00"}, + }, + UncoveredDatabases: []string{"omni_postgres/reports", "redis-cache"}, + }} + writeFixture(t, "backup-status-envelope/valid/fleet.json", status) + writeFixture(t, "backup-status-envelope/valid/unconfigured.json", backupStatusDTO{MachineInterface: MachineInterface, Host: "192.0.2.10", ObservedAt: at, FleetStatus: backup.FleetStatus{ + Entries: []backup.StatusEntry{}, UncoveredDatabases: []string{}, + }}) + + writeFixture(t, "backup-run-envelope/valid/partial.json", backupRunEnvelope{MachineInterface: MachineInterface, FleetRunResult: backup.FleetRunResult{ + Status: backup.RunPartial, StartedAt: at, DurationMs: 8123, Encrypted: true, + Targets: []backup.FleetTargetResult{ + {Kind: backup.TargetAccessory, App: "blog", Accessory: "db", Container: "blog-db", Engine: "postgres", Status: backup.TargetOK, ID: "20261104-031704-1f2e3d4c5b6a7988", SizeBytes: 1048576, DurationMs: 900, Encrypted: true, Pruned: 2, + Objects: []backup.BackupObject{{Key: "s3://teploy-backups/prod/blog/accessories/db/20261104-031704-1f2e3d4c5b6a7988.pgdump.age", Size: 1048576, Encrypted: true}}}, + {Kind: backup.TargetAdopted, App: backup.AdoptedApp, Accessory: "omni_postgres", Container: "omni_postgres", Engine: "postgres", Status: backup.TargetFailed, Reason: "dumping omni: exit status 1", Encrypted: true, FailedDatabases: map[string]string{"omni": "exit status 1"}}, + {Kind: backup.TargetVolumes, App: "blog", Status: backup.TargetExcluded, Reason: "volumes are 50.00 GB, over the policy's volume_max_gb 20.00; not backed up", SizeBytes: 53687091200, Encrypted: true, SizeGuard: true}, + }, + UncoveredContainers: []string{"redis-cache"}, + Omissions: []string{"blog/volumes: volumes are 50.00 GB, over the policy's volume_max_gb 20.00; not backed up", "redis-cache: a database container no policy entry covers (adopt or exclude it)"}, + Heartbeats: []backup.HeartbeatResult{{Host: "dash.example.internal", OK: true, Status: 200}}, + }}) + writeFixture(t, "backup-run-envelope/valid/skipped.json", backupRunEnvelope{MachineInterface: MachineInterface, FleetRunResult: backup.FleetRunResult{ + Status: "skipped", StartedAt: at, Targets: []backup.FleetTargetResult{}, Error: "another run holds the lock", + }}) +} diff --git a/internal/cli/dr.go b/internal/cli/dr.go index 73ea87a..3ec4291 100644 --- a/internal/cli/dr.go +++ b/internal/cli/dr.go @@ -8,6 +8,7 @@ import ( "io" "os" "os/signal" + "strings" "github.com/spf13/cobra" "github.com/useteploy/teploy/internal/backup" @@ -74,7 +75,11 @@ func drStore(ctx context.Context, exec ssh.Executor, bucket, region, endpoint, d if err := backup.ValidateRegion(r); err != nil { return nil, err } - return backup.S3BundleStore{S3: s3Config(bucket, r, endpoint)}, nil + cfg := s3Config(bucket, r, endpoint) + if err := backup.ApplyPolicyEncryption(ctx, exec, &cfg); err != nil { + return nil, err + } + return backup.S3BundleStore{S3: cfg}, nil } target, err := backup.ReadTarget(ctx, exec) if err != nil { @@ -83,7 +88,52 @@ func drStore(ctx context.Context, exec ssh.Executor, bucket, region, endpoint, d if !TargetConfigured(target) { return nil, fmt.Errorf("a bundle target is required: --bucket (S3), --dir (server directory), or a stored server backup target (`teploy server backup-target set`)") } - return backup.S3BundleStore{S3: TargetS3(target)}, nil + cfg := TargetS3(target) + if err := backup.ApplyPolicyEncryption(ctx, exec, &cfg); err != nil { + return nil, err + } + return backup.S3BundleStore{S3: cfg}, nil +} + +// withBundleIdentity puts a local identity file on the server for this restore +// only, in a private directory removed by the returned cleanup. An encrypted +// bundle restored on a host that did not make it can be read no other way; that +// is the one time the recovery key leaves the owner's machine, and only when +// asked for by flag. +func withBundleIdentity(ctx context.Context, exec ssh.Executor, store backup.BundleStore, identity string) (backup.BundleStore, string, func(), error) { + none := func() {} + if identity == "" { + return store, "", none, nil + } + body, err := os.ReadFile(identity) + if err != nil { + return nil, "", none, fmt.Errorf("reading the identity file: %w", err) + } + out, err := exec.Run(ctx, "umask 077; mktemp -d /var/tmp/teploy-dr-identity.XXXXXXXX") + if err != nil { + return nil, "", none, fmt.Errorf("creating a private directory for the identity: %w", err) + } + dir := strings.TrimSpace(out) + cleanup := func() { exec.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(dir)) } + path := dir + "/identity" + if err := exec.Upload(ctx, strings.NewReader(string(body)), path, "0600"); err != nil { + cleanup() + return nil, "", none, fmt.Errorf("placing the identity: %w", err) + } + if s3, ok := store.(backup.S3BundleStore); ok { + s3.S3.ExtraIdentity = path + store = s3 + } + return store, path, cleanup, nil +} + +// bundleRecipient is the recovery key an included age key is wrapped to. +func bundleRecipient(ctx context.Context, exec ssh.Executor) (string, error) { + p, err := backup.ReadFleetPolicy(ctx, exec) + if err != nil { + return "", err + } + return p.Recipient, nil } func newDRCreateCmd(flags *Flags, version string) *cobra.Command { @@ -130,11 +180,18 @@ func newDRCreateCmd(flags *Flags, version string) *cobra.Command { progress = os.Stderr } client := backup.NewClient(executor, progress) + recipient := "" + if includeAgeKey { + if recipient, err = bundleRecipient(ctx, executor); err != nil { + return err + } + } m, err := client.CreateBundle(ctx, backup.BundleOptions{ App: appCfg.App, Config: *appCfg, IncludeSecrets: includeSecrets, IncludeAgeKey: includeAgeKey, + OwnerRecipient: recipient, StopApp: stopApp, VolumeQuiesced: quiescedVolumes, Version: version, @@ -159,7 +216,7 @@ func newDRCreateCmd(flags *Flags, version string) *cobra.Command { } drStoreFlags(cmd, &bucket, ®ion, &endpoint, &dir) cmd.Flags().BoolVar(&includeSecrets, "include-secrets", false, "include encrypted secret material (age ciphertexts, resolved .env, accessory credentials) — never default") - cmd.Flags().BoolVar(&includeAgeKey, "include-age-key", false, "also include /deployments/.age-key so a fresh host can decrypt (requires --include-secrets)") + cmd.Flags().BoolVar(&includeAgeKey, "include-age-key", false, "also include the server's age key, wrapped to the recovery key, so a fresh host can decrypt (requires --include-secrets and a policy --recipient)") cmd.Flags().BoolVar(&stopApp, "stop-app", false, "stop app containers for quiesced volume snapshots (restarted after)") cmd.Flags().StringSliceVar(&quiescedVolumes, "quiesced-volume", nil, "volume NAME whose writer you stopped by hand (repeatable); recorded as quiesced, operator-asserted") return cmd @@ -266,7 +323,7 @@ func newDRShowCmd(flags *Flags) *cobra.Command { } func newDRRestoreCmd(flags *Flags) *cobra.Command { - var bucket, region, endpoint, dir string + var bucket, region, endpoint, dir, identity string cmd := &cobra.Command{ Use: "restore ", Short: "Restore a DR bundle to an ISOLATED target and validate it (live state untouched)", @@ -276,7 +333,10 @@ func newDRRestoreCmd(flags *Flags) *cobra.Command { "RPO/RTO. Nothing under /deployments is touched — promotion happens only\n" + "via `teploy dr cutover`.\n" + "Missing secret keys (references mode) and undecryptable material fail\n" + - "BEFORE any mutation.", + "BEFORE any mutation.\n" + + "--identity is the recovery key file: needed for a bundle encrypted on another\n" + + "host or carrying a wrapped age key. It is placed on the target privately for\n" + + "this restore and removed afterwards.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { appCfg, err := config.LoadApp(".") @@ -299,10 +359,16 @@ func newDRRestoreCmd(flags *Flags) *cobra.Command { progress = os.Stderr } client := backup.NewClient(executor, progress) + store, identityPath, cleanupIdentity, err := withBundleIdentity(ctx, executor, store, identity) + if err != nil { + return err + } + defer cleanupIdentity() receipt, err := client.RestoreBundleIsolated(ctx, backup.BundleRestoreOptions{ - App: appCfg.App, - ID: args[0], - Config: *appCfg, + App: appCfg.App, + ID: args[0], + Config: *appCfg, + Identity: identityPath, }, store) if err != nil { return err @@ -322,6 +388,7 @@ func newDRRestoreCmd(flags *Flags) *cobra.Command { }, } drStoreFlags(cmd, &bucket, ®ion, &endpoint, &dir) + cmd.Flags().StringVar(&identity, "identity", "", "recovery key file, for a bundle encrypted elsewhere or a wrapped age key") return cmd } diff --git a/internal/cli/machineinterface.go b/internal/cli/machineinterface.go index 682b1f2..c11626f 100644 --- a/internal/cli/machineinterface.go +++ b/internal/cli/machineinterface.go @@ -159,6 +159,13 @@ const ( // size_bytes, created_at, key; schedules discovered from teploy cron // markers) and restore quiesces the app under its fenced lock. CapBackupJSONV1 = "backup-json-v1" + // CapBackupFleetV1 is the fleet engine (Slice 4): `server backup-policy` + // (one cron line, `backup run --all`, adopted containers, GFS, heartbeats), + // `backup status --json`, `backup fetch`, `backup keygen`, payload encryption + // (.age objects), per-database series (`--database`, `--adopted`) and + // previous-prefix lookup. backup-status-envelope, backup-run-envelope and + // backup-policy-envelope pin the JSON. + CapBackupFleetV1 = "backup-fleet-v1" // CapSecretStdinV1: `secret set KEY --stdin` reads the value verbatim // from stdin (never argv), and secret set/get/list/rm/rotate accept // folderless `--app X --host S` (provider from the app's Teploy @@ -219,6 +226,7 @@ func MachineCapabilities() []string { CapTriggerPollV1, CapBackupTargetV1, CapBackupJSONV1, + CapBackupFleetV1, CapSecretStdinV1, CapReleasesJsonV1, CapRollbackJsonV1, diff --git a/internal/cli/machineinterface_test.go b/internal/cli/machineinterface_test.go index bde5325..e234328 100644 --- a/internal/cli/machineinterface_test.go +++ b/internal/cli/machineinterface_test.go @@ -104,7 +104,7 @@ func TestCapabilityTokenRegistry(t *testing.T) { "template-var-stdin", "trigger-convergence-v1", } - want = append(want, "actor-flag-v1", "events-v1", "config-home-v1", "undeployed-list-v1", "lock-wait-v1", "config-merge-v1", "config-unprivileged-v1", "deploy-home-v1", "backup-target-v1", "backup-json-v1") + want = append(want, "actor-flag-v1", "events-v1", "config-home-v1", "undeployed-list-v1", "lock-wait-v1", "config-merge-v1", "config-unprivileged-v1", "deploy-home-v1", "backup-target-v1", "backup-json-v1", "backup-fleet-v1") want = append(want, "secret-stdin-v1", "releases-json-v1", "rollback-json-v1", "redeploy-v1", "env-config-guard-v1", "git-source-v1", "trigger-poll-v1") sort.Strings(want) got := MachineCapabilities() diff --git a/internal/cli/root.go b/internal/cli/root.go index 3dfc9d4..b93d771 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -28,6 +28,8 @@ func teployVersionForServer() string { return cliBuildVersion } // Flags holds the global CLI flags, passed to subcommands explicitly. type Flags struct { + // NoBackup skips the backup_before_hooks backup (`deploy --no-backup`). + NoBackup bool Host string User string Key string @@ -210,12 +212,11 @@ func recordAppEvent(ctx context.Context, exec ssh.Executor, app, kind string, er // backupRunDetail is the backup.run event detail: what was written, where, // how big, how long it took, and the consistency the artifact can claim. -// Payload encryption (G1) is not implemented yet, so encrypted is false. func backupRunDetail(res backup.BackupResult, kind, accessory string) map[string]any { detail := map[string]any{ "id": res.ID, "kind": kind, "size_bytes": res.SizeBytes, "key": res.Key, "duration_ms": res.DurationMs, "engine": res.Engine, "consistency": res.Consistency, - "encrypted": false, + "encrypted": res.Encrypted, } if accessory != "" { detail["accessory"] = accessory diff --git a/internal/cli/server.go b/internal/cli/server.go index 0ba2e05..d0dd928 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -28,6 +28,7 @@ func newServerCmd(flags *Flags) *cobra.Command { cmd.AddCommand(newServerListCmd(flags)) cmd.AddCommand(newServerStatusCmd(flags)) cmd.AddCommand(newServerBackupTargetCmd(flags)) + cmd.AddCommand(newServerBackupPolicyCmd(flags)) return cmd } From 61520ca488920c78aca9183263e8f5329b76e9af Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Wed, 4 Nov 2026 22:34:36 -0700 Subject: [PATCH 04/17] feat(cli): backup before remove --purge and deploy hooks; purge names what it deletes remove --purge stores a fresh backup of every accessory and the volumes first, refuses if that fails or no target exists (--no-backup), names what it will delete in its prompt, and removes containers with docker rm -v. backup_before_hooks (default off) does the same for a deploy with a pre_deploy hook. Co-Authored-By: Claude Opus 5.5 --- internal/cli/backup_destructive_test.go | 86 ++++++++++++++++++++ internal/cli/deploy.go | 9 +++ internal/cli/remove.go | 100 ++++++++++++++++++++++-- internal/config/app.go | 17 ++-- internal/docker/docker.go | 8 ++ 5 files changed, 209 insertions(+), 11 deletions(-) create mode 100644 internal/cli/backup_destructive_test.go diff --git a/internal/cli/backup_destructive_test.go b/internal/cli/backup_destructive_test.go new file mode 100644 index 0000000..ced8aa8 --- /dev/null +++ b/internal/cli/backup_destructive_test.go @@ -0,0 +1,86 @@ +package cli + +import ( + "context" + "errors" + "io" + "strings" + "testing" + + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" +) + +func TestPurgeRefusesWithNoTarget(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "absent\n"}) + err := backupBeforePurge(context.Background(), mock, "a", []string{"db"}, true, io.Discard) + if err == nil || !strings.Contains(err.Error(), "--no-backup") || !strings.Contains(err.Error(), "no backup target") { + t.Fatalf("err = %v", err) + } + // Nothing to destroy: no target needed. + if err := backupBeforePurge(context.Background(), ssh.NewMockExecutor("h"), "a", nil, false, io.Discard); err != nil { + t.Fatal(err) + } +} + +func TestPurgeRemovesContainersWithTheirVolumes(t *testing.T) { + cmds := append([]ssh.MockCommand{ + removeTestContainers(), + {Match: "docker stop -t 10", Output: ""}, + {Match: "docker rm ", Output: ""}, + {Match: "rm -rf '/deployments/scratch'", Output: ""}, + {Match: "rmdir '/deployments/scratch'", Output: ""}, + }, caddyMocks()...) + exec := ssh.NewMockExecutor("h", append(cmds, removeAutomationMocks()...)...) + if _, err := executeRemove(context.Background(), exec, "scratch", []string{"scratch.com"}, removeOptions{Purge: true}, io.Discard); err != nil { + t.Fatal(err) + } + saw := false + for _, c := range exec.Calls { + if strings.HasPrefix(c, "docker rm ") { + if !strings.HasPrefix(c, "docker rm -v ") { + t.Fatalf("a purge left anonymous volumes behind: %s", c) + } + saw = true + } + } + if !saw { + t.Fatalf("no docker rm: %v", exec.Calls) + } + // Without --purge a container's anonymous volume is not this command's to take. + cmds = append([]ssh.MockCommand{ + removeTestContainers(), + {Match: "docker stop -t 10", Output: ""}, + {Match: "docker rm ", Output: ""}, + {Match: "rm -rf '/deployments/scratch", Output: ""}, + {Match: "rmdir '/deployments/scratch'", Output: ""}, + {Match: "find ", Output: ""}, + {Match: "rm -f", Output: ""}, + }, caddyMocks()...) + exec = ssh.NewMockExecutor("h", append(cmds, removeAutomationMocks()...)...) + _, _ = executeRemove(context.Background(), exec, "scratch", []string{"scratch.com"}, removeOptions{}, io.Discard) + for _, c := range exec.Calls { + if strings.HasPrefix(c, "docker rm -v ") { + t.Fatalf("no purge, no -v: %s", c) + } + } +} + +// The deploy hook backup needs no target until there is something to protect. +func TestBackupBeforeDeployHook(t *testing.T) { + cfg := &config.AppConfig{App: "a", Accessories: map[string]config.AccessoryConfig{"db": {Image: "postgres:17"}}} + none := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "docker inspect", Err: errors.New("exit status 1: no such object")}) + if err := backupBeforeDeployHook(context.Background(), none, cfg, io.Discard); err != nil { + t.Fatalf("a first deploy has nothing to protect: %v", err) + } + running := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "docker inspect -f '{{.Config.Image}}'", Output: "postgres:17\n"}, + ssh.MockCommand{Match: "docker inspect -f '{{range .Config.Env}}", Output: "PG_MAJOR=17\n"}, + ssh.MockCommand{Match: "docker inspect -f", Output: "{}\n[]\n[]\n"}, + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "absent\n"}, + ) + err := backupBeforeDeployHook(context.Background(), running, cfg, io.Discard) + if err == nil || !strings.Contains(err.Error(), "--no-backup") { + t.Fatalf("err = %v", err) + } +} diff --git a/internal/cli/deploy.go b/internal/cli/deploy.go index 7deff40..2021de0 100644 --- a/internal/cli/deploy.go +++ b/internal/cli/deploy.go @@ -109,6 +109,7 @@ a machine error envelope on stderr whose code distinguishes the classes cmd.Flags().StringVar(&image, "image", "", "Docker image to deploy (skips build if set)") cmd.Flags().StringVar(&version, "version", "", "version identifier (default: git short hash)") cmd.Flags().BoolVar(&skipDNSCheck, "skip-dns-check", false, "skip DNS validation (for proxied domains like Cloudflare)") + cmd.Flags().BoolVar(&deployNoBackup, "no-backup", false, "skip the backup_before_hooks backup") cmd.Flags().IntVar(¶llel, "parallel", 0, "max concurrent deploys for multi-server (default: from teploy.yml or 1)") cmd.Flags().StringVarP(&destination, "destination", "d", "", "destination overlay (e.g. staging merges teploy.staging.yml)") cmd.Flags().StringVar(&appName, "app", "", "app name for ad-hoc deploy (bypasses teploy.yml)") @@ -852,6 +853,14 @@ func deployBuiltImageFenced(ctx context.Context, executor ssh.Executor, appCfg * } } + // backup_before_hooks: a pre_deploy hook is usually a migration, and a + // migration rewrites data with no way down. + if appCfg.BackupBeforeHooks && appCfg.Hooks.PreDeploy != "" && !deployNoBackup { + if err := backupBeforeDeployHook(ctx, rawExecutor, appCfg, out); err != nil { + return err + } + } + multiNotifier := buildNotifier(appCfg) var deployErr error if lk != nil { diff --git a/internal/cli/remove.go b/internal/cli/remove.go index 1736890..219ab7a 100644 --- a/internal/cli/remove.go +++ b/internal/cli/remove.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/spf13/cobra" + "github.com/useteploy/teploy/internal/backup" "github.com/useteploy/teploy/internal/build" "github.com/useteploy/teploy/internal/caddy" "github.com/useteploy/teploy/internal/confighome" @@ -23,7 +24,7 @@ import ( func newRemoveCmd(flags *Flags) *cobra.Command { var appName string - var yes, purge bool + var yes, purge, noBackup bool var redirect string cmd := &cobra.Command{ Use: "remove", @@ -40,7 +41,9 @@ so the server history can still answer what happened to the app. Data is preserved by default: volumes/, accessory data, and running accessory containers are left in place so a removal is never silently destructive. -Pass --purge to delete those too (irreversible). +Pass --purge to delete those too (irreversible). A fresh backup of every +accessory and of the volumes is stored first; if that fails, or the server has +no backup target, the purge is refused unless --no-backup. Pass --redirect to leave a permanent redirect behind for the app's domains. The redirect is written as a plain, unmanaged Caddy block (no TEPLOY @@ -49,13 +52,14 @@ markers), so later teploy operations never touch it. Every step is idempotent — re-running after a partial failure is safe.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - return runRemove(flags, appName, yes, purge, redirect) + return runRemove(flags, appName, yes, purge, noBackup, redirect) }, } cmd.Flags().StringVar(&appName, "app", "", "app name — act on server state instead of teploy.yml (requires --host)") cmd.Flags().BoolVarP(&yes, "yes", "y", false, "skip the confirmation prompt") cmd.Flags().BoolVar(&purge, "purge", false, "also delete volumes, accessory data, and accessory containers (irreversible)") cmd.Flags().StringVar(&redirect, "redirect", "", "leave a permanent redirect to this URL for the app's domains") + cmd.Flags().BoolVar(&noBackup, "no-backup", false, "with --purge: skip the automatic backup taken first") return cmd } @@ -76,7 +80,59 @@ type removeSummary struct { KeptConfig bool `json:"kept_config,omitempty"` } -func runRemove(flags *Flags, appName string, yes, purge bool, redirect string) error { +// purgeInventory names what a purge would destroy, for the prompt and the backup. +func purgeInventory(ctx context.Context, exec ssh.Executor, app string) (accessoriesNames []string, hasVolumes bool, err error) { + containers, err := docker.NewClient(exec).ListContainers(ctx, app) + if err != nil { + return nil, false, fmt.Errorf("listing containers: %w", err) + } + for _, c := range containers { + if c.Labels["teploy.role"] == "accessory" { + // Older accessories carry no teploy.accessory label: the name still + // says which, and one left out here would be destroyed unprotected. + name := c.Labels["teploy.accessory"] + if name == "" { + name = strings.TrimPrefix(c.Name, app+"-") + } + accessoriesNames = append(accessoriesNames, name) + } + } + slices.Sort(accessoriesNames) + out, err := exec.Run(ctx, "d="+ssh.ShellQuote("/deployments/"+app+"/volumes")+`; if [ -d "$d" ] && [ -n "$(ls -A "$d" 2>/dev/null)" ]; then echo yes; fi; true`) + if err != nil { + return nil, false, fmt.Errorf("checking volumes: %w", err) + } + return accessoriesNames, strings.TrimSpace(out) == "yes", nil +} + +// backupBeforePurge stores a copy of everything a purge is about to destroy. +func backupBeforePurge(ctx context.Context, exec ssh.Executor, app string, accs []string, volumes bool, out io.Writer) error { + if len(accs) == 0 && !volumes { + return nil + } + cfg, err := resolvePreBackupTarget(ctx, exec, "remove --purge") + if err != nil { + return err + } + client := backup.NewClient(exec, out) + for _, acc := range accs { + id, err := preOperationBackup(ctx, client, exec, app, acc, "", cfg, "remove --purge") + if err != nil { + return err + } + fmt.Fprintf(out, "Backed up %s first: %s\n", backup.ContainerName(app, acc), id) + } + if volumes { + id, err := preOperationBackup(ctx, client, exec, app, "", "", cfg, "remove --purge") + if err != nil { + return err + } + fmt.Fprintf(out, "Backed up %s's volumes first: %s\n", app, id) + } + return nil +} + +func runRemove(flags *Flags, appName string, yes, purge, noBackup bool, redirect string) error { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() @@ -95,6 +151,15 @@ func runRemove(flags *Flags, appName string, yes, purge bool, redirect string) e return fmt.Errorf("app %q has no domains recorded in its state — cannot write a redirect", appCfg.App) } + var purgeAccessories []string + var purgeVolumes bool + if purge { + purgeAccessories, purgeVolumes, err = purgeInventory(ctx, executor, appCfg.App) + if err != nil { + return err + } + } + if !yes { fmt.Printf("Remove %s from %s:\n", appCfg.App, executor.Host()) fmt.Println(" - stop and remove its containers") @@ -104,7 +169,18 @@ func runRemove(flags *Flags, appName string, yes, purge bool, redirect string) e fmt.Println(" - remove its Caddy route") } if purge { - fmt.Println(" - DELETE volumes, accessory data, and accessory containers (--purge)") + fmt.Println(" - DELETE, irreversibly (--purge):") + if purgeVolumes { + fmt.Printf(" volumes: /deployments/%s/volumes\n", appCfg.App) + } + for _, acc := range purgeAccessories { + fmt.Printf(" accessory: %s, with its data in /deployments/%s/accessories/%s\n", backup.ContainerName(appCfg.App, acc), appCfg.App, acc) + } + if noBackup { + fmt.Println(" NO backup is taken first (--no-backup)") + } else { + fmt.Println(" a fresh backup of each is stored first") + } } else { fmt.Println(" - delete deploy state (volumes and accessories are preserved)") } @@ -117,6 +193,12 @@ func runRemove(flags *Flags, appName string, yes, purge bool, redirect string) e if flags.JSON { out = io.Discard } + if purge && !noBackup { + if err := backupBeforePurge(ctx, executor, appCfg.App, purgeAccessories, purgeVolumes, backupProgressOut(flags)); err != nil { + recordAppEvent(ctx, executor, appCfg.App, events.KindRemove, err, map[string]any{"purge": purge, "redirect": redirect}) + return err + } + } summary, err := executeRemove(ctx, executor, appCfg.App, domains, removeOptions{Purge: purge, Redirect: redirect}, out) recordAppEvent(ctx, executor, appCfg.App, events.KindRemove, err, map[string]any{"purge": purge, "redirect": redirect}) if err != nil { @@ -200,7 +282,13 @@ func executeRemove(ctx context.Context, exec ssh.Executor, app string, domains [ return nil, err } } - if err := dk.Remove(ctx, c.Name); err != nil { + // A purge takes the container's anonymous volumes with it: nothing else + // can ever reach them, and leaving them is disk nobody can see. + remove := dk.Remove + if opts.Purge { + remove = dk.RemoveWithVolumes + } + if err := remove(ctx, c.Name); err != nil { return nil, err } summary.RemovedContainers = append(summary.RemovedContainers, c.Name) diff --git a/internal/config/app.go b/internal/config/app.go index 018c1c3..7073b59 100644 --- a/internal/config/app.go +++ b/internal/config/app.go @@ -690,11 +690,15 @@ type AppConfig struct { // everything — historical behavior. Set to 2 or 3 to enable auto-prune // with a rollback window. Container-deploy only; static deploy uses // KeepReleases instead. - KeepVersions int `yaml:"keep_versions,omitempty" toml:"keep_versions"` - Hooks HooksConfig `yaml:"hooks,omitempty" toml:"hooks"` - Volumes map[string]string `yaml:"volumes,omitempty" toml:"volumes"` - VolumeOwnership map[string]VolumeOwnership `yaml:"volume_ownership,omitempty" toml:"volume_ownership"` - Processes map[string]string `yaml:"processes,omitempty" toml:"processes"` + KeepVersions int `yaml:"keep_versions,omitempty" toml:"keep_versions"` + Hooks HooksConfig `yaml:"hooks,omitempty" toml:"hooks"` + // BackupBeforeHooks backs every database accessory up before a deploy whose + // pre_deploy hook is about to run (migrations rewrite data with no way down). + // A failed backup aborts the deploy; `--no-backup` overrides. Default off. + BackupBeforeHooks bool `yaml:"backup_before_hooks,omitempty" toml:"backup_before_hooks"` + Volumes map[string]string `yaml:"volumes,omitempty" toml:"volumes"` + VolumeOwnership map[string]VolumeOwnership `yaml:"volume_ownership,omitempty" toml:"volume_ownership"` + Processes map[string]string `yaml:"processes,omitempty" toml:"processes"` // Env are plain environment variables passed to the app container. Values // may reference ${VAR} from the local environment (expanded at deploy // time). Secrets set via `teploy secret` override these key-for-key. @@ -1800,6 +1804,9 @@ func mergeConfigs(base, overlay *AppConfig) { if len(overlay.Publish) > 0 { base.Publish = overlay.Publish } + if overlay.BackupBeforeHooks { + base.BackupBeforeHooks = true + } if overlay.Hooks.PreDeploy != "" { base.Hooks.PreDeploy = overlay.Hooks.PreDeploy } diff --git a/internal/docker/docker.go b/internal/docker/docker.go index 01ba203..e9f128b 100644 --- a/internal/docker/docker.go +++ b/internal/docker/docker.go @@ -685,6 +685,14 @@ func (c *Client) Remove(ctx context.Context, name string) error { return nil } +// RemoveWithVolumes removes a container and its anonymous volumes. +func (c *Client) RemoveWithVolumes(ctx context.Context, name string) error { + if _, err := c.exec.Run(ctx, "docker rm -v "+ssh.ShellQuote(name)); err != nil { + return fmt.Errorf("removing container %s: %w", name, err) + } + return nil +} + // HostPort returns the container's host-mapped port — the port a health // check on this machine connects to at http://localhost:, as opposed // to InternalPort (the container-internal port Caddy dials over the From 5ec63bc7ed0f4a5885b6ecd54f321ecdaaeac7c8 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Wed, 4 Nov 2026 22:34:36 -0700 Subject: [PATCH 05/17] feat(doctor,setup): backup-policy check; setup installs aws and age doctor flags a policy with no target, no cron line, no recipient, stale runs, no heartbeat, a missing aws or age, a target that does not answer, leftover workspaces and uploads, and databases nothing backs up. setup installs the aws CLI, through AWS's bundle where the distribution has none (Ubuntu 24.04), with modes every user can execute; server backup-target set names the exact remedy. Co-Authored-By: Claude Opus 5.5 --- internal/cli/doctor.go | 86 +++++++++++++++++++++++++++++- internal/cli/doctor_backup_test.go | 68 +++++++++++++++++++++++ internal/cli/doctor_test.go | 31 ++++++++--- internal/cli/setup.go | 25 +++++++++ internal/cli/setup_test.go | 16 +++++- 5 files changed, 216 insertions(+), 10 deletions(-) create mode 100644 internal/cli/doctor_backup_test.go diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index db3bdfc..5770692 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -22,6 +22,7 @@ import ( "slices" "strconv" "strings" + "time" "github.com/spf13/cobra" "github.com/useteploy/teploy/internal/backup" @@ -203,7 +204,7 @@ func doctorRun(ctx context.Context, deps doctorDeps, flags *Flags, serverName st if executor == nil { const sshDown = "restore SSH connectivity (see the ssh check above), then re-run teploy doctor" - for _, name := range []string{"docker", "disk", "registry", "caddy", "compatibility", "repair-debt", "secret-exposure", "server-tools", "config-home", "backup-cron", "age-identity", "git-trigger", "scheduler", "restore-leftovers", "backup-coverage"} { + for _, name := range []string{"docker", "disk", "registry", "caddy", "compatibility", "repair-debt", "secret-exposure", "server-tools", "config-home", "backup-cron", "age-identity", "git-trigger", "scheduler", "restore-leftovers", "backup-coverage", "backup-policy"} { skipFail(name, "SSH unreachable", sshDown) } } else { @@ -229,6 +230,7 @@ func doctorRun(ctx context.Context, deps doctorDeps, flags *Flags, serverName st pgReports, pgErr := backup.NewClient(executor, io.Discard).AccessoryReports(ctx) report.Checks = append(report.Checks, doctorRestoreLeftoversCheck(ctx, executor, pgReports)) report.Checks = append(report.Checks, doctorBackupCoverageCheck(pgReports, pgErr)) + report.Checks = append(report.Checks, doctorBackupPolicyCheck(ctx, executor, time.Now().UTC())) } report.summarize() @@ -972,11 +974,91 @@ func doctorBackupCoverageCheck(pg []backup.AccessoryReport, err error) doctorChe if len(gaps) > 0 { return doctorCheck{Name: "backup-coverage", Result: doctorWarn, Detail: "databases not covered by the accessory's backups: " + strings.Join(gaps, "; "), - Remediation: "postgres: list them in the accessory's backup_databases: in teploy.yml (\"all\" for every one) and recreate the accessory. mysql/mariadb: back each schema up as its own accessory until G5"} + Remediation: "postgres: list them in the accessory's backup_databases: in teploy.yml (\"all\" for every one) and recreate the accessory. mysql/mariadb: the same list, or \"all\". A server backup policy (teploy server backup-policy set) covers every database on its own"} } return doctorCheck{Name: "backup-coverage", Result: doctorOK, Detail: fmt.Sprintf("%d database accessory(ies): every database is covered", len(pg))} } +// doctorBackupPolicyCheck reports what would make a server's backups fail or go +// quiet: a policy with no target or no running job, no recovery key, stale runs, +// no aws CLI (three production servers had none), no age, a killed run's +// leftovers, and databases nothing backs up. It reads the same facts Dash does. +func doctorBackupPolicyCheck(ctx context.Context, exec ssh.Executor, now time.Time) doctorCheck { + sctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + st, err := backup.ReadFleetStatus(sctx, exec, now) + if err != nil { + return doctorCheck{Name: "backup-policy", Result: doctorWarn, Detail: "could not read the backup status: " + err.Error()} + } + cronBased := st.Policy.Configured + for _, e := range st.Entries { + cronBased = cronBased || e.Schedule == "own" + } + var fails, warns, remedies []string + if (cronBased || st.Target.Configured) && !st.AWSCLI { + fails = append(fails, "the aws CLI is missing, so every backup here fails") + remedies = append(remedies, backup.AWSCLIRemedy) + } + if st.Policy.Configured { + if !st.Target.Configured { + fails = append(fails, "the backup policy has no target, so every run fails") + remedies = append(remedies, "teploy server backup-target set --host --bucket --prefix ") + } + if st.Policy.Interval != "" && !st.Policy.CronInstalled { + fails = append(fails, "the backup policy's cron line is not installed, so no run happens") + remedies = append(remedies, "teploy server backup-policy set --host --interval "+st.Policy.Interval) + } + if st.Policy.Recipient == "" { + warns = append(warns, "no recovery key is set, so backups are stored unencrypted and app .env files are left out of them") + remedies = append(remedies, "teploy backup keygen, then teploy server backup-policy set --host --recipient ") + } else if !st.AgeCLI { + fails = append(fails, "the policy encrypts backups but age is not installed, so every backup is refused") + remedies = append(remedies, "teploy setup, or on the server: apt-get install age") + } + switch { + case st.Policy.Interval == "": + case st.LastRun == nil: + warns = append(warns, "the policy has never run") + remedies = append(remedies, "wait for its cron line, or run teploy backup run --all on the server") + case st.LastRun.Stale: + fails = append(fails, fmt.Sprintf("the last backup run was at %s, past twice the %s interval: cron is not running it", st.LastRun.At.Format(time.RFC3339), st.Policy.Interval)) + remedies = append(remedies, "check the cron daemon and run teploy backup run --all on the server") + case st.LastRun.Status != backup.RunOK: + warns = append(warns, fmt.Sprintf("the last backup run ended %s (%d failed)", st.LastRun.Status, st.LastRun.TargetsFailed)) + remedies = append(remedies, "teploy backup status --host shows which target and why") + } + if st.Policy.HeartbeatCount == 0 { + warns = append(warns, "no heartbeat URL is set, so a job that stops running raises no alert") + remedies = append(remedies, "printf '\\n' | teploy server backup-policy set --host --heartbeat-stdin") + } + } + if st.Target.Configured && st.AWSCLI && !st.Target.Reachable { + fails = append(fails, "the backup target did not answer: "+st.Target.Error) + remedies = append(remedies, "check the bucket, endpoint and credentials (teploy server backup-target set)") + } + if len(st.LeftoverWorkspaces) > 0 { + warns = append(warns, "killed or kept backup operations left "+strings.Join(st.LeftoverWorkspaces, ", ")+" (dumps and possibly a credential file)") + remedies = append(remedies, "the next teploy backup run --all removes workspaces and identities; check a kept restore's files before removing them by hand, and docker rm -fv a verify scratch") + } + if len(st.LeftoverUploads) > 0 { + warns = append(warns, fmt.Sprintf("%d upload(s) a killed run never published are still in the bucket", len(st.LeftoverUploads))) + remedies = append(remedies, "the next teploy backup run --all removes them") + } + if st.Policy.Configured && len(st.UncoveredDatabases) > 0 { + warns = append(warns, "not backed up: "+strings.Join(st.UncoveredDatabases, ", ")) + remedies = append(remedies, "teploy server backup-policy set --host --adopt [:[:all]], or --exclude it") + } + switch { + case len(fails) > 0: + return doctorCheck{Name: "backup-policy", Result: doctorFail, Detail: strings.Join(append(fails, warns...), "; "), Remediation: strings.Join(remedies, "; ")} + case len(warns) > 0: + return doctorCheck{Name: "backup-policy", Result: doctorWarn, Detail: strings.Join(warns, "; "), Remediation: strings.Join(remedies, "; ")} + case !st.Policy.Configured: + return doctorCheck{Name: "backup-policy", Result: doctorOK, Detail: "no backup policy on this server (teploy server backup-policy set covers every database with one cron line)"} + } + return doctorCheck{Name: "backup-policy", Result: doctorOK, Detail: fmt.Sprintf("%d target(s), %s, encrypted", len(st.Entries), st.LastRun.Status)} +} + // doctorAgeIdentityScript prints three facts about the age identity's // location: the new path (present | denied | absent), the legacy path // (real | marker | denied | absent) and whether this user could move it diff --git a/internal/cli/doctor_backup_test.go b/internal/cli/doctor_backup_test.go new file mode 100644 index 0000000..677e274 --- /dev/null +++ b/internal/cli/doctor_backup_test.go @@ -0,0 +1,68 @@ +package cli + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/ssh" +) + +func doctorPolicyMock(policy, target string, tools string, cron, events string) *ssh.MockExecutor { + return ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: policy}, + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: target}, + ssh.MockCommand{Match: "command -v aws >/dev/null", Output: tools}, + ssh.MockCommand{Match: "crontab -l", Output: cron}, + ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: ""}, + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "( c=$(python3", Output: ""}, + ssh.MockCommand{Match: "python3 -c", Output: events}, + ssh.MockCommand{Match: "for d in /deployments/*/volumes", Output: ""}, + ssh.MockCommand{Match: "for d in /var/tmp/teploy-backup.", Output: ""}, + ) +} + +func TestDoctorBackupPolicyCheck(t *testing.T) { + now := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) + policy := func(extra string) string { + return "path /etc/teploy/backup-policy.json\n" + `{"v":"1","interval":"1h"` + extra + `}` + } + target := "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"b","region":"r"}` + cron := "5 * * * * /usr/local/bin/teploy backup run --all # [teploy-backup-all]\n" + good := `{"v":1,"id":"1","seq":1,"ts":"2026-11-04T11:30:00Z","app":"_fleet","kind":"backup.all","outcome":"succeeded","detail":{"status":"ok","targets_ok":2,"targets_failed":0}}` + "\n" + stale := strings.ReplaceAll(good, "2026-11-04T11:30:00Z", "2026-11-01T11:30:00Z") + full := `,"recipient":"` + testRecipient + `","heartbeat_urls":["https://d/x"]` + for _, tc := range []struct { + name, policy, target, tools, cron, events string + result, want string + }{ + {"no policy, nothing configured", "absent\n", "absent\n", "age\n", "", "", doctorOK, "no backup policy"}, + {"all good", policy(full), target, "aws\nage\n", cron, good, doctorOK, "encrypted"}, + {"no target", policy(full), "absent\n", "aws\nage\n", cron, good, doctorFail, "no target"}, + {"no aws", policy(full), target, "age\n", cron, good, doctorFail, "aws CLI is missing"}, + {"no recipient", policy(`,"heartbeat_urls":["https://d/x"]`), target, "aws\nage\n", cron, good, doctorWarn, "unencrypted"}, + {"recipient but no age", policy(full), target, "aws\n", cron, good, doctorFail, "age is not installed"}, + {"no cron line", policy(full), target, "aws\nage\n", "", good, doctorFail, "cron line is not installed"}, + {"stale", policy(full), target, "aws\nage\n", cron, stale, doctorFail, "past twice"}, + {"never ran", policy(full), target, "aws\nage\n", cron, "", doctorWarn, "never run"}, + {"no heartbeat", policy(`,"recipient":"` + testRecipient + `"`), target, "aws\nage\n", cron, good, doctorWarn, "no heartbeat"}, + {"target only, no aws", "absent\n", target, "", "", "", doctorFail, "aws CLI is missing"}, + } { + t.Run(tc.name, func(t *testing.T) { + c := doctorBackupPolicyCheck(context.Background(), doctorPolicyMock(tc.policy, tc.target, tc.tools, tc.cron, tc.events), now) + if c.Result != tc.result || !strings.Contains(c.Detail, tc.want) { + t.Fatalf("%+v", c) + } + if c.Result != doctorOK && c.Remediation == "" { + t.Fatalf("no remedy: %+v", c) + } + }) + } + // The aws remedy is exact. + c := doctorBackupPolicyCheck(context.Background(), doctorPolicyMock(policy(full), target, "age\n", cron, good), now) + if !strings.Contains(c.Remediation, "awscli") || !strings.Contains(c.Remediation, "teploy setup") { + t.Fatalf("%s", c.Remediation) + } +} diff --git a/internal/cli/doctor_test.go b/internal/cli/doctor_test.go index b1783f7..93a5ba8 100644 --- a/internal/cli/doctor_test.go +++ b/internal/cli/doctor_test.go @@ -21,7 +21,7 @@ import ( // against (adding checks is additive at the end; renaming/removing is a // machine-interface bump). var doctorCheckOrder = []string{ - "git", "config", "ssh", "docker", "disk", "registry", "caddy", "compatibility", "repair-debt", "secret-exposure", "server-tools", "config-home", "backup-cron", "age-identity", "git-trigger", "scheduler", "restore-leftovers", "backup-coverage"} + "git", "config", "ssh", "docker", "disk", "registry", "caddy", "compatibility", "repair-debt", "secret-exposure", "server-tools", "config-home", "backup-cron", "age-identity", "git-trigger", "scheduler", "restore-leftovers", "backup-coverage", "backup-policy"} // doctorHappyMock registers every remote read a fully healthy target // answers, for the app config returned by doctorTestApp. @@ -41,6 +41,13 @@ func doctorHappyMock() *ssh.MockExecutor { ssh.MockCommand{Match: "sched_daemon=no", Output: "daemon=yes\nenabled=enabled\ncrontab=yes\nteploy_entries=0\n"}, ssh.MockCommand{Match: "for d in /var/tmp/teploy-pre-restore.*", Output: ""}, ssh.MockCommand{Match: "docker ps --filter label=teploy.role=accessory", Output: ""}, + ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "absent\n"}, + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "absent\n"}, + ssh.MockCommand{Match: "command -v aws >/dev/null", Output: "aws\nage\n"}, + ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: ""}, + ssh.MockCommand{Match: "for d in /deployments/*/volumes", Output: ""}, + ssh.MockCommand{Match: "for d in /var/tmp/teploy-backup", Output: ""}, + ssh.MockCommand{Match: "python3 -c", Output: ""}, ) } @@ -89,8 +96,8 @@ func TestDoctorAllChecksPass(t *testing.T) { if ex == nil { t.Fatal("expected an executor back") } - if report.Summary != (doctorSummary{OK: 18, Warn: 0, Fail: 0}) { - t.Fatalf("summary = %+v, want 18 ok", report.Summary) + if report.Summary != (doctorSummary{OK: 19, Warn: 0, Fail: 0}) { + t.Fatalf("summary = %+v, want 19 ok", report.Summary) } if len(report.Checks) != len(doctorCheckOrder) { t.Fatalf("got %d checks, want %d", len(report.Checks), len(doctorCheckOrder)) @@ -137,7 +144,7 @@ func TestDoctorJSONStableShape(t *testing.T) { t.Fatalf("machine_interface = %v, want %d", doc["machine_interface"], MachineInterface) } checks, ok := doc["checks"].([]any) - if !ok || len(checks) != 18 { + if !ok || len(checks) != 19 { t.Fatalf("checks = %#v, want 18 entries", doc["checks"]) } for i, raw := range checks { @@ -198,6 +205,12 @@ func TestDoctorHumanTable(t *testing.T) { ssh.MockCommand{Match: "sched_daemon=no", Output: "daemon=yes\nenabled=enabled\ncrontab=yes\nteploy_entries=0\n"}, ssh.MockCommand{Match: "for d in /var/tmp/teploy-pre-restore.*", Output: ""}, ssh.MockCommand{Match: "docker ps --filter label=teploy.role=accessory", Output: ""}, + ssh.MockCommand{Match: "for p in /etc/teploy/backup-", Output: "absent\n"}, + ssh.MockCommand{Match: "command -v aws >/dev/null", Output: "aws\nage\n"}, + ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: ""}, + ssh.MockCommand{Match: "for d in /deployments/*/volumes", Output: ""}, + ssh.MockCommand{Match: "for d in /var/tmp/teploy-backup", Output: ""}, + ssh.MockCommand{Match: "python3 -c", Output: ""}, ) report, _ := doctorRun(context.Background(), doctorTestDeps(mock), &Flags{}, "", doctorTestApp(), nil) var out bytes.Buffer @@ -211,7 +224,7 @@ func TestDoctorHumanTable(t *testing.T) { if !strings.Contains(rendered, "fix:") { t.Fatalf("table missing the remediation line:\n%s", rendered) } - if !strings.Contains(rendered, "Summary: 17 ok, 0 warn, 1 fail") { + if !strings.Contains(rendered, "Summary: 18 ok, 0 warn, 1 fail") { t.Fatalf("table summary line wrong:\n%s", rendered) } assertDoctorReadOnlyCalls(t, mock.Calls) @@ -684,7 +697,7 @@ func TestDoctorEndToEndAllOK(t *testing.T) { if err := runDoctor(doctorTestDeps(mock), &Flags{}, "", &out); err != nil { t.Fatalf("runDoctor: %v", err) } - if !strings.Contains(out.String(), "Summary: 18 ok, 0 warn, 0 fail") { + if !strings.Contains(out.String(), "Summary: 19 ok, 0 warn, 0 fail") { t.Fatalf("human report wrong:\n%s", out.String()) } assertDoctorReadOnlyCalls(t, mock.Calls) @@ -712,6 +725,12 @@ func assertDoctorReadOnlyCalls(t *testing.T, calls []string) { "sched_daemon=no; for f in /proc/", "for d in /var/tmp/teploy-pre-restore.*", "docker ps --filter label=teploy.role=accessory", + "for p in /etc/teploy/backup-", + "command -v aws >/dev/null", + "ids=$(docker ps -aq", + "for d in /deployments/*/volumes", + "for d in /var/tmp/teploy-backup", + "python3 -c", } for _, call := range calls { allowed := false diff --git a/internal/cli/setup.go b/internal/cli/setup.go index 3a7c926..fd02083 100644 --- a/internal/cli/setup.go +++ b/internal/cli/setup.go @@ -695,6 +695,31 @@ func setupStages() []setupStage { return nil }, }, + { + name: "backup tools", + affects: "aws CLI (apt, else AWS's own bundle) and age (apt, if missing) — the backup transport and payload encryption; a failure here warns, since nothing but backups needs them", + run: func(ctx context.Context, exec ssh.Executor, w io.Writer, env *setupEnv) error { + fmt.Fprintln(w, "Checking aws CLI and age...") + if _, err := exec.Run(ctx, "command -v aws"); err == nil { + fmt.Fprintln(w, " aws CLI already installed") + } else { + fmt.Fprintln(w, " Installing the aws CLI...") + if out, err := exec.Run(ctx, env.sudo+"sh -c "+ssh.ShellQuote(backup.AWSCLIInstallScript)); err != nil { + // Three production servers had none and every backup failed + // at run time: say what backups will do and how to fix it. + fmt.Fprintf(w, " Warning: installing the aws CLI failed (%v %s) — backups will fail until it is present: %s\n", err, strings.TrimSpace(out), backup.AWSCLIRemedy) + } else { + fmt.Fprintln(w, " aws CLI installed") + } + } + if err := secret.NewManager(exec).EnsureAge(ctx); err != nil { + fmt.Fprintf(w, " Warning: installing age failed (%v) — encrypted backups will be refused until it is present\n", err) + } else { + fmt.Fprintln(w, " age present") + } + return nil + }, + }, { name: "age identity location", affects: "/etc/teploy (created 0700) — the age identity moves there from /deployments/.age-key when one exists; the move is verified by decrypting one secret before the old file is replaced by a public-key marker", diff --git a/internal/cli/setup_test.go b/internal/cli/setup_test.go index 6e6a686..5653fe5 100644 --- a/internal/cli/setup_test.go +++ b/internal/cli/setup_test.go @@ -721,6 +721,8 @@ func TestSetupServer_InterruptedRunIsResumable(t *testing.T) { // rsync, the teploy network, the directories, the Caddyfile, and a // running modern caddy — everything the interrupted run got to. phase2 := ssh.NewMockExecutor("1.2.3.4", + ssh.MockCommand{Match: "command -v aws", Output: "/usr/bin/aws"}, + ssh.MockCommand{Match: "which age", Output: "/usr/bin/age"}, ssh.MockCommand{Match: "whoami", Output: "root"}, ssh.MockCommand{Match: "docker --version", Output: "Docker version 24.0.0"}, ssh.MockCommand{Match: "rsync --version", Output: "rsync version 3.2.7"}, @@ -783,6 +785,8 @@ func TestSetupServer_ConnectionLossRecoversMidStage(t *testing.T) { ssh.MockCommand{Match: "docker --version", Output: "Docker version 24.0.0"}, ssh.MockCommand{Match: "rsync --version", Output: "rsync version 3.2.7"}, ssh.MockCommand{Match: "for t in git python3", Output: ""}, + ssh.MockCommand{Match: "command -v aws", Output: "/usr/bin/aws"}, + ssh.MockCommand{Match: "which age", Output: "/usr/bin/age"}, ssh.MockCommand{Match: "sched_daemon=no", Output: "daemon=yes\nenabled=enabled\ncrontab=yes\nteploy_entries=0\n"}, ssh.MockCommand{Match: "ufw status", Err: fmt.Errorf("exit status 1: ufw: command not found")}, ssh.MockCommand{Match: "systemctl is-active firewalld", Err: fmt.Errorf("exit status 3: inactive")}, @@ -886,9 +890,13 @@ func TestSetupCronDaemonStage(t *testing.T) { } t.Run("stopped and disabled gets enabled", func(t *testing.T) { mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "command -v aws", Output: "/usr/bin/aws"}, + ssh.MockCommand{Match: "which age", Output: "/usr/bin/age"}, ssh.MockCommand{Match: "sched_daemon=no", Output: "daemon=no\nenabled=disabled\ncrontab=yes\nteploy_entries=1\n", Once: true}, ssh.MockCommand{Match: "command -v systemctl", Output: "/usr/bin/systemctl"}, ssh.MockCommand{Match: "sudo sh -c 'systemctl enable --now cron", Output: ""}, + ssh.MockCommand{Match: "command -v aws", Output: "/usr/bin/aws"}, + ssh.MockCommand{Match: "which age", Output: "/usr/bin/age"}, ssh.MockCommand{Match: "sched_daemon=no", Output: "daemon=yes\nenabled=enabled\ncrontab=yes\nteploy_entries=1\n"}, ) var buf bytes.Buffer @@ -900,7 +908,9 @@ func TestSetupCronDaemonStage(t *testing.T) { } }) t.Run("running is left alone", func(t *testing.T) { - mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "sched_daemon=no", Output: "daemon=yes\nenabled=enabled\ncrontab=yes\nteploy_entries=0\n"}) + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "command -v aws", Output: "/usr/bin/aws"}, + ssh.MockCommand{Match: "which age", Output: "/usr/bin/age"}, + ssh.MockCommand{Match: "sched_daemon=no", Output: "daemon=yes\nenabled=enabled\ncrontab=yes\nteploy_entries=0\n"}) var buf bytes.Buffer if err := stage.run(context.Background(), mock, &buf, &setupEnv{}); err != nil { t.Fatal(err) @@ -910,7 +920,9 @@ func TestSetupCronDaemonStage(t *testing.T) { } }) t.Run("masked is reported, not unmasked", func(t *testing.T) { - mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "sched_daemon=no", Output: "daemon=no\nenabled=masked\ncrontab=yes\nteploy_entries=0\n"}) + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "command -v aws", Output: "/usr/bin/aws"}, + ssh.MockCommand{Match: "which age", Output: "/usr/bin/age"}, + ssh.MockCommand{Match: "sched_daemon=no", Output: "daemon=no\nenabled=masked\ncrontab=yes\nteploy_entries=0\n"}) var buf bytes.Buffer if err := stage.run(context.Background(), mock, &buf, &setupEnv{}); err != nil { t.Fatal(err) From 3ba14c2a8d2024fb3dc4f5f410be8d566ddf7493 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Wed, 4 Nov 2026 22:44:13 -0700 Subject: [PATCH 06/17] test(backup): fleet engine end to end against real engines, MinIO and the aws CLI TestFleetIntegration runs the real linux binary on the colima fixture: mixed engines with an adopted multi-schema mysql and hostile names, a database appearing between runs, one schema restored alone, server-identity verify and a corrupt object, a legacy object after encryption, recovery with only the owner key, kills mid-upload and before publishing, a stuck lock holder, overlap, partial failure, heartbeats, exact GFS sets in a real bucket, prefix moves, shared namespaces, status, destructive operations refusing and backing up, a wrapped DR bundle, doctor. The engine suite's fake aws gains head-object, stdin/stdout copies and recursive listing, and its executor routes RunStream through the fake too. Co-Authored-By: Claude Opus 5.5 --- internal/backup/engines_integration_test.go | 27 +- internal/backup/postgres_integration_test.go | 2 +- internal/cli/fleet_integration_test.go | 1181 ++++++++++++++++++ 3 files changed, 1207 insertions(+), 3 deletions(-) create mode 100644 internal/cli/fleet_integration_test.go diff --git a/internal/backup/engines_integration_test.go b/internal/backup/engines_integration_test.go index 480dbbc..408abcf 100644 --- a/internal/backup/engines_integration_test.go +++ b/internal/backup/engines_integration_test.go @@ -36,14 +36,24 @@ import ( const fakeAWS = `#!/bin/sh set -eu root=__ROOT__ +if [ "$1" = s3api ]; then + # head-object --bucket B --key K [--region R] [--endpoint-url E] + [ "$2" = head-object ] || exit 1 + shift 2; b=""; k="" + while [ $# -gt 0 ]; do case "$1" in --bucket) b=$2; shift 2;; --key) k=$2; shift 2;; *) shift;; esac; done + f="$root/$b/$k"; [ -f "$f" ] || { echo "An error occurred (404) when calling the HeadObject operation" >&2; exit 254; } + printf '{"ContentLength":%s,"Metadata":{"sha256":"%s"}}\n' "$(wc -c < "$f" | tr -d ' ')" "$(sha256sum "$f" | cut -d' ' -f1)" + exit 0 +fi [ "$1" = s3 ] || { echo "fake aws: only s3 is supported" >&2; exit 2; } op=$2; shift 2 args="" recursive=no while [ $# -gt 0 ]; do case "$1" in - --region|--endpoint-url) shift 2 ;; + --region|--endpoint-url|--metadata) shift 2 ;; --recursive) recursive=yes; shift ;; + --only-show-errors) shift ;; *) args="$args $1"; shift ;; esac done @@ -52,13 +62,22 @@ map() { case "$1" in s3://*) printf '%s/%s' "$root" "${1#s3://}" ;; *) printf '% case "$op" in cp) src=$(map "$1"); dst=$(map "$2") + if [ "$1" = - ]; then mkdir -p "$(dirname "$dst")"; cat > "$dst"; exit 0; fi [ -f "$src" ] || { echo "fatal error: An error occurred (404) when calling the HeadObject operation: Key \"$1\" does not exist" >&2; exit 1; } + if [ "$2" = - ]; then cat "$src"; exit 0; fi mkdir -p "$(dirname "$dst")" cp "$src" "$dst" ;; rm) rm -f "$(map "$1")" ;; ls) p=$(map "$1") + if [ "$recursive" = yes ]; then + [ -d "${p%/}" ] || exit 1 + (cd "$root/${1#s3://}" 2>/dev/null && find . -type f | sed 's|^\./||') | while read -r f; do + printf '2026-11-03 00:00:00 %s %s\n' "$(wc -c < "${p%/}/$f" | tr -d ' ')" "${1#s3://}$f" + done | sed "s| itbucket/| |" + exit 0 + fi case "$p" in */) dir=${p%/}; pre="" ;; *) dir=$(dirname "$p"); pre=$(basename "$p") ;; @@ -96,6 +115,10 @@ func (p pathExecutor) Run(ctx context.Context, cmd string) (string, error) { return p.Executor.Run(ctx, p.wrap(cmd)) } +func (p pathExecutor) RunStream(ctx context.Context, cmd string, stdout, stderr io.Writer) error { + return p.Executor.RunStream(ctx, p.wrap(cmd), stdout, stderr) +} + func (p pathExecutor) RunInput(ctx context.Context, cmd string, stdin io.Reader) error { return p.Executor.RunInput(ctx, p.wrap(cmd), stdin) } @@ -129,7 +152,7 @@ func newITFixture(t *testing.T, ctx context.Context) itFixture { t.Fatalf("creating fixture workspace: %v", err) } t.Cleanup(func() { - base.Run(context.Background(), "sudo -n rm -rf "+ssh.ShellQuote(work)+" /tmp/teploy-restore.* /var/tmp/teploy-pre-restore.teployit* || rm -rf "+ssh.ShellQuote(work)) + base.Run(context.Background(), "sudo -n rm -rf "+ssh.ShellQuote(work)+" /var/tmp/teploy-restore.* /var/tmp/teploy-pre-restore.teployit* || rm -rf "+ssh.ShellQuote(work)) }) if err := base.Upload(ctx, strings.NewReader(strings.ReplaceAll(fakeAWS, "__ROOT__", work+"/s3")), work+"/bin/aws", "0755"); err != nil { t.Fatalf("installing fake aws: %v", err) diff --git a/internal/backup/postgres_integration_test.go b/internal/backup/postgres_integration_test.go index 49d529c..f646379 100644 --- a/internal/backup/postgres_integration_test.go +++ b/internal/backup/postgres_integration_test.go @@ -29,7 +29,7 @@ func (f itFixture) dropKeptTmp(err error) { } if _, after, ok := strings.Cut(err.Error(), "restore files kept in "); ok { dir := strings.Fields(after)[0] - if strings.HasPrefix(dir, "/tmp/teploy-restore.") { + if strings.HasPrefix(dir, "/var/tmp/teploy-restore.") { f.base.Run(context.Background(), "rm -rf "+ssh.ShellQuote(dir)) } } diff --git a/internal/cli/fleet_integration_test.go b/internal/cli/fleet_integration_test.go new file mode 100644 index 0000000..a254eca --- /dev/null +++ b/internal/cli/fleet_integration_test.go @@ -0,0 +1,1181 @@ +//go:build integration + +// End-to-end verification of the fleet backup engine (Slice 4): the real linux +// binary on the colima SSH+Docker fixture, real engines, a real S3-compatible +// store (MinIO, driven by the real aws CLI, never the fake one), real age. The +// VM needs aws, age and a running cron, and a user-owned /deployments. MinIO is +// cgr.dev/chainguard/minio: the official images are no longer publicly pullable. +// +// TEPLOY_FAULT_HOST=127.0.0.1: TEPLOY_FAULT_USER=tyler \ +// TEPLOY_FAULT_KEY=~/.colima/_lima/_config/user \ +// go test -tags integration -p 2 -run TestFleetIntegration -v -timeout 40m ./internal/cli +// +// Containers are capped and removed on exit. +package cli + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/backup" + "github.com/useteploy/teploy/internal/config" + "github.com/useteploy/teploy/internal/ssh" +) + +const ( + itDir = "/tmp/teploy-fleet-it" + itBin = itDir + "/teploy" + itShims = itDir + "/shims" + itMinioPort = "19100" + itHookPort = "19200" + itEndpoint = "http://127.0.0.1:" + itMinioPort + itAK = "teployitaccess" + itSK = "teployitsecret1234567890" + itDBPW = "fixture-db-password-XYZ" + itToken = "SECRETHEARTBEATTOKEN" +) + +type fleetIT struct { + t *testing.T + exec ssh.Executor + ctx context.Context + s3 backup.S3Config + owner string // owner identity file on the VM + pub string // owner recipient + local string // darwin/linux binary for the --host commands + host string + user string + key string +} + +func (f *fleetIT) sh(cmd string) string { + f.t.Helper() + out, err := f.exec.Run(f.ctx, cmd) + if err != nil { + f.t.Fatalf("command failed: %s\nerror: %v\noutput: %s", cmd, err, out) + } + return out +} + +// awsIT runs the real aws CLI with the fixture's credentials. +func (f *fleetIT) awsIT(args string) string { + f.t.Helper() + return f.sh("AWS_ACCESS_KEY_ID=" + itAK + " AWS_SECRET_ACCESS_KEY=" + itSK + " aws --region us-east-1 --endpoint-url " + itEndpoint + " " + args) +} + +// keys lists the bucket keys under prefix, sorted. +func (f *fleetIT) keys(prefix string) []string { + f.t.Helper() + res := ssh.RunDetailed(f.ctx, f.exec, "AWS_ACCESS_KEY_ID="+itAK+" AWS_SECRET_ACCESS_KEY="+itSK+" aws --region us-east-1 --endpoint-url "+itEndpoint+" s3 ls --recursive "+ssh.ShellQuote("s3://itfleet/"+prefix)) + var out []string + for _, line := range strings.Split(string(res.Stdout), "\n") { + if fs := strings.Fields(line); len(fs) >= 4 { + out = append(out, fs[3]) + } + } + sort.Strings(out) + return out +} + +// runAll runs `teploy backup run --all --json` on the VM. PATH may be shimmed. +func (f *fleetIT) runAll(env string) (backupRunEnvelope, int, string) { + f.t.Helper() + res := ssh.RunDetailedLimit(f.ctx, f.exec, env+" "+itBin+" backup run --all --json", 16<<20) + if res.Err != nil { + f.t.Fatalf("backup run --all: %v", res.Err) + } + var env2 backupRunEnvelope + if err := json.Unmarshal(lastJSONLine(res.Stdout), &env2); err != nil { + f.t.Fatalf("parsing the run envelope: %v\nstdout: %s\nstderr: %s", err, res.Stdout, res.Stderr) + } + return env2, res.ExitCode, string(res.Stdout) + string(res.Stderr) +} + +func lastJSONLine(b []byte) []byte { + lines := strings.Split(strings.TrimSpace(string(b)), "\n") + return []byte(lines[len(lines)-1]) +} + +func (f *fleetIT) target(r backupRunEnvelope, container string) backup.FleetTargetResult { + f.t.Helper() + for _, t := range r.Targets { + if t.Container == container || (container == t.App+"/volumes" && t.Kind == backup.TargetVolumes) { + return t + } + } + f.t.Fatalf("no target %q in %+v", container, r.Targets) + return backup.FleetTargetResult{} +} + +// cli runs the local binary against the VM (the --host commands). stdin optional. +func (f *fleetIT) cli(stdin string, args ...string) (string, error) { + f.t.Helper() + all := append([]string{"--host", f.host, "--user", f.user, "--key", f.key}, args...) + c := exec.CommandContext(f.ctx, f.local, all...) + c.Env = append(os.Environ(), "TEPLOY_SERVER_BINARY="+itBin) + if stdin != "" { + c.Stdin = strings.NewReader(stdin) + } + out, err := c.CombinedOutput() + return string(out), err +} + +func (f *fleetIT) setPolicy(p backup.FleetPolicy) { + f.t.Helper() + if _, _, err := backup.WriteFleetPolicy(f.ctx, f.exec, p); err != nil { + f.t.Fatalf("writing the policy: %v", err) + } +} + +func (f *fleetIT) psql(container, db, sql string) string { + f.t.Helper() + return strings.TrimSpace(f.sh("docker exec " + container + " psql -X -qtA -v ON_ERROR_STOP=1 -U appuser -d " + db + " -c " + ssh.ShellQuote(sql))) +} + +func (f *fleetIT) mysql(sql string) string { + f.t.Helper() + return strings.TrimSpace(f.sh("docker exec -e MYSQL_PWD=" + itDBPW + " wordpress-shared-mysql mysql -u root -N -B -e " + ssh.ShellQuote(sql))) +} + +func (f *fleetIT) waitFor(what, cmd string) { + f.t.Helper() + for i := 0; i < 120; i++ { + if _, err := f.exec.Run(f.ctx, cmd); err == nil { + return + } + time.Sleep(time.Second) + } + f.t.Fatalf("timed out waiting for %s", what) +} + +func TestFleetIntegration(t *testing.T) { + host, user, keyPath := os.Getenv("TEPLOY_FAULT_HOST"), os.Getenv("TEPLOY_FAULT_USER"), os.Getenv("TEPLOY_FAULT_KEY") + if host == "" || user == "" || keyPath == "" { + t.Skip("needs TEPLOY_FAULT_HOST, TEPLOY_FAULT_USER, TEPLOY_FAULT_KEY") + } + ctx, cancel := context.WithTimeout(context.Background(), 35*time.Minute) + defer cancel() + cctx, ccancel := context.WithTimeout(ctx, 10*time.Second) + defer ccancel() + vm, err := ssh.Connect(cctx, ssh.ConnectConfig{Host: host, User: user, KeyPath: keyPath}) + if err != nil { + t.Fatalf("connecting fixture: %v", err) + } + t.Cleanup(func() { vm.Close() }) + f := &fleetIT{t: t, exec: vm, ctx: ctx, host: host, user: user, key: keyPath} + + containers := []string{"teploy-it-minio", "teployit-db", "teployit-cache", "teployit2-db", "wordpress-shared-mysql", "omni_postgres", "teploy-it-hook", "teploy-it-scratch"} + cleanup := func() { + c := context.Background() + vm.Run(c, "docker rm -fv "+strings.Join(containers, " ")+" >/dev/null 2>&1; docker ps -aq --filter name=teploy-backup-verify- | xargs -r docker rm -fv >/dev/null 2>&1; pkill -f teploy-it-hook.py; true") + vm.Run(c, "rm -rf "+itDir+" /deployments/teployit /deployments/teployit2 /deployments/_adopted /deployments/.teploy/apps/teployit /deployments/.teploy/apps/teployit2 /deployments/.teploy/apps/_adopted /deployments/.teploy/apps/_fleet /var/tmp/teploy-backup.* /var/tmp/teploy-pre-restore.* /var/tmp/teploy-verify.* /var/tmp/teploy-restore.* /var/tmp/teploy-dr /tmp/teploy-pwned; crontab -r 2>/dev/null; rm -f \"$HOME\"/.config/teploy/backup-policy.json \"$HOME\"/.config/teploy/backup-target.json \"$HOME\"/.config/teploy/backup-run.lock; true") + } + cleanup() + t.Cleanup(cleanup) + + // Binaries: linux for the VM, this platform for the --host commands. + tmp := t.TempDir() + root, _ := filepath.Abs("../..") + build := func(goos, goarch, out string) { + c := exec.Command("go", "build", "-o", out, "./cmd/teploy") + c.Dir = root + c.Env = append(os.Environ(), "GOOS="+goos, "GOARCH="+goarch, "CGO_ENABLED=0") + if b, err := c.CombinedOutput(); err != nil { + t.Fatalf("building %s/%s: %v\n%s", goos, goarch, err, b) + } + } + arch := strings.TrimSpace(f.sh("uname -m")) + goarch := "arm64" + if arch == "x86_64" { + goarch = "amd64" + } + build("linux", goarch, filepath.Join(tmp, "teploy-linux")) + f.local = filepath.Join(tmp, "teploy-local") + build(runtime.GOOS, runtime.GOARCH, f.local) + f.sh("mkdir -p " + itShims) + body, err := os.ReadFile(filepath.Join(tmp, "teploy-linux")) + if err != nil { + t.Fatal(err) + } + if err := vm.Upload(ctx, strings.NewReader(string(body)), itBin, "0755"); err != nil { + t.Fatalf("uploading the binary: %v", err) + } + + // MinIO, the owner's recovery key, the stored target. + f.sh("docker run -d --name teploy-it-minio --memory 512m -p 127.0.0.1:" + itMinioPort + ":9000 -e MINIO_ROOT_USER=" + itAK + " -e MINIO_ROOT_PASSWORD=" + itSK + " cgr.dev/chainguard/minio:latest server /tmp/data >/dev/null") + f.waitFor("minio", "curl -fs "+itEndpoint+"/minio/health/live") + f.awsIT("s3 mb s3://itfleet") + keygen := func(file string) string { + c := exec.Command(f.local, "backup", "keygen") + b, err := c.Output() + if err != nil { + t.Fatalf("keygen: %v", err) + } + if err := vm.Upload(ctx, strings.NewReader(string(b)), file, "0600"); err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(string(b), "\n") { + if v, ok := strings.CutPrefix(line, "# public key: "); ok { + return v + } + } + t.Fatalf("keygen printed no public key: %s", b) + return "" + } + f.owner = itDir + "/owner.txt" + f.pub = keygen(f.owner) + armor, err := backup.EncryptCredentials(ctx, vm, itAK, itSK) + if err != nil { + t.Fatalf("encrypting credentials: %v", err) + } + f.s3 = backup.S3Config{Bucket: "itfleet", Region: "us-east-1", Endpoint: itEndpoint} + if _, _, err := backup.WriteTarget(ctx, vm, backup.Target{Bucket: "itfleet", Region: "us-east-1", Endpoint: itEndpoint}, armor); err != nil { + t.Fatalf("writing the target: %v", err) + } + target, err := backup.ReadTarget(ctx, vm) + if err != nil { + t.Fatal(err) + } + f.s3 = TargetS3(target) + + // A heartbeat receiver that records what it was asked for. + hook := `import http.server +class H(http.server.BaseHTTPRequestHandler): + def do_GET(self): + code = 500 if "broken" in self.path else 200 + open("` + itDir + `/hook.log","a").write(self.path+"\n") + self.send_response(code); self.end_headers() + def log_message(self,*a): pass +http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() +` + if err := vm.Upload(ctx, strings.NewReader(hook), itDir+"/teploy-it-hook.py", "0644"); err != nil { + t.Fatal(err) + } + f.sh("setsid nohup python3 " + itDir + "/teploy-it-hook.py >/dev/null 2>&1 < /dev/null & true") + f.waitFor("hook", "curl -fs http://127.0.0.1:"+itHookPort+"/ready") + + // The fleet: a labelled postgres, a labelled redis, an adopted multi-schema + // mysql holding hostile schema names, an unmanaged postgres nothing adopts, + // and one app's volumes with an .env. + f.sh("docker run -d --name teployit-db --memory 256m --label teploy.role=accessory --label teploy.app=teployit --label teploy.accessory=db -e POSTGRES_DB=appdb -e POSTGRES_USER=appuser -e POSTGRES_PASSWORD=" + itDBPW + " postgres:17-alpine >/dev/null") + f.waitFor("postgres", "docker exec teployit-db pg_isready -h 127.0.0.1 -U appuser") + f.psql("teployit-db", "appdb", "CREATE TABLE t(id int); INSERT INTO t VALUES (1),(2),(3)") + f.sh("docker exec teployit-db psql -X -q -U appuser -d postgres -c 'CREATE DATABASE reports'") + f.psql("teployit-db", "reports", "CREATE TABLE r(id int); INSERT INTO r VALUES (7)") + f.sh("docker run -d --name teployit-cache --memory 64m --label teploy.role=accessory --label teploy.app=teployit --label teploy.accessory=cache redis:7-alpine >/dev/null") + f.waitFor("redis", "docker exec teployit-cache redis-cli ping") + f.sh("docker exec teployit-cache redis-cli set k v >/dev/null") + f.sh("docker run -d --name wordpress-shared-mysql --memory 768m -e MYSQL_ROOT_PASSWORD=" + itDBPW + " mysql:8 >/dev/null") + f.waitFor("mysql", "docker exec -e MYSQL_PWD="+itDBPW+" wordpress-shared-mysql mysql --protocol=TCP -h 127.0.0.1 -u root -e 'SELECT 1'") + hostile := "wp_evil'; touch /tmp/teploy-pwned; '" + spaced := "wp space $(touch /tmp/teploy-pwned)" + for _, s := range []string{"wp_a", "wp_b", hostile, spaced} { + ident := "`" + s + "`" + f.mysql("CREATE DATABASE " + ident + "; CREATE TABLE " + ident + ".posts(id int); INSERT INTO " + ident + ".posts VALUES (1),(2)") + } + f.sh("mkdir -p /deployments/teployit/volumes/data && printf marker-v1 > /deployments/teployit/volumes/data/marker.txt && printf 'SECRET_ENV=plain-env-value\\n' > /deployments/teployit/.env") + + retention := backup.FleetRetention{KeepLast: 5} + base := backup.FleetPolicy{ + V: "1", Interval: "1h", Retention: retention, Volumes: true, + Adopted: []backup.AdoptedEntry{{Container: "wordpress-shared-mysql", Engine: "mysql", Databases: backup.DatabaseSelector{All: true}}}, + HeartbeatURLs: []string{"http://127.0.0.1:" + itHookPort + "/api/push/" + itToken}, + } + + var legacyID string + t.Run("no recovery key: plaintext, encrypted false, .env left out", func(t *testing.T) { + f.t = t + f.setPolicy(base) + res, code, out := f.runAll("") + if code != 0 || res.Status != backup.RunOK { + t.Fatalf("exit %d status %s: %+v\n%s", code, res.Status, res.Targets, out) + } + if res.Encrypted { + t.Fatal("encrypted must be false with no recipient") + } + db := f.target(res, "teployit-db") + legacyID = db.ID + if db.Encrypted || !strings.HasSuffix(db.Objects[len(db.Objects)-1].Key, ".pgdump") { + t.Fatalf("db object: %+v", db) + } + vol := f.target(res, "teployit/volumes") + if len(vol.Warnings) == 0 || !strings.Contains(vol.Warnings[0], ".env was left out") { + t.Fatalf("no .env warning: %+v", vol) + } + f.sh("mkdir -p " + itDir + "/v && AWS_ACCESS_KEY_ID=" + itAK + " AWS_SECRET_ACCESS_KEY=" + itSK + " aws --region us-east-1 --endpoint-url " + itEndpoint + " s3 cp " + vol.Objects[0].Key + " " + itDir + "/v/a.tar.gz --only-show-errors") + if strings.Contains(f.sh("tar -tzf "+itDir+"/v/a.tar.gz"), ".env") { + t.Fatal("an unencrypted volume archive contains .env") + } + }) + + var firstAll backupRunEnvelope + t.Run("mixed engines, adopted multi-schema, hostile names, everything encrypted", func(t *testing.T) { + f.t = t + p := base + p.Recipient = f.pub + f.setPolicy(p) + res, code, out := f.runAll("") + firstAll = res + if code != 0 || res.Status != backup.RunOK { + t.Fatalf("exit %d status %s: %+v\n%s", code, res.Status, res.Targets, out) + } + if !res.Encrypted { + t.Fatal("encrypted must be true") + } + for _, name := range []string{"teployit-db", "teployit-cache", "wordpress-shared-mysql", "teployit/volumes"} { + tr := f.target(res, name) + if tr.Status != backup.TargetOK || !tr.Encrypted || len(tr.Objects) == 0 { + t.Fatalf("%s: %+v", name, tr) + } + for _, o := range tr.Objects { + if !strings.HasSuffix(o.Key, ".age") { + t.Fatalf("%s stored plaintext: %s", name, o.Key) + } + } + } + // One object per schema, so one site restores alone. + mysql := f.target(res, "wordpress-shared-mysql") + dbs := map[string]bool{} + for _, o := range mysql.Objects { + dbs[o.Database] = true + } + for _, want := range []string{"wp_a", "wp_b", spaced} { + if !dbs[want] { + t.Fatalf("schema %q has no object: %+v", want, mysql.Objects) + } + } + // mysqldump cannot address a quoted name: never a silent success. + if dbs[hostile] || !strings.Contains(strings.Join(res.Omissions, "\n"), hostile) { + t.Fatalf("a schema mysqldump cannot address must be named as not backed up: %+v", res.Omissions) + } + // Every sidecar is encrypted too. + pgKeys := f.keys("teployit/accessories/db/") + for _, k := range pgKeys { + if strings.Contains(k, firstAll.Targets[0].ID) && !strings.HasSuffix(k, ".age") { + t.Fatalf("plaintext sidecar: %s", k) + } + } + for _, want := range []string{"/globals/", "/meta/", "/databases/reports/"} { + found := false + for _, k := range pgKeys { + found = found || (strings.Contains(k, want) && strings.HasSuffix(k, ".age")) + } + if !found { + t.Fatalf("no encrypted %s object: %v", want, pgKeys) + } + } + // Hostile names went through quoting, not through a shell. + if strings.TrimSpace(f.sh("if [ -e /tmp/teploy-pwned ]; then echo yes; fi")) != "" { + t.Fatal("a hostile schema name executed a command") + } + // omni_postgres does not exist yet, so it is not in the uncovered list. + for _, c := range res.UncoveredContainers { + if c == "omni_postgres" || c == "teployit-db" || c == "wordpress-shared-mysql" { + t.Fatalf("uncovered: %v", res.UncoveredContainers) + } + } + // Heartbeat: up, with no status=down. + log := f.sh("cat " + itDir + "/hook.log") + last := lastLine(log) + if !strings.Contains(last, "/api/push/"+itToken) || strings.Contains(last, "status=down") { + t.Fatalf("heartbeat: %q", log) + } + // No credential or token in anything a person reads. + for _, secretValue := range []string{itToken, itDBPW, itSK} { + if strings.Contains(out, secretValue) { + t.Fatalf("output leaks %q", secretValue) + } + } + events := f.sh("cat /deployments/.teploy/apps/_fleet/events.jsonl /deployments/.teploy/apps/_adopted/events.jsonl /deployments/.teploy/apps/teployit/events.jsonl") + for _, secretValue := range []string{itToken, itDBPW, itSK} { + if strings.Contains(events, secretValue) { + t.Fatalf("events leak %q", secretValue) + } + } + }) + + t.Run("new database appearing between runs is covered with no command", func(t *testing.T) { + f.t = t + f.sh("docker run -d --name teployit2-db --memory 128m --label teploy.role=accessory --label teploy.app=teployit2 --label teploy.accessory=db -e POSTGRES_DB=two -e POSTGRES_USER=appuser -e POSTGRES_PASSWORD=" + itDBPW + " postgres:17-alpine >/dev/null") + f.waitFor("second postgres", "docker exec teployit2-db pg_isready -h 127.0.0.1 -U appuser") + f.psql("teployit2-db", "two", "CREATE TABLE z(id int)") + f.mysql("CREATE DATABASE wp_new; CREATE TABLE wp_new.posts(id int)") + f.sh("docker exec teployit-db psql -X -q -U appuser -d postgres -c 'CREATE DATABASE fresh'") + f.psql("teployit-db", "fresh", "CREATE TABLE f(id int)") + res, code, out := f.runAll("") + if code != 0 { + t.Fatalf("exit %d: %s", code, out) + } + if tr := f.target(res, "teployit2-db"); tr.Status != backup.TargetOK { + t.Fatalf("new container: %+v", tr) + } + if len(f.keys("_adopted/accessories/wordpress-shared-mysql/databases/wp_new/")) == 0 { + t.Fatal("a new schema has no object") + } + if len(f.keys("teployit/accessories/db/databases/fresh/")) == 0 { + t.Fatal("a new database in a labelled accessory has no object") + } + // Containers are gone next run: not backed up, not an error for a labelled one. + f.sh("docker rm -fv teployit2-db >/dev/null") + }) + + t.Run("one schema restores alone; other schemas untouched; safety backup taken", func(t *testing.T) { + f.t = t + id := "" + for _, o := range f.target(firstAll, "wordpress-shared-mysql").Objects { + if o.Database == "wp_a" { + parts := strings.Split(o.Key, "/") + id = strings.Split(parts[len(parts)-1], ".")[0] + } + } + f.mysql("INSERT INTO wp_a.posts VALUES (99); INSERT INTO wp_b.posts VALUES (99)") + out, err := f.cli("", "backup", "restore", id, "--adopted", "--accessory", "wordpress-shared-mysql", "--database", "wp_a", "--yes", "--json") + if err != nil { + t.Fatalf("restore: %v\n%s", err, out) + } + var dto backupRestoreDTO + if err := json.Unmarshal(lastJSONLine([]byte(out)), &dto); err != nil { + t.Fatalf("restore JSON: %v\n%s", err, out) + } + if !dto.OK || dto.PreBackupID == "" { + t.Fatalf("restore: %+v", dto) + } + if got := f.mysql("SELECT COUNT(*) FROM wp_a.posts"); got != "2" { + t.Fatalf("wp_a rows = %s after restore", got) + } + if got := f.mysql("SELECT COUNT(*) FROM wp_b.posts"); got != "3" { + t.Fatalf("wp_b was touched: %s rows", got) + } + // The pre-restore safety backup holds the row the restore just removed. + if len(f.keys("_adopted/accessories/wordpress-shared-mysql/databases/wp_a/"+dto.PreBackupID)) == 0 { + t.Fatalf("no pre-restore backup %s", dto.PreBackupID) + } + // A hostile schema through the same path, quoting and all. + var hostileID string + for _, o := range f.target(firstAll, "wordpress-shared-mysql").Objects { + if o.Database == spaced { + parts := strings.Split(o.Key, "/") + hostileID = strings.Split(parts[len(parts)-1], ".")[0] + } + } + f.mysql("INSERT INTO `" + spaced + "`.posts VALUES (99)") + out, err = f.cli("", "backup", "restore", hostileID, "--adopted", "--accessory", "wordpress-shared-mysql", "--database", spaced, "--yes", "--no-backup", "--json") + if err != nil { + t.Fatalf("hostile restore: %v\n%s", err, out) + } + if got := f.mysql("SELECT COUNT(*) FROM `" + spaced + "`.posts"); got != "2" { + t.Fatalf("hostile schema rows = %s after restore", got) + } + if strings.TrimSpace(f.sh("if [ -e /tmp/teploy-pwned ]; then echo yes; fi")) != "" { + t.Fatal("a hostile schema name executed a command during restore") + } + }) + + t.Run("verify runs on the server with its own identity; corrupt object fails", func(t *testing.T) { + f.t = t + out, err := f.cli("", "--json", "accessory", "verify-backup", "wordpress-shared-mysql", "--adopted", "--database", "wp_a") + if err != nil { + t.Fatalf("verify: %v\n%s", err, out) + } + if !strings.Contains(out, `"ok": true`) || !strings.Contains(out, `"encrypted": true`) || !strings.Contains(out, "tables=1") { + t.Fatalf("verify result: %s", out) + } + // Corrupt the newest object of wp_b: verification must fail, loudly. + keys := f.keys("_adopted/accessories/wordpress-shared-mysql/databases/wp_b/") + var newest string + for _, k := range keys { + if strings.HasSuffix(k, ".age") && !strings.Contains(k, "/meta/") { + newest = k + } + } + f.sh("head -c 2000 /dev/urandom > " + itDir + "/garbage") + f.awsIT("s3 cp " + itDir + "/garbage s3://itfleet/" + newest + " --only-show-errors") + out, err = f.cli("", "--json", "accessory", "verify-backup", "wordpress-shared-mysql", "--adopted", "--database", "wp_b") + if err == nil || !strings.Contains(out, `"ok": false`) { + t.Fatalf("a corrupted object must fail verification: err=%v\n%s", err, out) + } + f.awsIT("s3 rm s3://itfleet/" + newest + " --only-show-errors") + }) + + t.Run("legacy unencrypted object restores after encryption is on", func(t *testing.T) { + f.t = t + f.psql("teployit-db", "appdb", "INSERT INTO t VALUES (100)") + out, err := f.cli("", "backup", "restore", legacyID, "--adopted=false", "--app", "teployit", "--accessory", "db", "--yes", "--json") + if err != nil { + t.Fatalf("restore: %v\n%s", err, out) + } + if got := f.psql("teployit-db", "appdb", "SELECT COUNT(*) FROM t"); got != "3" { + t.Fatalf("rows = %s", got) + } + }) + + t.Run("recovery on a clean host with only the owner identity", func(t *testing.T) { + f.t = t + var key string + for _, o := range f.target(firstAll, "teployit-db").Objects { + if o.Database == "" { + key = o.Key + } + } + clean := itDir + "/clean" + f.sh("mkdir -p " + clean) + // A wrong identity is refused and leaves nothing behind. + wrong := itDir + "/wrong.txt" + if b, err := exec.Command(f.local, "backup", "keygen").Output(); err != nil { + t.Fatal(err) + } else if err := vm.Upload(ctx, strings.NewReader(string(b)), wrong, "0600"); err != nil { + t.Fatal(err) + } + cleanEnv := "cd " + clean + " && env -i PATH=/usr/local/bin:/usr/bin:/bin HOME=" + clean + " TEPLOY_S3_ACCESS_KEY=" + itAK + " TEPLOY_S3_SECRET_KEY=" + itSK + " " + if res := ssh.RunDetailed(ctx, vm, cleanEnv+itBin+" backup fetch "+ssh.ShellQuote(key)+" --identity "+wrong+" --endpoint "+itEndpoint); res.ExitCode == 0 { + t.Fatalf("a wrong identity decrypted: %s", res.Stdout) + } + if left := strings.TrimSpace(f.sh("ls -A " + clean + " | grep -v '^.aws$'; true")); left != "" { + t.Fatalf("a failed fetch left files: %s", left) + } + // The owner's own key, no server identity, no policy, no target. + res := ssh.RunDetailed(ctx, vm, cleanEnv+itBin+" backup fetch "+ssh.ShellQuote(key)+" --identity "+f.owner+" --endpoint "+itEndpoint+" -o restored.pgdump") + if res.Failed() { + t.Fatalf("fetch: %s %s", res.Stdout, res.Stderr) + } + f.sh("docker run -d --name teploy-it-scratch --memory 256m -e POSTGRES_USER=appuser -e POSTGRES_PASSWORD=x -e POSTGRES_DB=appdb postgres:17-alpine >/dev/null") + f.waitFor("scratch", "docker exec teploy-it-scratch pg_isready -h 127.0.0.1 -U appuser") + f.sh("docker exec -i teploy-it-scratch pg_restore -U appuser -d appdb < " + clean + "/restored.pgdump") + if got := f.psql("teploy-it-scratch", "appdb", "SELECT COUNT(*) FROM t"); got != "3" { + t.Fatalf("recovered rows = %s", got) + } + f.sh("docker rm -fv teploy-it-scratch >/dev/null") + }) + + t.Run("killed mid-upload, then killed before publishing, then a good run", func(t *testing.T) { + f.t = t + realAWS := strings.TrimSpace(f.sh("command -v aws")) + // A shim: the first .incoming upload hangs, as a slow link would. + shim := "#!/bin/sh\ncase \"$*\" in *.incoming*) case \"$*\" in *--metadata*) touch " + itDir + "/uploading; sleep 600;; esac;; esac\nexec " + realAWS + " \"$@\"\n" + if err := vm.Upload(ctx, strings.NewReader(shim), itShims+"/aws", "0755"); err != nil { + t.Fatal(err) + } + before2 := f.keys("teployit/accessories/db/") + startKilled := func(marker string) { + f.sh("rm -f " + marker + "; PATH=" + itShims + ":$PATH setsid nohup " + itBin + " backup run --all > " + itDir + "/killed.log 2>&1 < /dev/null & echo $! > " + itDir + "/pid") + f.waitFor("the run to reach its upload", "test -e "+marker) + } + startKilled(itDir + "/uploading") + // Overlap: a second run stands down while the first holds the lock. + res, code, _ := f.runAll("") + if code != 0 || res.Status != "skipped" { + t.Fatalf("overlap: exit %d status %s", code, res.Status) + } + f.sh("kill -9 -- -$(cat " + itDir + "/pid); sleep 1; true") + // Nothing a later run could misread: no final object appeared, the lock + // died with the process, the workspace is a visible leftover. + if got := f.keys("teployit/accessories/db/"); strings.Join(got, ",") != strings.Join(before2, ",") { + t.Fatalf("a killed run changed the final objects:\nbefore %v\nafter %v", before2, got) + } + // A partial object left at the verification-pending key, as a provider + // with weaker semantics or a future bug might: never a backup. + partial := "teployit/accessories/db/.incoming/29991231-235959-0123456789abcdef.pgdump.age" + f.sh("printf partial > " + itDir + "/partial") + f.awsIT("s3 cp " + itDir + "/partial s3://itfleet/" + partial + " --only-show-errors") + // A truncated object at a FINAL key is also refused by the floor. + bad := "teployit/accessories/db/29991231-235959-fedcba9876543210.pgdump.age" + f.awsIT("s3 cp " + itDir + "/partial s3://itfleet/" + bad + " --only-show-errors") + + // Kill it again after the upload finished, before publication. + shim2 := "#!/bin/sh\ncase \"$*\" in *'s3 cp s3://'*.incoming*s3://*) touch " + itDir + "/publishing; sleep 600;; esac\nexec " + realAWS + " \"$@\"\n" + if err := vm.Upload(ctx, strings.NewReader(shim2), itShims+"/aws", "0755"); err != nil { + t.Fatal(err) + } + startKilled(itDir + "/publishing") + f.sh("kill -9 -- -$(cat " + itDir + "/pid); sleep 1; true") + + // The next run needs no cleanup and takes no misreading from any of it. + res, code, out := f.runAll("") + if code != 0 || res.Status != backup.RunOK { + t.Fatalf("the run after the kills: exit %d %s: %s", code, res.Status, out) + } + good := f.target(res, "teployit-db") + rows, err := backup.NewClient(vm, os.Stderr).ListBackupsRecursive(ctx, "teployit", f.s3) + if err != nil { + t.Fatal(err) + } + for _, r := range rows { + if r.Accessory == "db" && r.Database == "" && r.ID == "29991231-235959-fedcba9876543210" && r.Restorable { + t.Fatal("a below-floor object lists as restorable") + } + } + id, _, err := backup.NewClient(vm, os.Stderr).LatestBackupDate(ctx, "teployit", "db", f.s3) + if err != nil { + t.Fatal(err) + } + if id != good.ID { + t.Fatalf("latest = %s, want the good run's %s: a partial upload counted", id, good.ID) + } + f.awsIT("s3 rm s3://itfleet/" + bad + " --only-show-errors") + f.awsIT("s3 rm s3://itfleet/" + partial + " --only-show-errors") + // A holder alive and stuck past every deadline it had: the next run ends it. + startKilled(itDir + "/publishing") + holderPID := strings.TrimSpace(f.sh("cat " + itDir + "/pid")) + f.sh(`printf '{"pid":` + holderPID + `,"started_at":"2020-01-01T00:00:00Z"}' > "$HOME"/.config/teploy/backup-run.lock`) + res, code, out = f.runAll("") + if code != 0 || res.Status != backup.RunOK { + t.Fatalf("a stuck holder must not block forever: exit %d %s: %s", code, res.Status, out) + } + if alive := ssh.RunDetailed(ctx, vm, "kill -0 "+holderPID); !alive.Failed() { + t.Fatalf("the stuck holder %s is still alive", holderPID) + } + f.sh("kill -9 -- -" + holderPID + " 2>/dev/null; rm -rf /var/tmp/teploy-backup.* " + itShims + "/aws") + }) + + t.Run("partial failure across targets: others done, nothing pruned, exit and heartbeat say so", func(t *testing.T) { + f.t = t + p := base + p.Recipient = f.pub + p.HeartbeatURLs = append(p.HeartbeatURLs, "http://127.0.0.1:"+itHookPort+"/broken/"+itToken, "http://127.0.0.1:1/dead/"+itToken) + f.setPolicy(p) + f.sh("docker stop teployit-cache >/dev/null") + beforeKeys := f.keys("teployit/accessories/cache/") + res, code, out := f.runAll("") + f.sh("docker start teployit-cache >/dev/null") + f.waitFor("redis", "docker exec teployit-cache redis-cli ping") + if code == 0 || res.Status != backup.RunPartial { + t.Fatalf("exit %d status %s\n%s", code, res.Status, out) + } + if tr := f.target(res, "teployit-cache"); tr.Status != backup.TargetFailed || tr.Pruned != 0 { + t.Fatalf("cache: %+v", tr) + } + if tr := f.target(res, "teployit-db"); tr.Status != backup.TargetOK { + t.Fatalf("db must still be backed up: %+v", tr) + } + if got := f.keys("teployit/accessories/cache/"); strings.Join(got, ",") != strings.Join(beforeKeys, ",") { + t.Fatalf("a failed target's objects changed:\n%v\n%v", beforeKeys, got) + } + log := f.sh("cat " + itDir + "/hook.log") + if !strings.Contains(lastLines(log, 2), "status=down") { + t.Fatalf("heartbeat must say down: %s", log) + } + // A broken and a dead receiver did not fail what was backed up, and their + // tokens are nowhere in the output. + for _, h := range res.Heartbeats { + if strings.Contains(h.Host, itToken) || strings.Contains(h.Error, itToken) { + t.Fatalf("heartbeat result leaks the token: %+v", h) + } + } + if strings.Contains(out, itToken) { + t.Fatalf("output leaks the token: %s", out) + } + if lastEvent := f.sh("tail -n 1 /deployments/.teploy/apps/_fleet/events.jsonl"); !strings.Contains(lastEvent, `"status":"partial"`) || !strings.Contains(lastEvent, `"outcome":"failed"`) { + t.Fatalf("event: %s", lastEvent) + } + f.setPolicy(backup.FleetPolicy{V: "1", Interval: "1h", Retention: retention, Volumes: true, Recipient: f.pub, Adopted: base.Adopted, HeartbeatURLs: base.HeartbeatURLs}) + }) + + t.Run("GFS prunes exactly the expected set, sidecars included, in a real bucket", func(t *testing.T) { + f.t = t + // Twelve weeks of history for one accessory, mixed sidecars, one bad object. + seed := []string{} + start := time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC) + for i := 0; i < 90; i += 3 { + seed = append(seed, start.AddDate(0, 0, i).Format("20060102-150405")) + } + f.sh("head -c 2000 /dev/urandom > " + itDir + "/blob") + seedScript := "export AWS_ACCESS_KEY_ID=" + itAK + " AWS_SECRET_ACCESS_KEY=" + itSK + " AWS_DEFAULT_REGION=us-east-1; s() { aws --endpoint-url " + itEndpoint + " s3 cp --only-show-errors " + itDir + "/blob \"s3://itfleet/gfsapp/accessories/db/$1\"; }; " + for _, id := range seed { + seedScript += fmt.Sprintf("s %s.pgdump.age; s globals/%s.sql.gz.age; s meta/%s.json.age; s databases/extra/%s.pgdump.age; ", id, id, id, id) + } + f.sh(seedScript) + client := backup.NewClient(vm, os.Stderr) + policy := backup.RetentionPolicy{KeepLast: 2, KeepDaily: 3, KeepWeekly: 4, KeepMonthly: 2} + deleted, err := client.PruneBackups(ctx, "gfsapp", "accessories/db", f.s3, policy) + if err != nil { + t.Fatalf("prune: %v (deleted %v)", err, deleted) + } + want := map[string]bool{} + for _, id := range resticForget(seed, policy) { + want[id] = true + } + if len(want) >= len(seed) || len(want) < 5 { + t.Fatalf("a useless fixture: keeps %d of %d", len(want), len(seed)) + } + got := map[string]bool{} + for _, k := range f.keys("gfsapp/accessories/db/") { + parts := strings.Split(k, "/") + got[strings.Split(parts[len(parts)-1], ".")[0]] = true + } + if fmt.Sprint(sortedKeysOf(got)) != fmt.Sprint(sortedKeysOf(want)) { + t.Fatalf("kept %v, want %v", sortedKeysOf(got), sortedKeysOf(want)) + } + // Sidecars and per-database objects are gone with their dump, kept with it. + rest := f.keys("gfsapp/accessories/db/") + if len(rest) != 4*len(want) { + t.Fatalf("%d objects left for %d backups: %v", len(rest), len(want), rest) + } + }) + + t.Run("never prunes across servers", func(t *testing.T) { + f.t = t + client := backup.NewClient(vm, os.Stderr) + // Another server's marker in this namespace. + f.sh("printf x > " + itDir + "/o") + f.awsIT("s3 cp " + itDir + "/o s3://itfleet/gfsapp/.owners/aaaaaaaaaaaaaaaa --only-show-errors") + before := f.keys("gfsapp/accessories/db/") + _, err := client.PruneBackups(ctx, "gfsapp", "accessories/db", f.s3, backup.RetentionPolicy{KeepLast: 1}) + if err == nil || !strings.Contains(err.Error(), "shared backup namespace") { + t.Fatalf("err = %v", err) + } + if got := f.keys("gfsapp/accessories/db/"); len(got) != len(before) { + t.Fatalf("pruned across servers: %v", got) + } + // Per-server prefixes: a sibling prefix is invisible to this one's prune. + f.sh("head -c 2000 /dev/urandom > " + itDir + "/blob") + f.awsIT("s3 cp " + itDir + "/blob s3://itfleet/server-b/shared/accessories/db/20200101-000000.pgdump --only-show-errors") + f.awsIT("s3 cp " + itDir + "/blob s3://itfleet/server-b/shared/accessories/db/20200102-000000.pgdump --only-show-errors") + f.awsIT("s3 cp " + itDir + "/blob s3://itfleet/server-a/shared/accessories/db/20200101-000000.pgdump --only-show-errors") + f.awsIT("s3 cp " + itDir + "/blob s3://itfleet/server-a/shared/accessories/db/20200102-000000.pgdump --only-show-errors") + a := f.s3 + a.Prefix = "server-a" + if _, err := client.PruneBackups(ctx, "shared", "accessories/db", a, backup.RetentionPolicy{KeepLast: 1}); err != nil { + t.Fatal(err) + } + if got := f.keys("server-a/"); len(got) != 1 { + t.Fatalf("server-a kept %v", got) + } + if got := f.keys("server-b/"); len(got) != 2 { + t.Fatalf("server-a's prune reached server-b: %v", got) + } + }) + + t.Run("prefix change: old objects found and restorable, never pruned unless owned", func(t *testing.T) { + f.t = t + rows, err := backup.NewClient(vm, os.Stderr).ListBackupsRecursive(ctx, "teployit", f.s3) + if err != nil || len(rows) == 0 { + t.Fatalf("listing: %v %v", rows, err) + } + oldID := "" + for _, r := range rows { + if r.Accessory == "db" && r.Database == "" && r.Restorable && strings.HasSuffix(r.Key, ".age") { + oldID = r.ID + break + } + } + if oldID == "" { + t.Fatalf("no restorable db object: %+v", rows) + } + // Move the target onto a per-server prefix through the real command. + cred := itAK + "\n" + itSK + "\n" + out, err := f.cli(cred, "server", "backup-target", "set", "--bucket", "itfleet", "--endpoint", itEndpoint, "--prefix", "srv1") + if err != nil { + t.Fatalf("target set: %v\n%s", err, out) + } + if !strings.Contains(out, `Previous prefix ""`) { + t.Fatalf("prefix move not reported: %s", out) + } + target, err := backup.ReadTarget(ctx, vm) + if err != nil { + t.Fatal(err) + } + cfg := TargetS3(target) + if cfg.Prefix != "srv1" || len(cfg.PreviousPrefixes) != 1 { + t.Fatalf("target: %+v", cfg) + } + client := backup.NewClient(vm, os.Stderr) + // New run goes under the prefix; list finds both; old id restores. + res, code, runOut := f.runAll("") + if code != 0 { + t.Fatalf("run after prefix move: %s", runOut) + } + if tr := f.target(res, "teployit-db"); !strings.Contains(tr.Objects[len(tr.Objects)-1].Key, "s3://itfleet/srv1/teployit/") { + t.Fatalf("new objects not under the prefix: %+v", tr.Objects) + } + rows2, err := client.ListBackupsRecursive(ctx, "teployit", cfg) + if err != nil { + t.Fatal(err) + } + var sawOld, sawNew bool + for _, r := range rows2 { + sawOld = sawOld || (r.Previous && r.ID == oldID) + sawNew = sawNew || (!r.Previous && strings.Contains(r.Key, "/srv1/")) + } + if !sawOld || !sawNew { + t.Fatalf("list across prefixes: old=%v new=%v", sawOld, sawNew) + } + out, err = f.cli("", "backup", "restore", oldID, "--app", "teployit", "--accessory", "db", "--yes", "--no-backup", "--json") + if err != nil { + t.Fatalf("restoring from the old prefix: %v\n%s", err, out) + } + // Prune never touches the unowned old prefix. + before := f.keys("teployit/accessories/db/") + if _, err := client.PruneBackups(ctx, "teployit", "accessories/db", cfg, backup.RetentionPolicy{KeepLast: 1}); err != nil { + t.Fatal(err) + } + if got := f.keys("teployit/accessories/db/"); len(got) != len(before) { + t.Fatalf("pruned an unowned previous prefix: %d -> %d", len(before), len(got)) + } + // Owned: it ages out under retention, and the newest of all survives. + owned := cfg + owned.PreviousPrefixes = []backup.PreviousPrefix{{Prefix: "", Owned: true}} + if _, err := client.PruneBackups(ctx, "teployit", "accessories/db", owned, backup.RetentionPolicy{KeepLast: 1}); err != nil { + t.Fatal(err) + } + if got := f.keys("teployit/accessories/db/"); len(got) >= len(before) { + t.Fatalf("owned previous prefix did not age out: %v", got) + } + id, _, err := client.LatestBackupDate(ctx, "teployit", "db", owned) + if err != nil || id == "" { + t.Fatalf("latest after pruning: %q %v", id, err) + } + }) + + t.Run("status names every database, adopted and unmanaged, with no secrets", func(t *testing.T) { + f.t = t + f.sh("docker run -d --name omni_postgres --memory 128m -e POSTGRES_USER=appuser -e POSTGRES_PASSWORD=" + itDBPW + " postgres:17-alpine >/dev/null") + f.waitFor("unmanaged postgres", "docker exec omni_postgres pg_isready -h 127.0.0.1 -U appuser") + res, _, _ := f.runAll("") + found := false + for _, c := range res.UncoveredContainers { + found = found || c == "omni_postgres" + } + if !found { + t.Fatalf("an unmanaged postgres is not reported uncovered: %+v", res) + } + out, err := f.cli("", "--json", "backup", "status") + if err != nil { + t.Fatalf("status: %v\n%s", err, out) + } + for _, secretValue := range []string{itToken, itDBPW, itSK, itAK} { + if strings.Contains(out, secretValue) { + t.Fatalf("status leaks %q: %s", secretValue, out) + } + } + var st backupStatusDTO + if err := json.Unmarshal(lastJSONLine([]byte(out)), &st); err != nil { + t.Fatalf("status JSON: %v\n%s", err, out) + } + entry := func(name string) backup.StatusEntry { + for _, e := range st.Entries { + if e.Name == name { + return e + } + } + t.Fatalf("no entry %q in %+v", name, st.Entries) + return backup.StatusEntry{} + } + if e := entry("omni_postgres"); e.Coverage != backup.CoverageUncovered || !e.Unmanaged { + t.Fatalf("omni_postgres: %+v", e) + } + if e := entry("wordpress-shared-mysql"); e.Coverage != backup.CoverageAdopted || !e.Adopted || e.Newest == nil || !e.Newest.Encrypted || len(e.Databases) == 0 { + t.Fatalf("mysql: %+v", e) + } + if e := entry("teployit-db"); e.Coverage != backup.CoverageCovered || e.LastSuccess == nil || e.Newest == nil { + t.Fatalf("db: %+v", e) + } + if e := entry("teployit/volumes"); e.Schedule != "policy" || e.Newest == nil { + t.Fatalf("volumes: %+v", e) + } + if !st.Encrypted || !st.Target.Reachable { + t.Fatalf("summary: %+v", st) + } + if st.LastRun == nil { + t.Fatal("no last run") + } + sawUncovered := false + for _, u := range st.UncoveredDatabases { + sawUncovered = sawUncovered || u == "omni_postgres" + } + if !sawUncovered { + t.Fatalf("uncovered_databases: %v", st.UncoveredDatabases) + } + }) + + t.Run("adopting by name: an underscore, no labels", func(t *testing.T) { + f.t = t + f.psql("omni_postgres", "postgres", "CREATE DATABASE omni") + f.psql("omni_postgres", "omni", "CREATE TABLE o(id int); INSERT INTO o VALUES (1)") + out, err := f.cli("", "server", "backup-policy", "set", "--adopt", "omni_postgres:postgres:all", "--volume-max-gb", "0.000001") + if err != nil { + t.Fatalf("policy set: %v\n%s", err, out) + } + if !strings.Contains(out, "Adopted: omni_postgres") || strings.Contains(out, itToken) { + t.Fatalf("policy set output: %s", out) + } + crontab := f.sh("crontab -l") + if strings.Count(crontab, "backup run --all") != 1 || !strings.Contains(crontab, "[teploy-backup-all]") { + t.Fatalf("crontab: %s", crontab) + } + // Twice: still one line. + if out, err := f.cli("", "server", "backup-policy", "set", "--interval", "6h"); err != nil { + t.Fatalf("policy set: %v\n%s", err, out) + } + if crontab := f.sh("crontab -l"); strings.Count(crontab, "backup run --all") != 1 { + t.Fatalf("crontab grew: %s", crontab) + } + res, code, runOut := f.runAll("") + if code != 0 { + t.Fatalf("run: %s", runOut) + } + tr := f.target(res, "omni_postgres") + if tr.Status != backup.TargetOK { + t.Fatalf("omni: %+v", tr) + } + if len(f.keys("srv1/_adopted/accessories/omni_postgres/databases/omni/")) == 0 { + t.Fatal("no per-database object for an adopted postgres") + } + if len(f.keys("srv1/_adopted/accessories/omni_postgres/globals/")) == 0 { + t.Fatal("no globals for an adopted postgres cluster") + } + // Size guard: visible in the result, the event and status, nothing stored. + vol := f.target(res, "teployit/volumes") + if vol.Status != backup.TargetExcluded || !vol.SizeGuard { + t.Fatalf("size guard: %+v", vol) + } + if !strings.Contains(strings.Join(res.Omissions, "\n"), "volume_max_gb") { + t.Fatalf("omissions: %v", res.Omissions) + } + if ev := f.sh("tail -n 3 /deployments/.teploy/apps/teployit/events.jsonl"); !strings.Contains(ev, `"outcome":"refused"`) { + t.Fatalf("size guard not in events: %s", ev) + } + // Restore and verify address the adopted postgres. + out, err = f.cli("", "--json", "accessory", "verify-backup", "omni_postgres", "--adopted", "--database", "omni") + if err != nil || !strings.Contains(out, `"ok": true`) { + t.Fatalf("verify adopted postgres: %v\n%s", err, out) + } + id := "" + for _, o := range tr.Objects { + if o.Database == "omni" { + parts := strings.Split(o.Key, "/") + id = strings.Split(parts[len(parts)-1], ".")[0] + } + } + f.psql("omni_postgres", "omni", "DELETE FROM o") + out, err = f.cli("", "backup", "restore", id, "--adopted", "--accessory", "omni_postgres", "--database", "omni", "--yes", "--json") + if err != nil { + t.Fatalf("restore adopted postgres: %v\n%s", err, out) + } + if got := f.psql("omni_postgres", "omni", "SELECT COUNT(*) FROM o"); got != "1" { + t.Fatalf("restored rows = %s", got) + } + }) + + t.Run("destructive operations back up first and refuse when they cannot", func(t *testing.T) { + f.t = t + cur, err := backup.ReadTarget(ctx, vm) + if err != nil { + t.Fatal(err) + } + base := "" + if cur.Prefix != "" { + base = cur.Prefix + "/" + } + const state = `{"schema_version":2,"deployment_type":"container","ingress_mode":"host","updated_at":"2026-09-23T09:00:00Z","image_ref":"alpine:3","operation_id":"op1","generation":2,"current_port":3000,"current_hash":"h2"}` + f.sh("mkdir -p /deployments/teployit2/volumes/data && printf keep > /deployments/teployit2/volumes/data/f && chmod go-w /deployments/teployit2") + if err := vm.Upload(ctx, strings.NewReader(state), "/deployments/teployit2/state.json", "0600"); err != nil { + t.Fatal(err) + } + f.sh("docker run -d --name teployit2-db --memory 128m --label teploy.role=accessory --label teploy.app=teployit2 --label teploy.accessory=db -e POSTGRES_DB=two -e POSTGRES_USER=appuser -e POSTGRES_PASSWORD=" + itDBPW + " postgres:17-alpine >/dev/null") + f.waitFor("postgres", "docker exec teployit2-db pg_isready -h 127.0.0.1 -U appuser") + f.psql("teployit2-db", "two", "CREATE TABLE z(id int)") + // Upgrade, from a folder with teploy.yml. + dir := t.TempDir() + yml := "app: teployit2\nserver: " + f.host + "\ndomain: teployit2.example.test\nimage: alpine:3\naccessories:\n db:\n image: postgres:17-alpine\n env:\n POSTGRES_DB: two\n POSTGRES_USER: appuser\n POSTGRES_PASSWORD: " + itDBPW + "\n" + if err := os.WriteFile(filepath.Join(dir, "teploy.yml"), []byte(yml), 0o600); err != nil { + t.Fatal(err) + } + inDir := func(args ...string) (string, error) { + c := exec.CommandContext(ctx, f.local, append([]string{"--host", f.host, "--user", f.user, "--key", f.key}, args...)...) + c.Dir = dir + out, err := c.CombinedOutput() + return string(out), err + } + // The DB is down: the backup fails, so the upgrade is refused and nothing moves. + f.sh("docker stop teployit2-db >/dev/null") + before := f.sh("docker inspect -f '{{.Id}}' teployit2-db") + out, err := inDir("accessory", "upgrade", "db", "postgres:17-alpine") + if err == nil || !strings.Contains(out, "--no-backup") || !strings.Contains(out, "teployit2-db") { + t.Fatalf("upgrade with a failing backup: %v\n%s", err, out) + } + if after := f.sh("docker inspect -f '{{.Id}}' teployit2-db"); after != before { + t.Fatal("a refused upgrade replaced the container") + } + // Purge refuses the same way, naming what it would delete, and deletes nothing. + out, err = f.cli("", "remove", "--app", "teployit2", "--purge", "--yes") + if err == nil || !strings.Contains(out, "--no-backup") { + t.Fatalf("purge with a failing backup: %v\n%s", err, out) + } + if strings.TrimSpace(f.sh("cat /deployments/teployit2/volumes/data/f")) != "keep" { + t.Fatal("a refused purge deleted data") + } + f.sh("docker start teployit2-db >/dev/null") + f.waitFor("postgres", "docker exec teployit2-db pg_isready -h 127.0.0.1 -U appuser") + // Healthy: the upgrade backs up, then recreates. + keysBefore := len(f.keys(base + "teployit2/accessories/db/")) + out, err = inDir("accessory", "upgrade", "db", "postgres:17-alpine") + if err != nil || !strings.Contains(out, "Backed up the current data first") { + t.Fatalf("upgrade: %v\n%s", err, out) + } + if len(f.keys(base+"teployit2/accessories/db/")) <= keysBefore { + t.Fatal("no backup object from the upgrade") + } + f.waitFor("postgres", "docker exec teployit2-db pg_isready -h 127.0.0.1 -U appuser") + f.psql("teployit2-db", "two", "CREATE TABLE IF NOT EXISTS z(id int)") + // Healthy: the purge backs up the accessory and the volumes, then deletes. + keysBefore = len(f.keys(base + "teployit2/accessories/db/")) + out, err = f.cli("", "remove", "--app", "teployit2", "--purge", "--yes") + if err != nil { + t.Fatalf("purge: %v\n%s", err, out) + } + if len(f.keys(base+"teployit2/accessories/db/")) <= keysBefore || len(f.keys(base+"teployit2/volumes/")) == 0 { + t.Fatalf("no backups from the purge: %v", f.keys(base+"teployit2/")) + } + if res := ssh.RunDetailed(ctx, vm, "docker inspect teployit2-db >/dev/null 2>&1"); !res.Failed() { + t.Fatal("the purge left the accessory") + } + if strings.TrimSpace(f.sh("if [ -e /deployments/teployit2/volumes ]; then echo yes; fi")) != "" { + t.Fatal("the purge left the volumes") + } + }) + + t.Run("DR bundle: members encrypted, the age key wrapped to the owner, restored with the owner key", func(t *testing.T) { + f.t = t + const state = `{"schema_version":2,"deployment_type":"container","ingress_mode":"host","updated_at":"2026-09-23T09:00:00Z","image_ref":"alpine:3","operation_id":"op1","generation":2,"current_port":3000,"current_hash":"h2"}` + const release = `{"schema_version":1,"app":"teployit","hash":"h2","created_at":"2026-09-22T10:00:00Z","image_ref":"alpine:3","cmd":"sleep 600"}` + f.sh("mkdir -p /deployments/teployit/meta /deployments/teployit/volumes/data") + if err := vm.Upload(ctx, strings.NewReader(state), "/deployments/teployit/state.json", "0600"); err != nil { + t.Fatal(err) + } + if err := vm.Upload(ctx, strings.NewReader(release), "/deployments/teployit/meta/h2.json", "0600"); err != nil { + t.Fatal(err) + } + target, err := backup.ReadTarget(ctx, vm) + if err != nil { + t.Fatal(err) + } + cfg := TargetS3(target) + cfg.OwnerRecipient = f.pub + store := backup.S3BundleStore{S3: cfg} + appCfg := config.AppConfig{App: "teployit", Volumes: map[string]string{"data": "/app/data"}, Accessories: map[string]config.AccessoryConfig{ + "db": {Image: "postgres:17-alpine", Env: map[string]string{"POSTGRES_DB": "appdb", "POSTGRES_USER": "appuser", "POSTGRES_PASSWORD": itDBPW}}, + }} + client := backup.NewClient(vm, os.Stderr) + // No recovery key: the private key must not go anywhere. + if _, err := client.CreateBundle(ctx, backup.BundleOptions{App: "teployit", Config: appCfg, IncludeSecrets: true, IncludeAgeKey: true}, store); err == nil || !strings.Contains(err.Error(), "recovery key") { + t.Fatalf("--include-age-key without a recipient: %v", err) + } + m, err := client.CreateBundle(ctx, backup.BundleOptions{App: "teployit", Config: appCfg, IncludeSecrets: true, IncludeAgeKey: true, OwnerRecipient: f.pub, Version: "it"}, store) + if err != nil { + t.Fatalf("CreateBundle: %v", err) + } + if !m.Secrets.AgeKeyWrapped { + t.Fatalf("manifest: %+v", m.Secrets) + } + prefix := "srv1/teployit/dr/" + m.ID + "/" + keys := strings.Join(f.keys(prefix), "\n") + for _, want := range []string{prefix + "accessories/db.pgdump.age", prefix + "volumes/data.tar.gz.age", prefix + "age-key.age", prefix + "manifest.json"} { + if !strings.Contains(keys, want+"\n") && !strings.HasSuffix(keys, want) { + t.Fatalf("missing %s in:\n%s", want, keys) + } + } + // The bundled key opens with the owner key, and matches the server's; + // the server's own identity cannot open it. + f.awsIT("s3 cp s3://itfleet/" + prefix + "age-key.age " + itDir + "/age-key.age --only-show-errors") + if res := ssh.RunDetailed(ctx, vm, "age -d -i /deployments/.age-key "+itDir+"/age-key.age"); !res.Failed() { + t.Fatal("the wrapped key must open only with the owner's key") + } + if got, want := f.sh("age -d -i "+f.owner+" "+itDir+"/age-key.age | grep AGE-SECRET-KEY"), f.sh("grep AGE-SECRET-KEY /deployments/.age-key"); got != want { + t.Fatalf("the bundled key is not the server's") + } + // Restore needs the owner key; it works with it, nothing live touched. + if _, err := client.RestoreBundleIsolated(ctx, backup.BundleRestoreOptions{App: "teployit", ID: m.ID, Config: appCfg}, store); err == nil || !strings.Contains(err.Error(), "--identity") { + t.Fatalf("a wrapped key without an identity: %v", err) + } + f.sh("rm -rf /var/tmp/teploy-dr/teployit") + receipt, err := client.RestoreBundleIsolated(ctx, backup.BundleRestoreOptions{App: "teployit", ID: m.ID, Config: appCfg, Identity: f.owner}, store) + if err != nil { + t.Fatalf("RestoreBundleIsolated: %v", err) + } + if !receipt.OK { + t.Fatalf("receipt: %+v", receipt) + } + vm.Run(ctx, "docker ps -aq --filter name=teploy-dr- | xargs -r docker rm -fv >/dev/null 2>&1; rm -rf /var/tmp/teploy-dr/teployit") + }) + + t.Run("doctor flags a policy with no target and no recipient", func(t *testing.T) { + f.t = t + if _, err := backup.UnsetTarget(ctx, vm); err != nil { + t.Fatal(err) + } + c := doctorBackupPolicyCheck(ctx, vm, time.Now().UTC()) + if c.Result != doctorFail || !strings.Contains(c.Detail, "no target") { + t.Fatalf("doctor: %+v", c) + } + res, code, out := f.runAll("") + if code == 0 || res.Status != backup.RunFailed { + t.Fatalf("a run with no target: exit %d %s\n%s", code, res.Status, out) + } + if log := f.sh("cat " + itDir + "/hook.log"); !strings.Contains(lastLines(log, 1), "status=down") { + t.Fatalf("no-target run must heartbeat down: %s", log) + } + }) +} + +func lastLine(s string) string { + return lastLines(s, 1) +} + +func lastLines(s string, n int) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + if len(lines) < n { + return strings.Join(lines, "\n") + } + return strings.Join(lines[len(lines)-n:], "\n") +} + +func sortedKeysOf(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// resticForget is restic's forget, written out independently of selectKept: a +// second derivation of the expected set. +func resticForget(ids []string, p backup.RetentionPolicy) []string { + sorted := append([]string{}, ids...) + sort.Sort(sort.Reverse(sort.StringSlice(sorted))) + keep := map[string]bool{sorted[0]: true} + var lastD, lastW, lastM string + d, w, m := p.KeepDaily, p.KeepWeekly, p.KeepMonthly + for i, id := range sorted { + ts, _ := time.Parse("20060102-150405", id) + if i < p.KeepLast { + keep[id] = true + } + day := ts.Format("2006-01-02") + y, wk := ts.ISOWeek() + week := fmt.Sprintf("%d-%d", y, wk) + month := ts.Format("2006-01") + if d > 0 && day != lastD { + lastD, d = day, d-1 + keep[id] = true + } + if w > 0 && week != lastW { + lastW, w = week, w-1 + keep[id] = true + } + if m > 0 && month != lastM { + lastM, m = month, m-1 + keep[id] = true + } + } + return sortedKeysOf(keep) +} From 50437b6956cddff51b7d63f065fe26b61d8e3bcc Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Wed, 4 Nov 2026 22:44:13 -0700 Subject: [PATCH 07/17] docs: fleet backup engine in the README, CHANGELOG and AUDIT_OPEN Co-Authored-By: Claude Opus 5.5 --- AUDIT_OPEN.md | 72 ++++++++++++++++++++++++++++--- CHANGELOG.md | 66 ++++++++++++++++++++++++++++ README.md | 116 ++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 245 insertions(+), 9 deletions(-) diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 34d3eea..f737fb9 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -1,5 +1,66 @@ # Open audit items +## 2026-11-05 — Dash auto-config Slice 4 CLI: fleet backup engine (G1, G3, G4, G5) + +Plan: `../_internal/DASH_AUTOCONFIG_PLAN_2026-11-03.md` (sections 2.1-2.3, 5). Branch +`slice4-fleet-engine`. Closes the Slice 2 opens on G5/one-object-per-database, role +hashes unencrypted in sidecars, and `backup list` having no rows for `backup_databases:`. + +Decisions worth reviewing: +- `max_age_days` is a keep rule (restic keep-within), unioned with the rest, not a cap: + it is what it already was with `keep_last`, and a cap could not express deploy-ovh's + "hourly for 48h, then daily/weekly/monthly". `max_age 2 + daily 7 + weekly 4 + monthly 12` + is that policy, with no separate hourly rule. +- The recovery key lives in the policy, not the target: Dash forgets bucket credentials, + and setting a target needs them. +- Upload commit is `.incoming/` + verify + server-side copy, sidecars first, primary last, + rather than a manifest: every existing reader already treats "the primary object exists" + as the backup existing, and legacy objects have no manifest to find. +- `--all` skips targets with a `run`/`legacy` cron line (override) but not `custom` ones: + a hand-written job is not evidence of anything, and policy coverage is how they retire. +- `.owners/` markers refuse a prune in a namespace another server wrote to, + because a reused prefix or a same-named app is the failure a prefix alone cannot catch. + +Verified live (colima, MinIO via the real aws CLI, real age, the real linux binary): +`go test -tags integration -run TestFleetIntegration ./internal/cli`, 17 scenarios. Only +unit-tested: the deploy hook, healthchecks.io's /fail form, the janitors' age thresholds and +multipart abort, setup's aws install, the space preflights' refusal, a bucket refusing deletes. +Regressions: `TEPLOY_IT_ENGINES=postgres,mysql,redis go test -tags integration ./internal/backup`. +The other Slice 2 engines were not re-run. The DR fixture still needs a 022 umask (open below). + +Open / deferred: +- A schema name holding `'` or `\`: `mysqldump` mangles it in its own `USE` (seen live on + mysql:8), so it is reported as not backed up in omissions and status, never silently. + A Postgres or MySQL name with `=`, a leading `-`, a control character or over 64 bytes + likewise. Fixing needs dumping through `--databases`-free stdin tricks; not worth it yet. +- Adopted MySQL dumps schemas, not `mysql.user`: per-site database users must be recreated + when recovering `wordpress-shared-mysql` onto a new server. The postgres side has globals. +- An adopted container's password comes from its env. A root password changed after creation + (deploy-ovh's was stale) fails the dump loudly. There is no policy credential yet. +- Mongo with auth still fails loudly (Slice 2 item), now as a failing target. +- `aws` stays the transport (G8 deferred). Credential charset is restricted as before. +- Restore and verify of an adopted database stop nothing; the operator stops its users. +- Volume and generic estimates use the live tree's size, so a small disk can refuse an + archive that would have compressed to fit (`--ignore-space-check`, or a larger + `TEPLOY_BACKUP_TMPDIR`). The previous object's size would sharpen this. +- A provider's own checksum is not verified beyond the 256 MiB read-back: above that it is + stored size plus the recorded SHA-256 plus the aws CLI's transfer checksums. +- Heartbeat redirects are followed by Go's default client. Dash's push monitor does not + redirect; a receiver that does would see the token on the redirect target. +- Dash must read `backup-fleet-v1` before using any of this (plan: version-gated). Behavior + Dash and scripts see: `remove --purge` now needs a target or `--no-backup`; restores take a + backup first; a `backup_databases:` name that is missing now fails that accessory's backup. +- Pre-existing, seen live: `accessory upgrade` stops and removes the container before + `EnsureRunning`, so a failure there (an unsafe directory mode, here) leaves the accessory + down until re-run. Its data and the new safety backup are intact. Not changed in this slice. +- `AGENTS.md` points at a repo `CLAUDE.md` that does not exist. +- `backup status` runs one bucket listing per app and adopted namespace, and reads each + app's event log. Fine at fleet sizes here; a very large bucket would want a cache. +- Encrypted objects cannot be restored by a CLI older than this one: the `.age` suffix is + invisible to its listing. Release order: CLI on servers first (plan section 4). +- `TEPLOY_SERVER_BINARY` was used in the live test; on real servers `ensureFleetBinary` + upgrades via the release pipeline, so the policy cannot be set until this ships. + ## 2026-11-03 — Dash auto-config Slice 2: backup correctness (G6, G7) Plan: `../_internal/DASH_AUTOCONFIG_PLAN_2026-11-03.md` (U3, U4, P1 G6/G7, @@ -65,8 +126,8 @@ its marker table; checks run before the app stops; `database` in the restore event; `backup list` works without cron. Open / deferred: -- G5 adoption of unmanaged containers and MySQL one-object-per-database: not - in this slice. MySQL/MariaDB backups cover one schema and warn about the rest. +- ~~G5 adoption of unmanaged containers and MySQL one-object-per-database~~ closed + by Slice 4. - A scheduled backup alerts on coverage only where `backup schedule` has uploaded the current script (rescheduling updates it). Dash alerting: Slice 4. - Safety copies are not rolled back automatically for mysql/mongo/dragonfly: @@ -77,10 +138,9 @@ Open / deferred: - Mongo restore (--drop) leaves collections created after the backup. - verify-backup covers the primary database only, not backup_databases: dumps; DR bundles carry the primary only (their manifest notes what is missing). -- `backup list` shows one row per id; backup_databases: dumps have no rows of - their own. -- Role password hashes are in the postgres globals sidecar and DR member, - unencrypted until G1 (the dump beside them is too). +- ~~`backup list` one row per id~~ closed by Slice 4 (`database` on each row). +- ~~Role password hashes unencrypted in sidecars~~ closed by G1 when a recovery key + is configured. - Redis/valkey restore and verify assume /data/dump.rdb (Bitnami redis differs). - Dragonfly's pre-restore snapshot in /var/tmp has no free-space preflight (its size is the in-memory dataset). diff --git a/CHANGELOG.md b/CHANGELOG.md index 79b725a..5f3f6b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,72 @@ All notable changes to teploy are documented here. Format follows [Keep a Change ## [Unreleased] +### Added + +- Server backup policy: `teploy server backup-policy set/show/unset`, one cron + line, `teploy backup run --all`, enumerating at run time every labelled + accessory and each database in it, app volumes and adopted containers. A new + database is covered with no command. An overlap lock (an `flock`: a killed run + cannot block the next), per-target and per-run deadlines, one target failing + does not stop the others, and the exit status and `backup.all` event report + `ok`/`partial`/`failed` plus every omission. A per-app schedule stays an override. +- Adopted containers: policy entries `{container, engine, databases: all|[...]}` + bring unlabelled containers (`omni_postgres`, `wordpress-shared-mysql`) under the + policy, addressed by docker name (`--adopted`) in `backup restore/list/create/prune` + and `accessory verify-backup`. +- One object per database. MySQL/MariaDB schemas and Postgres databases each get + their own series (`databases//`), restored and verified alone with + `--database`; `backup_databases:` now works on MySQL too. +- GFS retention: `--keep-daily/-weekly/-monthly` beside `--keep-last` and + `--max-age-days`, a union of rules with restic `forget` semantics. `--max-age-days` + is restic's keep-within, which is also what it always was with `--keep-last`. +- Payload encryption: with a recovery key (`teploy backup keygen`; `backup-policy set + --recipient`), every object is age-encrypted to the owner key and the server's + identity, sidecars and DR members included. `teploy backup fetch --identity` + downloads and decrypts locally, for the recovery drill and a lost server. +- Heartbeats: `heartbeat_urls` are pinged after every `--all` run, with + `?status=down` on anything but a clean run. Policy-managed backups do not use the + per-app alert script. +- `teploy backup status --host S --json`: every DB-like container, labelled or not, + and every app's volumes, with coverage state, last success and failure, newest + object, last verify, uncovered databases, target and policy. No secrets. +- Automatic backup before `backup restore`, `accessory restore`, `accessory upgrade` + and `remove --purge` (refused if it fails, `--no-backup` overrides), and optionally + before a deploy with a `pre_deploy` hook (`backup_before_hooks: true`). +- Volume size guard: `volume_max_gb` warns and excludes, visible in status and events. +- `teploy doctor` check `backup-policy`; `teploy setup` installs the aws CLI (AWS's + bundle where the distribution has none, as on Ubuntu 24.04) and age. + +### Changed + +- Volume backups leave `.env` out when no recovery key is configured, with a warning: + it holds resolved secrets. Restoring such a backup keeps the server's current `.env`. +- `dr create --include-age-key` needs a recovery key and wraps the server's private + key to it (a plaintext key in a bucket is the keys to everything); `dr restore + --identity ` unwraps it. Bundles made before this keep restoring. +- `remove --purge` needs a backup target or `--no-backup`, names what it will delete, + and removes containers with `docker rm -v`. +- Backups and accessory restores build in `/var/tmp` (`TEPLOY_BACKUP_TMPDIR`), not a + `/tmp` that may be tmpfs, and refuse a dump that would not fit: plaintext, ciphertext + and a reserve, measured against what the backup will occupy. +- `backup list` and status no longer treat below-floor objects as restorable, and a + newest good backup is never pruned even with only `--max-age-days` set. + +### Fixed + +- `server backup-target set --prefix` broke listing: keys were matched without the + prefix and rebuilt with it twice. Previous prefixes stay searchable after a change, + and are pruned only when marked owned; a namespace another server has written to is + never pruned. +- A backup's sidecars could outlive it and a volume backup's server-data sibling was + never pruned; a pruned backup now takes every object with its id. +- Upload integrity: a final key only ever holds an object whose size and checksum were + verified, so a killed run can never leave a partial upload that counts as a backup. +- `aws s3 ls` exiting 1 on an empty prefix read as a failure on a first backup. +- Cron's minimal PATH hid an `aws` or `age` in `/usr/local/bin` from scheduled runs. +- `backup run` volume archives failed on GNU tar's "file changed" from a live app; the + artifact is already labelled crash-consistent. + ## [0.1.50] - 2026-11-04 ### Fixed diff --git a/README.md b/README.md index 7b652bd..33094b3 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ standalone. | **Asset bridging** | Shared volume prevents 404s on hashed assets during deploys | | **Destinations** | `teploy deploy -d staging` — per-environment config overlays | | **Notifications** | Webhook notifications on deploy/rollback/failure | -| **Backups** | `teploy backup` — volumes to S3; `teploy server backup-target` — the per-server target | +| **Backups** | `teploy backup` — volumes and databases to S3; `teploy server backup-target` and `backup-policy` — one target and one policy per server; `teploy backup status` — every database at a glance | | **Templates** | One-command deploys for common self-hosted apps | | **Deploy locking** | `teploy lock` — freeze deploys during incidents; `teploy lock status --json` reports who holds the lock | | **Vulnerability gate** | `scan: true` — Trivy scan blocks deploys on fixable CRITICALs | @@ -135,6 +135,7 @@ drain_seconds: 10 # request drain: keep the old container serving in memory: 2g # cgroup RAM cap (docker units: 512m, 2g). Unset = unlimited. cpu: "1.5" # CPU cap in cores. Unset = unlimited. keep_versions: 3 # auto-prune older versions after deploy (0 = keep all, default) +backup_before_hooks: true # back up running databases before a deploy whose pre_deploy hook will run (default off) # `teploy pin ` protects a version from this prune # Build a Dockerfile that lives in a subdirectory (monorepos). Omit both @@ -669,6 +670,115 @@ starts cron, `backup schedule` (and `accessory backup --schedule`) refuse on a server without one, and `teploy doctor` fails when teploy crontab lines or `/etc/cron.d` jobs exist but no cron daemon runs them (other jobs warn). +#### Server policy: one cron line for the whole server + +```bash +teploy server backup-policy set --host prod --interval 1h \ + --max-age-days 2 --keep-daily 7 --keep-weekly 4 --keep-monthly 12 \ + --recipient age1... # encrypt (see below) +teploy server backup-policy set --host prod --adopt omni_postgres:postgres:all \ + --adopt wordpress-shared-mysql:mysql:all --volumes --volume-max-gb 20 +printf 'https://dash.example/api/push/TOKEN\n' | teploy server backup-policy set --host prod --heartbeat-stdin +teploy server backup-policy show --host prod [--json] # heartbeat URLs only as hosts +teploy server backup-policy unset --host prod # stored backups stay +teploy backup status --host prod --json # every database, one call +``` + +The policy lives on the server (`/etc/teploy/backup-policy.json`, 0600; a user +who cannot write `/etc/teploy` gets `~/.config/teploy/backup-policy.json`). One +cron line, `teploy backup run --all`, works out at run time what to back up: +every labelled accessory and every database in it (each with its own object), +app volumes when enabled, and adopted containers. A database deployed tomorrow is +covered with no command. A per-app `backup schedule` stays as an override for its +target; a hand-written job under a teploy marker does not count as one. + +Targets run one at a time. One failing does not stop the next, and nothing of a +failed target is pruned. The exit status is nonzero unless every target succeeded, +and `backup.all` (under the pseudo-app `_fleet`) and `--json` both report `ok`, +`partial` or `failed` plus `omissions`: everything left out (exclusions, size-guard +hits, a DB-like container nothing adopts, a schema the tools cannot address). A +clean status with omissions is not a complete backup. A run that finds another +running stands down; the lock is an `flock`, so a killed run cannot block the next, +and a holder alive past the 23h run deadline is killed. Each target has a 6h deadline. + +A killed run's leftovers are invisible to later runs and visible to doctor: a final +key only ever holds a verified object (the upload goes to a `.incoming/` key, its +size and SHA-256 are checked, up to 256 MiB its bytes are read back, and only then +is it copied into place), sidecars are stored before the primary object that commits +a backup, and objects below their engine's plausibility floor are never a good copy. + +**Retention** is a union of rules over the good backups, restic `forget` style: +`--keep-last N`; `--keep-daily/-weekly/-monthly N` keep the newest backup of the last +N UTC days, ISO weeks and months that have one (an empty period does not use one up); +`--max-age-days D` keeps everything younger than D days (restic's `keep-within`), and +anything older goes unless another rule keeps it. The newest good backup is never +deleted. A pruned backup takes its globals, meta and per-database objects with it. +The same flags work on `teploy backup prune`. + +**Heartbeats**: after every run the server GETs each URL, adding `?status=down` on +anything but `ok` (`/fail` for healthchecks.io). A failing receiver is reported and +never fails the backup; tokens appear in no log, event or output. Policy-managed +runs do not use the per-app alert script (one signal, one owner); `backup run --app` +still does. + +**Adopted containers** have no teploy labels and are addressed by their docker name, +under the `_adopted` namespace: +`teploy backup restore --adopted --accessory wordpress-shared-mysql --database wp_blog --host prod`, +`teploy accessory verify-backup wordpress-shared-mysql --adopted --database wp_blog --host prod`, +`teploy backup list --adopted --host prod`. Nothing that uses an adopted container +is stopped for you, and the prompt says so. MySQL and MariaDB get one object per +schema, so one WordPress site restores alone. Postgres clusters get one per database +plus the roles (`pg_dumpall --globals-only`). + +#### Encryption and recovery + +```bash +teploy backup keygen # prints a recovery keypair once; stored nowhere +teploy server backup-policy set --host prod --recipient age1... +# The drill, and the way back from a lost server (runs here, needs only aws, +# the bucket credentials and the recovery key): +TEPLOY_S3_ACCESS_KEY=... TEPLOY_S3_SECRET_KEY=... \ + teploy backup fetch s3://bucket/prod/app/accessories/db/.pgdump.age \ + --identity recovery.txt --endpoint https://s3.example.com +``` + +With a recipient every object, sidecars and DR members included, is age-encrypted +to the owner key and the server's own identity (`.age` suffix), the ciphertext is +proven to decrypt before it leaves the server, and a failure to encrypt fails the +backup rather than uploading plaintext. Restore and verify run on the server with +its own identity. Older objects keep restoring. Rotation is forward-only: older +objects still need the old key. Without a recipient backups keep working unencrypted +(`encrypted: false` everywhere, so Dash can warn), and `.env`, which holds resolved +secrets, is left out of volume archives with a warning (restoring such a backup keeps +the server's current `.env`). `dr create --include-age-key` needs a recipient and +wraps the server's private key to it alone; `dr restore --identity ` unwraps it, +placing the recovery key on the target privately for that restore. + +#### Shared buckets + +Give each server its own `--prefix` on `server backup-target set`. Changing the +prefix keeps the old one on record: list, verify and restore still find objects under +it (`previous_prefix: true`). Its objects are never pruned, since they may be another +server's, unless `--previous-prefix-owned` says only this server wrote there; then +they age out under the policy's retention. As a second guard every server leaves a +marker at `//.owners/`, and a namespace holding another +server's marker is never pruned. + +#### Before anything destructive + +`backup restore`, `accessory restore`, `accessory upgrade` and `remove --purge` first +store a fresh backup of what they are about to replace, through the same path. If +that fails, or a purge or upgrade has no target to save to, the operation is refused +with what is at stake named; `--no-backup` goes ahead without one. A database or +container that no longer exists has nothing to protect and is not a refusal. With +`backup_before_hooks: true` in `teploy.yml`, a deploy whose `hooks.pre_deploy` is set +does the same for each running database accessory (default off; `deploy --no-backup`). + +`teploy doctor` flags a policy with no target, no running cron line, no recipient or +stale runs, no heartbeat, a missing `aws` or `age`, a target that does not answer, +leftover workspaces and uploads, and databases nothing backs up. `teploy setup` +installs `aws` (AWS's bundle where the distribution has no package) and `age`. + Accessory backups are engine-aware. The engine is detected from the container — the accessory's `engine:` in `teploy.yml` (set on the container as the `teploy.engine` label: postgres, mysql, mariadb, redis, valkey, @@ -724,8 +834,8 @@ health-checked. On failure — or an unhealthy app afterwards — the pre-restore recovery copy is kept and named. Non-interactive restores require `--yes`. -`--keep-last N` is a floor (the newest N are always kept, even if old); -`--max-age-days D` deletes older backups but never below that floor. A failed +`--keep-last N` keeps the newest N even if old; `--max-age-days D` keeps everything +younger and lets the rest go (see Retention above for the full rule set). A failed backup never triggers a prune. Both `backup create` and scheduled cron backups alert on failure via the app's `notifications` config — `create` uses all channels; the headless `backup run` posts to the first **webhook** channel From 1a0f204da757cff3b2f5e41dc269a870fb08e2fc Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:40:30 -0700 Subject: [PATCH 08/17] fix(backup): adversarial review of the fleet engine Blockers: - purge, upgrade and the deploy hook backed up only the primary: remove --purge said "backed up", stored appdb and destroyed analytics. They now store every database and refuse what they cannot cover; only a restore is scoped to one. - A leftover backup-schedule.json decided a policy run's retention (keep_last 2 deleted seeded history, reporting ok). The policy applies; the leftover warns. High: - A run that stored nothing still pruned, and read as ok. No objects, no prune; a target with earlier backups and nothing now is failed, heartbeat down. - Per-database objects followed the primary's ids. Each series is selected by the policy on its own; the primary's kept ids are an extra keep only. - .owners failed open. A listing error refuses; empty machine-id refuses; a marker is written before the first prune; machine-id + hostname name it and the age recipient in its body tells clones apart; previous prefixes are checked too. - status never looked at the bucket. Covered needs a restorable recent object, and only policy runs count: a pre-op copy cannot mask a failing --all. - Owner-key recovery: -i only when the identity exists, dr --prefix, --identity on backup restore, accessory restore and verify-backup. - A non-root user reading a 0700 /etc/teploy saw no policy and uploaded plaintext. It asks through passwordless sudo, follows its own ~/.config policy, or refuses. - backup list hides per-database rows unless --databases: Dash restores by id and accessory alone, which would replace the primary. Medium and low: refused prunes reach run warnings, events and doctor, with `backup-target owners [--clear]`; --ignore-space-check reaches the pre-restore backup; zero targets fail unless allow_empty; .incoming at any depth; an inspect error is not "no container"; prune refuses a clock an hour off the store's or a future-dated backup (--ignore-clock); --previous-prefix-owned marks only the prefix being left (--claim-previous-prefix for others); AWS's bundle is PGP-checked against a pinned key and aws must actually run; --deadline and target_deadline; legacy cron lines reported per entry and warned about; ClickHouse and friends reported uncovered; --no-credentials targets work; policy --stdin cannot silently drop the recipient and --host no longer counts as a policy flag; multipart abort scoped to owned namespaces; tmp_dir and TEPLOY_BACKUP_TMPDIR workspaces in the janitor and doctor; postgresql:// names refused; a stale holder's child groups killed; --checksum-algorithm above the read-back limit; safety copies protected for 7 days. Contracts rev 17 regenerated: per-entry encrypted and schedule_form, last_run.warnings, allow_empty, tmp_dir, target_deadline. Co-Authored-By: Claude Opus 5.5 --- contracts/MANIFEST.md | 2 +- .../backup-status-envelope/valid/fleet.json | 44 +- .../schema/backup-list-envelope.schema.json | 2 +- .../schema/backup-policy-envelope.schema.json | 10 + .../schema/backup-status-envelope.schema.json | 26 +- internal/backup/accessory.go | 37 +- internal/backup/backup.go | 86 +++- internal/backup/fleet.go | 92 +++- internal/backup/fleet_test.go | 197 +++++++- internal/backup/policy.go | 48 +- internal/backup/restore.go | 5 + internal/backup/retention.go | 195 +++++--- internal/backup/runlock_unix.go | 39 +- internal/backup/status.go | 125 ++++-- internal/backup/store.go | 425 ++++++++++++++---- internal/backup/store_test.go | 294 +++++++++++- internal/backup/store_test_helpers_test.go | 3 + internal/cli/accessory.go | 31 +- internal/cli/backup.go | 108 +++-- internal/cli/backup_cron_test.go | 2 +- internal/cli/backup_destructive_test.go | 66 +++ internal/cli/backup_fleet.go | 217 +++++++-- internal/cli/backup_fleet_test.go | 115 ++++- internal/cli/backup_target.go | 129 +++++- internal/cli/contracts_golden_test.go | 15 +- internal/cli/doctor.go | 12 + internal/cli/doctor_backup_test.go | 5 +- internal/cli/doctor_test.go | 10 +- internal/cli/dr.go | 40 +- internal/cli/remove.go | 14 +- internal/cli/setup.go | 2 +- 31 files changed, 1994 insertions(+), 402 deletions(-) diff --git a/contracts/MANIFEST.md b/contracts/MANIFEST.md index 4a07e7c..6c6e141 100644 --- a/contracts/MANIFEST.md +++ b/contracts/MANIFEST.md @@ -10,7 +10,7 @@ Neutron/Nucleus dependency and a public mirror. | Corpus rev | Emitting CLI | Machine Interface | Notes | |---|---|---|---| -| 17 | main (Dash auto-config Slice 4: fleet backup engine) | 2 | Additive. New artifacts backup-status-envelope (`backup status --json`: every DB-like container and app volume with coverage state, last success/failure, newest object, last verify, plus target and policy summaries; no secrets), backup-run-envelope (`backup run --all --json`) and backup-policy-envelope (`server backup-policy show/set --json`), fixtures GENERATED from the real DTOs. backup-list rows gain optional `database`, `encrypted`, `previous_prefix`; backup-create gains `encrypted` and `objects`; backup-restore gains `pre_backup_id`; backup-target gains `previous_prefixes`. Event kinds `backup.all` and `backup_policy.set` under the pseudo-app `_fleet`; adopted containers log under `_adopted`. version-handshake advertises `backup-fleet-v1` (regenerated mi2.json). No MI bump. | +| 17 | main (Dash auto-config Slice 4: fleet backup engine) | 2 | Additive. New artifacts backup-status-envelope (`backup status --json`: every DB-like container and app volume with coverage state, last success/failure, newest object, last verify, plus target and policy summaries; no secrets), backup-run-envelope (`backup run --all --json`) and backup-policy-envelope (`server backup-policy show/set --json`), fixtures GENERATED from the real DTOs. backup-list rows gain optional `encrypted`, `previous_prefix`, and `database` only with `--databases` or `--adopted` (a caller restoring by id alone would replace the primary); backup-create gains `encrypted` and `objects`; backup-restore gains `pre_backup_id`; backup-target gains `previous_prefixes`. Event kinds `backup.all` and `backup_policy.set` under the pseudo-app `_fleet`; adopted containers log under `_adopted`. version-handshake advertises `backup-fleet-v1` (regenerated mi2.json). No MI bump. | | 16 | main (server-authority spec P3 / plan D4, git sources) | 2 | Additive. New artifacts source-show-envelope (`config source show --json`: repository/branch/poll, public credential facts — type, deploy-key public half + fingerprint, token username, pinned host-key fingerprints — and the per-ref admission state), source-key-envelope (`config source key --json` for ssh/token and `host-key --json`) and trigger-poll-envelope (`trigger poll --json`: one result object per app — ref, head, admitted, reason vocabulary incl. force-push/retry-limit/not-due, deployed_release), all fixtures GENERATED from the real DTOs. version-handshake advertises `git-source-v1` (git: code source, per-app credentials outside /deployments, build_secrets) and `trigger-poll-v1` (the one admitter + reset/retry/install-timer) (regenerated mi2.json). No MI bump. | | 15 | main (server-authority spec D3: backup target) | 2 | Additive. One backup target per server, stored outside /deployments with age-encrypted credentials and read at run time (`server backup-target set/show/unset`; `backup-target-v1`). New artifacts backup-target-envelope, backup-list-envelope, backup-create-envelope, backup-restore-envelope — fixtures GENERATED from the real DTO encoders (`backup list --json` now reports parsed rows {id, kind, accessory, size_bytes, created_at, key} plus schedules discovered from teploy cron markers; the old bare-array-of-raw-`aws s3 ls`-lines `backup list --json` shape is gone — it was never a corpus artifact and pinned no consumer). `backup create/restore --json` envelopes; restore quiesces the app under its fenced lock (backup-json-v1). version-handshake advertises `backup-target-v1` and `backup-json-v1` (regenerated mi2.json). No MI bump. | | 14 | main (server-authority plan D2, folderless reads) | 2 | Additive. New artifacts releases-envelope (`teploy releases --app X --host S --json`: container release records newest first with current/rollback_available (+reason); static apps list retained releases — both via the envelope, the folder-static legacy bare array is unchanged), rollback-result (`teploy rollback [--to X] --json`: {app, server, from, to, ok}; failures keep the error envelope) and redeploy-result (`teploy redeploy [--json]`: {app, server, home, release, image, ok}). Fixtures GENERATED from the real DTOs. version-handshake advertises `secret-stdin-v1` (secret set KEY --stdin; folderless secret set/get/list/rm/rotate), `releases-json-v1`, `rollback-json-v1`, `redeploy-v1` (fresh deploy of the current release's recorded image, env re-resolved) and `env-config-guard-v1` (env set/unset refuse keys a Teploy-home config declares) (regenerated mi2.json). No MI bump. || 13 | main (server-authority spec P2 support) | 2 | Additive. New artifact config-merge-envelope (`teploy config merge --base --file - --json`: merged config + conflicting keys, read-only), fixture GENERATED from the real configMergeDTO. version-handshake advertises `config-merge-v1`, `config-unprivileged-v1` (`config ... --unprivileged` refuses host-reaching changes) and `deploy-home-v1` (`deploy --app X --image T --home teploy` creates a new Teploy-home app) (regenerated mi2.json). No MI bump. | diff --git a/contracts/fixtures/backup-status-envelope/valid/fleet.json b/contracts/fixtures/backup-status-envelope/valid/fleet.json index 90c6589..4940c7b 100644 --- a/contracts/fixtures/backup-status-envelope/valid/fleet.json +++ b/contracts/fixtures/backup-status-envelope/valid/fleet.json @@ -62,6 +62,9 @@ "omissions": [ "redis-cache: a database container no policy entry covers (adopt or exclude it)" ], + "warnings": [ + "blog-db: prune: shared backup namespace: another server (aaaaaaaaaaaaaaaa) also stores blog backups" + ], "stale": false }, "entries": [ @@ -77,6 +80,7 @@ "adopted": false, "coverage": "covered", "schedule": "policy", + "encrypted": true, "last_success": "2026-11-04T03:17:04Z", "newest": { "id": "20261104-031704-1f2e3d4c5b6a7988", @@ -103,6 +107,7 @@ "adopted": true, "coverage": "adopted", "schedule": "policy", + "encrypted": true, "last_success": "2026-11-04T03:17:04Z", "newest": { "id": "20261104-031704-1f2e3d4c5b6a7988", @@ -135,6 +140,7 @@ "adopted": true, "coverage": "failing", "schedule": "policy", + "encrypted": true, "last_failure": "2026-11-04T02:17:04Z", "last_error": "dumping omni: exit status 1", "uncovered_databases": [ @@ -151,7 +157,38 @@ "adopted": false, "coverage": "uncovered", "reason": "no policy entry covers this container (adopt or exclude it)", - "schedule": "none" + "schedule": "none", + "encrypted": false + }, + { + "name": "shop-db", + "kind": "accessory", + "app": "shop", + "accessory": "db", + "engine": "postgres", + "detection": "env", + "state": "running", + "unmanaged": false, + "adopted": false, + "coverage": "stale", + "reason": "a run succeeded, but the bucket holds no restorable object for it", + "schedule": "own", + "schedule_form": "legacy", + "encrypted": false, + "last_success": "2026-11-04T03:17:04Z" + }, + { + "name": "analytics-ch", + "kind": "unmanaged", + "engine": "clickhouse-server", + "detection": "image-name", + "state": "running", + "unmanaged": true, + "adopted": false, + "coverage": "uncovered", + "reason": "teploy has no backup method for clickhouse-server: back it up another way and exclude it", + "schedule": "none", + "encrypted": false }, { "name": "blog/volumes", @@ -162,11 +199,16 @@ "coverage": "excluded", "reason": "50.00 GB is over the policy's volume_max_gb 20.00", "schedule": "policy", + "encrypted": true, "size_bytes": 53687091200 } ], "uncovered_databases": [ + "analytics-ch", "omni_postgres/reports", "redis-cache" + ], + "warnings": [ + "cron marker teploy-accessory-backup:shop:db is a pre-D3 shell line that uploads plaintext whatever the policy says: rewrite it with `teploy server backup-target set`" ] } diff --git a/contracts/schema/backup-list-envelope.schema.json b/contracts/schema/backup-list-envelope.schema.json index e6efc29..7089dea 100644 --- a/contracts/schema/backup-list-envelope.schema.json +++ b/contracts/schema/backup-list-envelope.schema.json @@ -97,7 +97,7 @@ }, "database": { "type": "string", - "description": "Present on one database's own object (rev 17)." + "description": "Present on one database's own object, only with `backup list --databases` or --adopted (rev 17): a caller must pass it as `restore --database`, or it would replace the primary." }, "encrypted": { "type": "boolean", diff --git a/contracts/schema/backup-policy-envelope.schema.json b/contracts/schema/backup-policy-envelope.schema.json index 4e8d3c5..e75e9bf 100644 --- a/contracts/schema/backup-policy-envelope.schema.json +++ b/contracts/schema/backup-policy-envelope.schema.json @@ -119,6 +119,16 @@ }, "encrypted": { "type": "boolean" + }, + "allow_empty": { + "type": "boolean", + "description": "Zero targets is not a failure." + }, + "tmp_dir": { + "type": "string" + }, + "target_deadline": { + "type": "string" } } } diff --git a/contracts/schema/backup-status-envelope.schema.json b/contracts/schema/backup-status-envelope.schema.json index bfc87a0..1e23422 100644 --- a/contracts/schema/backup-status-envelope.schema.json +++ b/contracts/schema/backup-status-envelope.schema.json @@ -185,6 +185,9 @@ "recipient": { "type": "string", "description": "The owner's recovery PUBLIC key." + }, + "allow_empty": { + "type": "boolean" } } }, @@ -238,6 +241,13 @@ "stale": { "type": "boolean", "description": "Older than twice the policy interval." + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Includes refused prunes: the backup is stored, retention has stopped." } } }, @@ -251,7 +261,8 @@ "unmanaged", "adopted", "coverage", - "schedule" + "schedule", + "encrypted" ], "properties": { "name": { @@ -298,7 +309,7 @@ "excluded", "adopted" ], - "description": "adopted = covered, via a policy entry. A failing or stale adopted container reports failing or stale, with adopted:true." + "description": "adopted = covered, via a policy entry. A failing or stale adopted container reports failing or stale, with adopted:true. covered and adopted require a restorable object in the bucket within the limit; only policy runs count for a policy-covered entry." }, "reason": { "type": "string" @@ -430,6 +441,17 @@ "size_bytes": { "type": "integer", "minimum": 0 + }, + "encrypted": { + "type": "boolean", + "description": "This entry's new objects will be encrypted: the server's setting, except under a legacy shell cron line." + }, + "schedule_form": { + "enum": [ + "run", + "legacy" + ], + "description": "The own schedule's form; legacy uploads plaintext whatever the policy says." } } } diff --git a/internal/backup/accessory.go b/internal/backup/accessory.go index 9a39152..bcaea75 100644 --- a/internal/backup/accessory.go +++ b/internal/backup/accessory.go @@ -30,6 +30,9 @@ type BackupOptions struct { SingleDatabase string // IgnoreSpace skips the free-space preflight. IgnoreSpace bool + // SafetyCopy marks what this backup stores as a destructive operation's way + // back: retention leaves it alone for safetyWindow. + SafetyCopy bool } // PrimaryDatabase names the database a default restore of this accessory would @@ -162,8 +165,19 @@ func (c *Client) AccessoryBackupWith(ctx context.Context, app, name, image strin failed[missing] = "listed for backup but not on the server" } + primary := true + switch { + case opts.SingleDatabase != "" && (!coverage.PrimaryIsSingle || coverage.PrimaryMissing): + primary = false + case (det.Engine == EnginePostgres || det.Engine == EngineMySQL || det.Engine == EngineMariaDB) && (adopted || (multi && coverage.PrimaryMissing)): + // Nothing named this container's primary database: every database has + // its own series. Pruning sees that as one series per database. + primary = false + } + // Sidecars and extra databases first: the primary object commits the backup. - if det.Engine == EnginePostgres { + // No database, no sidecar: a roles file alone is not a backup of anything. + if det.Engine == EnginePostgres && (primary || len(coverage.Extra) > 0) { if w := c.backupPostgresGlobals(ctx, pgT, timestamp, workDir, s3, app, name); w != "" { res.Warnings = append(res.Warnings, w) fmt.Fprintf(c.out, "Warning: %s\n", w) @@ -181,15 +195,6 @@ func (c *Client) AccessoryBackupWith(ctx context.Context, app, name, image strin fmt.Fprintf(c.out, "Backed up database %s\n", db) } - primary := true - switch { - case opts.SingleDatabase != "" && (!coverage.PrimaryIsSingle || coverage.PrimaryMissing): - primary = false - case (det.Engine == EnginePostgres || det.Engine == EngineMySQL || det.Engine == EngineMariaDB) && (adopted || (multi && coverage.PrimaryMissing)): - // Nothing named this container's primary database: every database has - // its own series. Pruning sees that as one series per database. - primary = false - } if primary { obj, consistency, perr := c.backupPrimary(ctx, app, name, det, env, pgT, workDir, timestamp, s3, opts) res.Consistency = consistency @@ -213,6 +218,9 @@ func (c *Client) AccessoryBackupWith(ctx context.Context, app, name, image strin } res.FailedDatabases = failed c.markNamespaceOwner(ctx, s3, app) + if opts.SafetyCopy { + c.markSafetyCopy(ctx, s3, seriesPrefix(s3, app, name, ""), timestamp, &res) + } if len(failed) > 0 { return res, joinDatabaseFailures(failed, nil) } @@ -220,6 +228,15 @@ func (c *Client) AccessoryBackupWith(ctx context.Context, app, name, image strin return res, nil } +// markSafetyCopy records an id as a safety copy. Failing to is reported, not fatal: +// the copy exists, and only its protection from retention is lost. +func (c *Client) markSafetyCopy(ctx context.Context, s3 S3Config, prefix, id string, res *BackupResult) { + key := prefix + safetyDir + "/" + id + ".safety" + if _, err := c.exec.Run(ctx, "printf '%s\\n' safety | "+s3.AWS("s3 cp - "+ssh.ShellQuote(key))); err != nil { + res.Warnings = append(res.Warnings, fmt.Sprintf("could not protect safety copy %s from retention: %v", id, err)) + } +} + // joinDatabaseFailures renders per-database failures into one error. func joinDatabaseFailures(failed map[string]string, primary error) error { if len(failed) == 0 { diff --git a/internal/backup/backup.go b/internal/backup/backup.go index b3a23c8..546ffff 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -256,8 +256,18 @@ type Client struct { // engineOverride forces a container's engine: an adopted entry's engine:, // which has no label to live on. engineOverride map[string]string - // deadline overrides BackupDeadline (tests shorten it). + // deadline overrides BackupDeadline (--deadline, the policy, tests). deadline time.Duration + // readBack overrides readBackMax (tests shorten it). + readBack int64 +} + +// SetDeadline overrides the per-operation deadline of backups, restores and +// verifies through this client. +func (c *Client) SetDeadline(d time.Duration) { + if d > 0 { + c.deadline = d + } } // NewClient creates a new backup client. @@ -421,6 +431,9 @@ func (c *Client) BackupVolumesWith(ctx context.Context, app string, s3 S3Config, res.Key, res.SizeBytes = stored.Key, stored.Size res.Objects = append(res.Objects, BackupObject{Key: stored.Key, Size: stored.Size, Encrypted: stored.Encrypted}) c.markNamespaceOwner(ctx, s3, app) + if opts.SafetyCopy { + c.markSafetyCopy(ctx, s3, fmt.Sprintf("s3://%s/%s/volumes/", s3.Base(), app), timestamp, &res) + } fmt.Fprintf(c.out, "Backup complete: %s\n", res.Key) return res, nil } @@ -788,26 +801,75 @@ func (c *Client) SetSchedule(ctx context.Context, schedule, command, marker stri // AWSCLIRemedy is the exact way to give a server the aws CLI. `teploy setup` // runs AWSCLIInstallScript; Ubuntu 24.04+ has no apt package for it. -const AWSCLIRemedy = "run `teploy setup` against the server, or on the server: apt-get install -y awscli || { curl -fsSL \"https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip\" -o /tmp/awscliv2.zip && python3 -m zipfile -e /tmp/awscliv2.zip /tmp/awscliv2 && chmod -R 755 /tmp/awscliv2/aws && /tmp/awscliv2/aws/install; }" - -// AWSCLIInstallScript installs the aws CLI: the distribution package when it -// has one, else AWS's own bundle. python3 unpacks it (unzip may be absent), which -// drops the file modes, so they are restored for every user before installing: -// with u+x alone the installed aws runs for root and "Permission denied"s for cron. +const AWSCLIRemedy = "run `teploy setup` against the server (it installs the distribution's package, else AWS's bundle with its signature checked), or on the server: apt-get install -y awscli, else AWS's installer as documented at https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html" + +// awsCLIKeyFingerprint and awsCLIPublicKey pin AWS's bundle signer, from +// docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html (key +// A6310ACC4672475C, expires 2027-07-01; a signature made before expiry stays +// VALIDSIG, and a new key means updating this, not trusting whatever is served). +const awsCLIKeyFingerprint = "FB5DB77FD5C118B80511ADA8A6310ACC4672475C" + +const awsCLIPublicKey = `-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF2Cr7UBEADJZHcgusOJl7ENSyumXh85z0TRV0xJorM2B/JL0kHOyigQluUG +ZMLhENaG0bYatdrKP+3H91lvK050pXwnO/R7fB/FSTouki4ciIx5OuLlnJZIxSzx +PqGl0mkxImLNbGWoi6Lto0LYxqHN2iQtzlwTVmq9733zd3XfcXrZ3+LblHAgEt5G +TfNxEKJ8soPLyWmwDH6HWCnjZ/aIQRBTIQ05uVeEoYxSh6wOai7ss/KveoSNBbYz +gbdzoqI2Y8cgH2nbfgp3DSasaLZEdCSsIsK1u05CinE7k2qZ7KgKAUIcT/cR/grk +C6VwsnDU0OUCideXcQ8WeHutqvgZH1JgKDbznoIzeQHJD238GEu+eKhRHcz8/jeG +94zkcgJOz3KbZGYMiTh277Fvj9zzvZsbMBCedV1BTg3TqgvdX4bdkhf5cH+7NtWO +lrFj6UwAsGukBTAOxC0l/dnSmZhJ7Z1KmEWilro/gOrjtOxqRQutlIqG22TaqoPG +fYVN+en3Zwbt97kcgZDwqbuykNt64oZWc4XKCa3mprEGC3IbJTBFqglXmZ7l9ywG +EEUJYOlb2XrSuPWml39beWdKM8kzr1OjnlOm6+lpTRCBfo0wa9F8YZRhHPAkwKkX +XDeOGpWRj4ohOx0d2GWkyV5xyN14p2tQOCdOODmz80yUTgRpPVQUtOEhXQARAQAB +tCFBV1MgQ0xJIFRlYW0gPGF3cy1jbGlAYW1hem9uLmNvbT6JAlQEEwEIAD4CGwMF +CwkIBwIGFQoJCAsCBBYCAwECHgECF4AWIQT7Xbd/1cEYuAURraimMQrMRnJHXAUC +akV0ygUJDqP4lQAKCRCmMQrMRnJHXFHjD/9eyZLYcKuQOlLvtqSDtUBiEZf6ZZjM +i3ygYH8rJNtuToUH+HvSpe819urJCquXhDrlK6N+aqW0hCLtNABJG/vsafIgvIYJ +hSGgpgtNnQyMV1jViRWqPjbouw8OkYKBThUfT1i2Y+wn58ifs6ODBCmTexWtXspA +Si+Gt49xDOW0APmbOPnI+a4HJW6tVEo6MWS0WjzpiBayR3d1A4pt4YrPfSdDgpLo +h2SLQqlRqvvVZJaWBjhkErNFpfsBA06sDcPEOb0G8LBUbR4WOcdvhe5LubJbZuxC +AG9kNPCVeQP1ixwjgjXKysaxeQ6rv0VzIQgRp6tLVLWhy6AKDNvLjFSsmXZ1Wl08 +Y/RlOHXlzLuQMRE6sR1wOdRxc9TsrNWTGiBK65cvSWOy03JeBkQQ8pesqltiyxI9 +U21kkgiXtTSKNGfKK8pO27D81YANhRqPK7iTp6kuFiY2WtOg90KTMNlIT+Ff85Y2 +b1rHj6Z0SrCkJujhWk3IBPic/wJgz01LEc/OAdUPlby90RJZcIBhSlWhT7mXnXIO +c0HWlNQrns2s3CTyYwZSiSlYe9ApeLwhjDo8NhbFuCAy61l6O5UsR4AfZxx/rGKv +2wFb1/RN/P4gNe6vmxZAPjR0AQcwD3tc2McimOLr/22kmPz8IH3I0X7WoSFr0Biz +E91G7bb0hOb/cA== +=knv7 +-----END PGP PUBLIC KEY BLOCK----- +` + +// AWSCLIInstallScript installs the aws CLI: the distribution package when it has +// one, else AWS's own bundle, which runs as root, so its signature is checked +// against the pinned key first (TLS alone is not enough for that). python3 unpacks +// it (unzip may be absent), which drops the file modes, so they are restored for +// every user before installing: with u+x alone the installed aws runs for root and +// "Permission denied"s for cron. Success is `aws --version` running, not `command +// -v aws`: the glibc bundle on a musl host is present and cannot start. const AWSCLIInstallScript = `set -e export DEBIAN_FRONTEND=noninteractive -if apt-get install -y -qq awscli >/dev/null 2>&1 && command -v aws >/dev/null 2>&1; then exit 0; fi +if apt-get install -y -qq awscli >/dev/null 2>&1 && aws --version >/dev/null 2>&1; then exit 0; fi +command -v gpg >/dev/null 2>&1 || apt-get install -y -qq gnupg >/dev/null 2>&1 || { echo "gpg is needed to verify AWS's bundle" >&2; exit 1; } d=$(mktemp -d /tmp/teploy-awscli.XXXXXXXX) trap 'rm -rf "$d"' EXIT -curl -fsSL --max-time 300 "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "$d/a.zip" +GNUPGHOME="$d/g"; export GNUPGHOME; mkdir -m 700 "$GNUPGHOME" +u="https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" +curl -fsSL --max-time 300 "$u" -o "$d/a.zip" +curl -fsSL --max-time 60 "$u.sig" -o "$d/a.sig" +cat > "$d/key" <<'TEPLOY_KEY' +` + awsCLIPublicKey + `TEPLOY_KEY +gpg --batch --quiet --import "$d/key" 2>/dev/null +gpg --batch --status-fd 1 --verify "$d/a.sig" "$d/a.zip" 2>/dev/null | grep -q "^\[GNUPG:\] VALIDSIG ` + awsCLIKeyFingerprint + ` " || { echo "AWS's bundle did not verify against the pinned signing key; nothing was installed" >&2; exit 1; } python3 -m zipfile -e "$d/a.zip" "$d/x" chmod -R 755 "$d/x/aws" "$d/x/aws/install" --update -command -v aws >/dev/null 2>&1` +aws --version >/dev/null 2>&1 || { echo "AWS's bundle installed but does not run here (a musl libc?): install your distribution's aws-cli package" >&2; exit 1; }` func (c *Client) ensureAWSCLI(ctx context.Context) error { - if _, err := c.exec.Run(ctx, "which aws"); err != nil { - return fmt.Errorf("aws CLI not found on server — %s", AWSCLIRemedy) + // That it runs, not only that it is there: a glibc bundle on a musl host is. + if _, err := c.exec.Run(ctx, "which aws >/dev/null && aws --version >/dev/null 2>&1"); err != nil { + return fmt.Errorf("aws CLI not found or not runnable on server — %s", AWSCLIRemedy) } return nil } diff --git a/internal/backup/fleet.go b/internal/backup/fleet.go index c04d997..9932495 100644 --- a/internal/backup/fleet.go +++ b/internal/backup/fleet.go @@ -61,6 +61,34 @@ type ContainerFact struct { // Unmanaged reports a container with no teploy role. func (f ContainerFact) Unmanaged() bool { return f.Role == "" } +// unsupportedDatabases are images of databases teploy has no dump method for. +// They classify as generic, so nothing else would ever mention them; the same +// family liveDBMarkers refuses to tar from a running container. +var unsupportedDatabases = map[string]bool{ + "clickhouse": true, "clickhouse-server": true, "elasticsearch": true, "opensearch": true, + "qdrant": true, "keydb": true, "cassandra": true, "scylla": true, "couchdb": true, + "couchbase": true, "influxdb": true, "neo4j": true, "cockroach": true, "rethinkdb": true, + "etcd": true, "milvus": true, "weaviate": true, "typesense": true, "meilisearch": true, +} + +// UnsupportedDatabase names a database teploy cannot back up in this container, +// "" when it can or this is no database. A teploy.engine label asserting generic +// is the operator saying it is plain files. +func (f ContainerFact) UnsupportedDatabase() string { + if f.EngineErr != nil || f.Engine.IsDB() || f.Engine.Source == "label" || f.Engine.Source == "env-not-server" { + return "" + } + if name := imageName(f.Image); unsupportedDatabases[name] { + return name + } + return "" +} + +// DBLike reports a container holding a database, whether or not teploy can dump it. +func (f ContainerFact) DBLike() bool { + return (f.EngineErr == nil && f.Engine.IsDB()) || f.UnsupportedDatabase() != "" +} + // inventoryFormat emits one JSON object per container. const inventoryFormat = `{"name":{{json .Name}},"image":{{json .Config.Image}},"state":{{json .State.Status}},"labels":{{json .Config.Labels}},"env":{{json .Config.Env}},"entrypoint":{{json .Config.Entrypoint}},"cmd":{{json .Config.Cmd}}}` @@ -238,15 +266,16 @@ func (c *Client) duKB(ctx context.Context, dir string) (int64, bool) { // EnumerateFleet lists this run's targets and the DB-like containers none of // them cover. It reads the server now; nothing is cached between runs. -func (c *Client) EnumerateFleet(ctx context.Context, p FleetPolicy) ([]FleetTarget, []string, error) { +func (c *Client) EnumerateFleet(ctx context.Context, p FleetPolicy) ([]FleetTarget, []string, []string, error) { facts, err := c.Inventory(ctx) if err != nil { - return nil, nil, err + return nil, nil, nil, err } cron, err := ReadCrontab(ctx, c.exec) if err != nil { - return nil, nil, err + return nil, nil, nil, err } + var warnings []string ownCron := map[string]bool{} for _, line := range cron { // A hand-written job under a teploy marker (custom) is not an override: @@ -254,6 +283,9 @@ func (c *Client) EnumerateFleet(ctx context.Context, p FleetPolicy) ([]FleetTarg // policy covering the database as well is how they get retired. if m := FindCronMarker(line); m != nil && m.Kind != KindFleet && m.Form != CronFormCustom { ownCron[m.App+"/"+m.Accessory] = true + if m.Form == CronFormLegacy { + warnings = append(warnings, fmt.Sprintf("cron marker %s is a pre-D3 shell line: it uploads plaintext whatever this policy says. Rewrite it with `teploy server backup-target set`", m.Tag)) + } } } policies := map[string]AppPolicy{} @@ -269,11 +301,14 @@ func (c *Client) EnumerateFleet(ctx context.Context, p FleetPolicy) ([]FleetTarg return ap } base := p.Retention.Policy() - override := func(kp *KindPolicy, r RetentionPolicy) RetentionPolicy { - if kp == nil || (kp.KeepLast == 0 && kp.MaxAgeDays == 0) { - return r + // A per-app schedule overrides the policy only while its cron line runs it, and + // then this run skips the target. A leftover backup-schedule.json with no line + // must not decide this policy's retention: applying one deleted seeded history + // in a single run, with a clean status. + staleStore := func(kp *KindPolicy, label string) { + if kp != nil && (kp.KeepLast != 0 || kp.MaxAgeDays != 0) { + warnings = append(warnings, fmt.Sprintf("%s has retention in its backup-schedule.json (keep_last %d, max_age_days %d) but no cron line runs it, so the policy's retention applies; the leftover has no effect", label, kp.KeepLast, kp.MaxAgeDays)) } - return RetentionPolicy{KeepLast: kp.KeepLast, MaxAgeDays: kp.MaxAgeDays} } adopted := map[string]*AdoptedEntry{} for i := range p.Adopted { @@ -305,12 +340,12 @@ func (c *Client) EnumerateFleet(ctx context.Context, p FleetPolicy) ([]FleetTarg case f.Name != ContainerName(f.App, f.Accessory): t.Status, t.Reason = TargetFailed, fmt.Sprintf("container %s does not match its labels (expected %s), so it cannot be addressed; adopt it by name", f.Name, ContainerName(f.App, f.Accessory)) default: - ap := readAppPolicy(f.App) - var kp *KindPolicy - if ap.Accessories != nil { - kp = ap.Accessories[f.Accessory] + t.Retention = base + if !ownCron[f.App+"/"+f.Accessory] { + if ap := readAppPolicy(f.App); ap.Accessories != nil { + staleStore(ap.Accessories[f.Accessory], f.Name) + } } - t.Retention = override(kp, base) switch { case p.Excluded(f.App, f.Accessory, f.Name): t.Status, t.Reason = TargetExcluded, "excluded by policy" @@ -330,7 +365,7 @@ func (c *Client) EnumerateFleet(ctx context.Context, p FleetPolicy) ([]FleetTarg targets = append(targets, t) continue } - if f.EngineErr == nil && f.Engine.IsDB() && !p.Excluded("", "", f.Name) { + if f.DBLike() && !p.Excluded("", "", f.Name) { uncovered = append(uncovered, f.Name) } } @@ -344,12 +379,13 @@ func (c *Client) EnumerateFleet(ctx context.Context, p FleetPolicy) ([]FleetTarg if p.Volumes { trees, err := c.listVolumeTrees(ctx) if err != nil { - return targets, uncovered, err + return targets, uncovered, warnings, err } for _, tr := range trees { - t := FleetTarget{Kind: TargetVolumes, App: tr.App, SizeBytes: tr.KB * 1024} - ap := readAppPolicy(tr.App) - t.Retention = override(ap.Volumes, base) + t := FleetTarget{Kind: TargetVolumes, App: tr.App, SizeBytes: tr.KB * 1024, Retention: base} + if !ownCron[tr.App+"/"] { + staleStore(readAppPolicy(tr.App).Volumes, tr.App+"/volumes") + } switch { case p.Excluded(tr.App, "volumes", ""): t.Status, t.Reason = TargetExcluded, "excluded by policy" @@ -361,7 +397,7 @@ func (c *Client) EnumerateFleet(ctx context.Context, p FleetPolicy) ([]FleetTarg targets = append(targets, t) } } - return targets, uncovered, nil + return targets, uncovered, warnings, nil } // RunFleetTarget backs one target up and, only if that fully succeeded, prunes. @@ -403,8 +439,26 @@ func (c *Client) RunFleetTarget(ctx context.Context, t FleetTarget, s3 S3Config, return r } r.Status = TargetOK + // Nothing stored is no new copy: the run that made none is the last one to + // delete old ones. And a target that used to have backups and now has nothing + // to back up (every database dropped) is an outage, not a quiet success. + if len(res.Objects) == 0 { + rows, lerr := c.ListBackupsRecursive(ctx, t.App, s3) + switch { + case lerr != nil: + r.Status, r.Reason = TargetFailed, "stored nothing, and could not check for earlier backups: "+lerr.Error() + default: + for _, row := range rows { + if row.Accessory == t.Name && row.Restorable { + r.Status, r.Reason = TargetFailed, "this target has earlier backups and now holds nothing to back up: its databases are gone. Nothing was pruned" + break + } + } + } + return r + } + c.CleanStaleIncoming(ctx, t.App, s3) if t.Retention.IsZero() { - c.CleanStaleIncoming(ctx, t.App, prefix, s3) return r } deleted, perr := c.PruneBackups(ctx, t.App, prefix, s3, t.Retention) diff --git a/internal/backup/fleet_test.go b/internal/backup/fleet_test.go index 63b3468..d44171a 100644 --- a/internal/backup/fleet_test.go +++ b/internal/backup/fleet_test.go @@ -110,7 +110,7 @@ func TestExcludedMatches(t *testing.T) { } func TestReadFleetPolicyFailsClosed(t *testing.T) { - mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "for p in", Err: errors.New("exit status 3: cannot read /etc/teploy/backup-policy.json")}) + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: policyReadMarker, Err: errors.New("exit status 3: cannot read /etc/teploy/backup-policy.json")}) if _, err := ReadFleetPolicy(context.Background(), mock); err == nil { t.Fatal("an unreadable policy must not read as no policy") } @@ -118,7 +118,7 @@ func TestReadFleetPolicyFailsClosed(t *testing.T) { if err := ApplyPolicyEncryption(context.Background(), mock, &cfg); err == nil { t.Fatal("an unreadable policy must refuse, not go on unencrypted") } - mock = ssh.NewMockExecutor("h", ssh.MockCommand{Match: "for p in", Output: "path /etc/teploy/backup-policy.json\n" + `{"v":"1","recipient":"` + testOwnerRecipient + `"}`}) + mock = ssh.NewMockExecutor("h", ssh.MockCommand{Match: policyReadMarker, Output: "path /etc/teploy/backup-policy.json\n" + `{"v":"1","recipient":"` + testOwnerRecipient + `"}`}) if err := ApplyPolicyEncryption(context.Background(), mock, &cfg); err != nil || cfg.OwnerRecipient != testOwnerRecipient { t.Fatalf("%v %+v", err, cfg) } @@ -154,7 +154,7 @@ func TestEnumerateFleet(t *testing.T) { ) p := FleetPolicy{Retention: FleetRetention{KeepLast: 3}, Volumes: true, VolumeMaxGB: 1, Excludes: []string{"gone/volumes"}, Adopted: []AdoptedEntry{{Container: "omni_postgres", Databases: DatabaseSelector{All: true}}, {Container: "missing", Databases: DatabaseSelector{All: true}}}} - targets, uncovered, err := NewClient(mock, io.Discard).EnumerateFleet(context.Background(), p) + targets, uncovered, _, err := NewClient(mock, io.Discard).EnumerateFleet(context.Background(), p) if err != nil { t.Fatal(err) } @@ -348,10 +348,10 @@ func TestFleetStatusCoverage(t *testing.T) { inventoryLine("stray", "redis:7", "running", nil, "REDIS_VERSION=7"), }, "\n") now := mustTime(t, "20260201-120000") - evA := `{"v":1,"id":"1","seq":1,"ts":"2026-02-01T11:00:00Z","app":"a","kind":"backup.run","outcome":"succeeded","detail":{"accessory":"db","kind":"accessory"}}` - evAd := `{"v":1,"id":"2","seq":1,"ts":"2026-01-01T11:00:00Z","app":"_adopted","kind":"backup.run","outcome":"failed","detail":{"accessory":"b_mysql","kind":"accessory","error":"boom"}}` + evA := `{"v":1,"id":"1","seq":1,"ts":"2026-02-01T11:00:00Z","app":"a","kind":"backup.run","outcome":"succeeded","detail":{"accessory":"db","kind":"accessory","policy_managed":true}}` + evAd := `{"v":1,"id":"2","seq":1,"ts":"2026-01-01T11:00:00Z","app":"_adopted","kind":"backup.run","outcome":"failed","detail":{"accessory":"b_mysql","kind":"accessory","error":"boom","policy_managed":true}}` mock := ssh.NewMockExecutor("h", - ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "path /etc/teploy/backup-policy.json\n" + `{"v":"1","interval":"1h","recipient":"` + testOwnerRecipient + `","heartbeat_urls":["https://d/api/push/SECRETTOKEN"],"adopted":[{"container":"b_mysql","databases":"all"},{"container":"gone","databases":"all"}]}`}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "path /etc/teploy/backup-policy.json\n" + `{"v":"1","interval":"1h","recipient":"` + testOwnerRecipient + `","heartbeat_urls":["https://d/api/push/SECRETTOKEN"],"adopted":[{"container":"b_mysql","databases":"all"},{"container":"gone","databases":"all"}]}`}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"bk","region":"r","prefix":"p","credentials":"SECRETARMOR"}`}, ssh.MockCommand{Match: "command -v aws >/dev/null", Output: "aws\nage\n"}, ssh.MockCommand{Match: "crontab -l", Output: "5 * * * * /usr/local/bin/teploy backup run --all # [teploy-backup-all]\n"}, @@ -360,7 +360,7 @@ func TestFleetStatusCoverage(t *testing.T) { ssh.MockCommand{Match: "( c=$(python3", Output: "2026-02-01 11:00:00 1000 p/a/accessories/db/20260201-110000.pgdump.age\n"}, ssh.MockCommand{Match: "python3 -c", Output: evA + "\n" + evAd + "\n"}, ssh.MockCommand{Match: "for d in /deployments/*/volumes", Output: ""}, - ssh.MockCommand{Match: "for d in /var/tmp/teploy-backup.", Output: "/var/tmp/teploy-backup.old\n"}, + ssh.MockCommand{Match: "for d in '/var/tmp'/teploy-backup.", Output: "/var/tmp/teploy-backup.old\n"}, ) st, err := ReadFleetStatus(context.Background(), mock, now) if err != nil { @@ -432,3 +432,186 @@ func TestEntryObjectsAndEvents(t *testing.T) { t.Fatal("a good object is evidence of a success") } } + +// ---- review regressions ---- + +// A leftover backup-schedule.json with no cron line must not decide retention. +func TestEnumerateIgnoresAStaleScheduleStore(t *testing.T) { + labels := map[string]string{"teploy.role": "accessory", "teploy.app": "a", "teploy.accessory": "db"} + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: inventoryLine("a-db", "postgres:17", "running", labels, "PG_MAJOR=17")}, + ssh.MockCommand{Match: "crontab -l", Output: ""}, + ssh.MockCommand{Match: "cat", Output: `{"accessories":{"db":{"keep_last":2}}}`}, + ) + p := FleetPolicy{Retention: FleetRetention{KeepDaily: 30, KeepWeekly: 8, KeepMonthly: 12}} + targets, _, warnings, err := NewClient(mock, io.Discard).EnumerateFleet(context.Background(), p) + if err != nil { + t.Fatal(err) + } + if got := targets[0].Retention; got.KeepDaily != 30 || got.KeepLast != 0 { + t.Fatalf("retention = %+v", got) + } + if len(warnings) != 1 || !strings.Contains(warnings[0], "no cron line runs it") { + t.Fatalf("warnings = %v", warnings) + } +} + +// Stored nothing: no prune, ever. With earlier backups, a failure. +func TestRunFleetTargetStoredNothing(t *testing.T) { + mk := func(listing string) *ssh.MockExecutor { + return newStoreMock("h", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "docker inspect -f", Output: "{}\n[]\n[]\n"}, + ssh.MockCommand{Match: "umask 077", Output: "/var/tmp/teploy-backup.t\n"}, + ssh.MockCommand{Match: "docker exec 'a-db' pg_dumpall", Output: ""}, + ssh.MockCommand{Match: "docker exec 'a-db' psql -X -qtA -v ON_ERROR_STOP=1 -U 'postgres' -d 'postgres' -c 'SELECT datname", Output: "postgres\n"}, + ssh.MockCommand{Match: "docker exec", Output: "0\n"}, + ssh.MockCommand{Match: "rm -rf", Output: ""}, + ssh.MockCommand{Match: "aws s3 ls", Output: listing}, + ssh.MockCommand{Match: "aws s3", Output: ""}, + ) + } + target := FleetTarget{Kind: TargetAccessory, App: "a", Name: "db", Container: "a-db", Fact: &ContainerFact{Image: "postgres:17", Env: map[string]string{"PG_MAJOR": "17", "POSTGRES_DB": "app"}}, Retention: RetentionPolicy{KeepLast: 1}} + mock := mk("2026-01-01 00:00:00 1000 a/accessories/db/20260101-000000.pgdump\n") + r := NewClient(mock, io.Discard).RunFleetTarget(context.Background(), target, S3Config{Bucket: "b", Region: "r"}, BackupOptions{}) + if r.Status != TargetFailed || !strings.Contains(r.Reason, "earlier backups") { + t.Fatalf("%+v", r) + } + for _, c := range mock.Calls { + if strings.Contains(c, "s3 rm") && !strings.Contains(c, ".incoming") { + t.Fatalf("pruned after storing nothing: %s", c) + } + } + // Nothing yet and no history: ok, and the summary names it. + mock = mk("") + r = NewClient(mock, io.Discard).RunFleetTarget(context.Background(), target, S3Config{Bucket: "b", Region: "r"}, BackupOptions{}) + if r.Status != TargetOK || len(r.Objects) != 0 { + t.Fatalf("%+v", r) + } +} + +// Databases teploy has no method for are generic, and would otherwise vanish. +func TestUnsupportedDatabasesAreReported(t *testing.T) { + inv := strings.Join([]string{ + inventoryLine("ch", "clickhouse/clickhouse-server:24", "running", nil), + inventoryLine("es", "docker.elastic.co/elasticsearch/elasticsearch:8", "running", nil), + inventoryLine("q", "qdrant/qdrant", "running", nil), + inventoryLine("web", "nginx", "running", nil), + inventoryLine("declared", "qdrant/qdrant", "running", map[string]string{EngineLabel: "generic"}), + }, "\n") + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: inv}, + ssh.MockCommand{Match: "crontab -l", Output: ""}, + ) + _, uncovered, _, err := NewClient(mock, io.Discard).EnumerateFleet(context.Background(), FleetPolicy{}) + if err != nil { + t.Fatal(err) + } + if strings.Join(uncovered, ",") != "ch,es,q" { + t.Fatalf("uncovered = %v", uncovered) + } +} + +// Covered needs a restorable object now, from what covers the entry. +func TestStatusCoverageNeedsAnObject(t *testing.T) { + labels := map[string]string{"teploy.role": "accessory", "teploy.app": "a", "teploy.accessory": "db"} + now := mustTime(t, "20260201-120000") + ev := func(ts, outcome, extra string) string { + return `{"v":1,"id":"1","seq":1,"ts":"` + ts + `","app":"a","kind":"backup.run","outcome":"` + outcome + `","detail":{"accessory":"db","kind":"accessory"` + extra + `}}` + } + run := func(events, listing string) StatusEntry { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: policyReadMarker, Output: "path /etc/teploy/backup-policy.json\n" + `{"v":"1","interval":"1h"}`}, + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"bk","region":"r"}`}, + ssh.MockCommand{Match: "command -v aws >/dev/null", Output: "aws\nage\n"}, + ssh.MockCommand{Match: "crontab -l", Output: "5 * * * * /usr/local/bin/teploy backup run --all # [teploy-backup-all]\n"}, + ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: inventoryLine("a-db", "redis:7", "running", labels, "REDIS_VERSION=7")}, + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "aws s3 ls", Output: listing}, + ssh.MockCommand{Match: "python3 -c", Output: events}, + ssh.MockCommand{Match: "for d in /deployments/*/volumes", Output: ""}, + ) + st, err := ReadFleetStatus(context.Background(), mock, now) + if err != nil { + t.Fatal(err) + } + return st.Entries[0] + } + fresh := "2026-02-01 11:00:00 1000 a/accessories/db/20260201-110000.rdb.gz\n" + if e := run(ev("2026-02-01T11:00:00Z", "succeeded", `,"policy_managed":true`), fresh); e.Coverage != CoverageCovered { + t.Fatalf("covered: %+v", e) + } + // The namespace is gone from the bucket. + if e := run(ev("2026-02-01T11:00:00Z", "succeeded", `,"policy_managed":true`), ""); e.Coverage != CoverageStale || !strings.Contains(e.Reason, "no restorable object") { + t.Fatalf("wiped: %+v", e) + } + // A pre-op single-database success does not mask a failing --all. + evs := ev("2026-02-01T10:00:00Z", "failed", `,"policy_managed":true,"error":"boom"`) + "\n" + ev("2026-02-01T11:00:00Z", "succeeded", `,"trigger":"this restore"`) + if e := run(evs, fresh); e.Coverage != CoverageFailing { + t.Fatalf("masked: %+v", e) + } + // Nor does a manual backup. + evs = ev("2026-02-01T10:00:00Z", "failed", `,"policy_managed":true,"error":"boom"`) + "\n" + ev("2026-02-01T11:00:00Z", "succeeded", "") + if e := run(evs, fresh); e.Coverage != CoverageFailing { + t.Fatalf("masked by a manual run: %+v", e) + } + // A successful run that is old, with an old object. + if e := run(ev("2026-01-01T11:00:00Z", "succeeded", `,"policy_managed":true`), "2026-01-01 11:00:00 1000 a/accessories/db/20260101-110000.rdb.gz\n"); e.Coverage != CoverageStale { + t.Fatalf("old: %+v", e) + } +} + +func TestStatusReportsALegacyLinePerEntry(t *testing.T) { + labels := map[string]string{"teploy.role": "accessory", "teploy.app": "a", "teploy.accessory": "db"} + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: policyReadMarker, Output: "path /etc/teploy/backup-policy.json\n" + `{"v":"1","interval":"1h","recipient":"` + testOwnerRecipient + `"}`}, + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "absent\n"}, + ssh.MockCommand{Match: "command -v aws >/dev/null", Output: "aws\nage\n"}, + ssh.MockCommand{Match: "crontab -l", Output: "0 5 * * * TEPLOY_S3_ACCESS_KEY=ak /usr/local/bin/teploy accessory backup db --local --app a --bucket b # [teploy-accessory-backup:a:db]\n"}, + ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: inventoryLine("a-db", "redis:7", "running", labels, "REDIS_VERSION=7")}, + ssh.MockCommand{Match: "python3 -c", Output: ""}, + ssh.MockCommand{Match: "for d in /deployments/*/volumes", Output: ""}, + ) + st, err := ReadFleetStatus(context.Background(), mock, mustTime(t, "20260201-120000")) + if err != nil { + t.Fatal(err) + } + e := st.Entries[0] + if e.Encrypted || e.ScheduleForm != CronFormLegacy || !st.Encrypted { + t.Fatalf("%+v", e) + } + if !strings.Contains(strings.Join(st.Warnings, ""), "pre-D3") { + t.Fatalf("%v", st.Warnings) + } +} + +// A policy this user cannot read is no policy to nobody: it fails closed. +func TestHiddenPolicyFailsClosed(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: policyReadMarker, Err: errors.New("exit status 3: cannot tell whether this server's backups must be encrypted")}) + _, err := ReadFleetPolicy(context.Background(), mock) + if !errors.Is(err, ErrPolicyHidden) { + t.Fatalf("%v", err) + } + var cfg S3Config + if err := ApplyPolicyEncryption(context.Background(), mock, &cfg); !errors.Is(err, ErrPolicyHidden) { + t.Fatalf("%v", err) + } +} + +func TestPolicyValidateNewFields(t *testing.T) { + if (FleetPolicy{TmpDir: "relative"}).Validate() == nil || (FleetPolicy{TmpDir: "/a b"}).Validate() == nil { + t.Fatal("tmp_dir") + } + if (FleetPolicy{TargetDeadline: "soon"}).Validate() == nil || (FleetPolicy{TargetDeadline: "12h", TmpDir: "/mnt/big"}).Validate() != nil { + t.Fatal("target_deadline") + } +} + +func TestChildPIDs(t *testing.T) { + if _, err := os.Stat("/proc/self/stat"); err != nil { + t.Skip("no /proc") + } + if len(childPIDs(1)) == 0 { + t.Fatal("pid 1 has children") + } +} diff --git a/internal/backup/policy.go b/internal/backup/policy.go index 14dae91..2a6367d 100644 --- a/internal/backup/policy.go +++ b/internal/backup/policy.go @@ -3,6 +3,7 @@ package backup import ( "context" "encoding/json" + "errors" "fmt" "hash/fnv" "net/url" @@ -102,6 +103,13 @@ type FleetPolicy struct { // Recipient is the owner's recovery public key (`teploy backup keygen`). // With one, every object is encrypted to it and the server's own identity. Recipient string `json:"recipient,omitempty"` + // AllowEmpty says a run with nothing to back up is fine. Without it, zero + // targets is a failure: a lost accessory and an empty docker daemon look the same. + AllowEmpty bool `json:"allow_empty,omitempty"` + // TmpDir builds backups on another disk. The janitor and doctor look there too. + TmpDir string `json:"tmp_dir,omitempty"` + // TargetDeadline replaces the 6h deadline of one target ("12h"). + TargetDeadline string `json:"target_deadline,omitempty"` // Path is where the policy was read from (diagnostics only). Path string `json:"-"` @@ -130,6 +138,14 @@ func (p FleetPolicy) Validate() error { return err } } + if p.TmpDir != "" && !tmpDirRE.MatchString(p.TmpDir) { + return fmt.Errorf("tmp_dir must be an absolute path of letters, digits and . _ - /") + } + if p.TargetDeadline != "" { + if d, err := time.ParseDuration(p.TargetDeadline); err != nil || d <= 0 { + return fmt.Errorf("target_deadline must be a positive duration such as 12h") + } + } for _, e := range p.Excludes { if err := validateExclude(e); err != nil { return err @@ -164,6 +180,8 @@ func (p FleetPolicy) Validate() error { return nil } +var tmpDirRE = regexp.MustCompile(`^/[A-Za-z0-9._/-]+$`) + var containerNameRE = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,254}$`) // ValidateContainerName applies docker's own name grammar. @@ -246,21 +264,43 @@ func IntervalCron(interval, seed string) (string, error) { return fmt.Sprintf("%d %d * * 0", m, hour), nil } -// readPolicyScript prints "absent" or "path

\n". A file that exists -// and cannot be read is a failure, not an absence: a non-root CLI user reading -// "no policy" from a root-owned file would upload plaintext. -const readPolicyScript = `for p in /etc/teploy/backup-policy.json "$HOME"/.config/teploy/backup-policy.json; do +// policyReadMarker starts readPolicyScript. +const policyReadMarker = `if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]; then` + +// readPolicyScript prints "absent" or "path

\n", or exits 3 when it +// cannot tell. Root keeps /etc/teploy 0700, and to a non-root user a file in there +// does not even exist (`[ -e ]` is false): it read as "no policy", which cleared +// the recipient and uploaded plaintext on a server configured to encrypt. So an +// untraversable /etc/teploy is asked through passwordless sudo when that works +// (deploy users usually have it, cron included); a user with its own policy in +// ~/.config has made its choice; anyone else gets a refusal, never a guess. +const readPolicyScript = policyReadMarker + ` + if command -v sudo >/dev/null 2>&1 && s=$(sudo -n sh -c 'if [ -e /etc/teploy/backup-policy.json ]; then echo present; cat /etc/teploy/backup-policy.json; else echo absent; fi' 2>/dev/null); then + case "$s" in present*) printf 'path /etc/teploy/backup-policy.json\n%s\n' "${s#present}"; exit 0;; esac + elif [ ! -e "$HOME"/.config/teploy/backup-policy.json ]; then + echo "cannot tell whether this server's backups must be encrypted: /etc/teploy is not readable by $(id -un). Connect as root, give this user passwordless sudo, run on the server: chmod 711 /etc/teploy, or set this user's own policy (teploy server backup-policy set)" >&2 + exit 3 + fi +fi +for p in /etc/teploy/backup-policy.json "$HOME"/.config/teploy/backup-policy.json; do if [ -e "$p" ] && [ ! -r "$p" ]; then echo "cannot read $p: run as its owner" >&2; exit 3; fi if [ -r "$p" ]; then printf 'path %s\n' "$p"; cat "$p"; exit 0; fi done echo absent` +// ErrPolicyHidden means a policy may exist that this user cannot read, so whether +// backups must be encrypted is unknown: everything fails closed on it. +var ErrPolicyHidden = errors.New("the backup policy cannot be read") + // ReadFleetPolicy loads the server's policy. No policy is (FleetPolicy{}, nil). func ReadFleetPolicy(ctx context.Context, exec ssh.Executor) (FleetPolicy, error) { res := ssh.RunDetailed(ctx, exec, readPolicyScript) if res.Err != nil { return FleetPolicy{}, fmt.Errorf("reading the backup policy: %w", res.Err) } + if res.ExitCode == 3 { + return FleetPolicy{}, fmt.Errorf("%w: %s", ErrPolicyHidden, oneLine(res.ExitErrorText())) + } if res.ExitCode != 0 { return FleetPolicy{}, fmt.Errorf("reading the backup policy: %s", res.ExitErrorText()) } diff --git a/internal/backup/restore.go b/internal/backup/restore.go index 5bf7b2b..4a2a617 100644 --- a/internal/backup/restore.go +++ b/internal/backup/restore.go @@ -141,6 +141,11 @@ type RestoreOptions struct { Database string // NoPreBackup skips the automatic backup the CLI takes before a restore. NoPreBackup bool + // IdentityFile is a local age identity (--identity) the CLI places on the + // server for this restore. + IdentityFile string + // Deadline overrides the operation deadline (--deadline). + Deadline time.Duration } // AccessoryRestoreWith is AccessoryRestoreKeep with options. diff --git a/internal/backup/retention.go b/internal/backup/retention.go index b79e5e0..749b3cb 100644 --- a/internal/backup/retention.go +++ b/internal/backup/retention.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "regexp" + "slices" "sort" "strings" "time" @@ -34,6 +35,8 @@ type RetentionPolicy struct { KeepWeekly int KeepMonthly int MaxAgeDays int + // IgnoreClock lets a prune run despite a clock disagreement (--ignore-clock). + IgnoreClock bool } // IsZero reports whether the policy prunes nothing. @@ -192,10 +195,25 @@ type pruneObject struct { type pruneTree struct { root []pruneObject // primary objects series map[string][]pruneObject // databases//: per-database objects - sidecars []pruneObject // globals/, meta/, databases//meta/, teploy/ siblings - incoming []pruneObject // verification-pending uploads + sidecars []pruneObject // globals/, meta/, safety/, databases//meta/, teploy/ siblings + incoming []pruneObject // verification-pending uploads, at any depth + // safety are ids of safety copies a destructive operation took, by the time + // they were taken: protected for safetyWindow. + safety map[string]time.Time } +// safetyWindow is how long a safety copy outlasts every retention rule: long enough +// to notice that a restore or an upgrade went wrong, and go back. +const safetyWindow = 7 * 24 * time.Hour + +// safetyDir marks an id as a safety copy: /safety/.safety. +const safetyDir = "safety" + +// maxClockSkew is how far this machine's clock may be from the store's, and an +// object's date from now, before a prune refuses: selection is all arithmetic on +// those two numbers, and a wrong clock deletes real backups with a clean status. +const maxClockSkew = time.Hour + func idOf(file string) string { id, _, _ := strings.Cut(file, ".") return id @@ -204,7 +222,7 @@ func idOf(file string) string { // loadPruneTree lists the prefix across the current location and owned previous // ones. Anything it does not recognise is never touched. func (c *Client) loadPruneTree(ctx context.Context, app, prefix string, s3 S3Config) (pruneTree, error) { - tree := pruneTree{series: map[string][]pruneObject{}} + tree := pruneTree{series: map[string][]pruneObject{}, safety: map[string]time.Time{}} for _, loc := range s3.locations() { if loc.isPrevious && !loc.previousOwned { continue @@ -231,10 +249,13 @@ func (c *Client) loadPruneTree(ctx context.Context, app, prefix string, s3 S3Con e.bad = k.Size < artifactFloor(file) obj := pruneObject{key: "s3://" + loc.Bucket + "/" + k.Key, id: idOf(file), entry: e, loc: loc} switch { + case slices.Contains(segs, incomingDir): + tree.incoming = append(tree.incoming, obj) case len(segs) == 1: tree.root = append(tree.root, obj) - case segs[0] == incomingDir: - tree.incoming = append(tree.incoming, obj) + case len(segs) == 2 && segs[0] == safetyDir: + tree.safety[obj.id] = e.time + tree.sidecars = append(tree.sidecars, obj) case len(segs) == 3 && segs[0] == "databases": obj.entry.name = segs[1] + "/" + file tree.series[segs[1]] = append(tree.series[segs[1]], obj) @@ -257,7 +278,12 @@ func (c *Client) loadPruneTree(ctx context.Context, app, prefix string, s3 S3Con } segs := strings.Split(k.Key, "/") if e, ok := newBackupEntry(segs[len(segs)-1]); ok { - tree.sidecars = append(tree.sidecars, pruneObject{key: "s3://" + loc.Bucket + "/" + k.Key, id: idOf(segs[len(segs)-1]), entry: e, loc: loc}) + obj := pruneObject{key: "s3://" + loc.Bucket + "/" + k.Key, id: idOf(segs[len(segs)-1]), entry: e, loc: loc} + if slices.Contains(segs, incomingDir) { + tree.incoming = append(tree.incoming, obj) + } else { + tree.sidecars = append(tree.sidecars, obj) + } } } } @@ -275,49 +301,49 @@ func entriesOf(objs []pruneObject) []backupEntry { // planPrune decides what goes: // -// - With primary objects, the primary series is selected by the policy; each -// database's objects follow their id's fate, except that a database's newest -// good object always stays (a database whose dump has been failing keeps its -// last good copy) and ids newer than the newest primary stay (a killed run's -// extras are harmless until a newer primary supersedes them). -// - Without primary objects (adopted containers), each database is its own -// series, selected by the policy. +// - The primary series and each database's series are each selected by the +// policy on their own: a database failing in the last run of each day still +// gets its keep_daily, and a primary that disappeared and came back does not +// decide a database's history. +// - An id the primary series keeps is also kept in every database's series (a +// consistent point in time), as an extra keep and never a reason to delete. +// - A safety copy a destructive operation took outlasts every rule for +// safetyWindow. // - Sidecars go when no surviving object has their id and they are older than // the newest backup. func planPrune(tree pruneTree, p RetentionPolicy, now time.Time) (primaries, sidecars []pruneObject) { surviving := map[string]bool{} - byName := func(objs []pruneObject, del []string) []pruneObject { - names := map[string]bool{} - for _, n := range del { - names[n] = true - } + protected := func(id string) bool { + t, ok := tree.safety[id] + return ok && now.Sub(t) < safetyWindow + } + pick := func(objs []pruneObject, keepToo func(pruneObject) bool) []pruneObject { + del := selectForDeletion(entriesOf(objs), p, now) byEntry := map[string]pruneObject{} for _, o := range objs { - if names[o.entry.name] { - byEntry[o.entry.name] = o - } else { - surviving[o.id] = true - } + byEntry[o.entry.name] = o } + doomed := map[string]bool{} out := make([]pruneObject, 0, len(del)) for _, n := range del { // the selection's order: newest first - if o, ok := byEntry[n]; ok { - out = append(out, o) + o := byEntry[n] + if protected(o.id) || (keepToo != nil && keepToo(o)) { + continue + } + doomed[n] = true + out = append(out, o) + } + for _, o := range objs { + if !doomed[o.entry.name] { + surviving[o.id] = true } } return out } - rootEntries := entriesOf(tree.root) - primaries = byName(tree.root, selectForDeletion(rootEntries, p, now)) + primaries = pick(tree.root, nil) rootKept := map[string]bool{} - var newestRoot time.Time - for _, o := range tree.root { - if surviving[o.id] { - rootKept[o.id] = true - } - if !o.entry.bad && o.entry.time.After(newestRoot) { - newestRoot = o.entry.time - } + for id := range surviving { + rootKept[id] = true } dbs := make([]string, 0, len(tree.series)) for db := range tree.series { @@ -325,20 +351,7 @@ func planPrune(tree pruneTree, p RetentionPolicy, now time.Time) (primaries, sid } sort.Strings(dbs) for _, db := range dbs { - objs := tree.series[db] - if newestRoot.IsZero() { - primaries = append(primaries, byName(objs, selectForDeletion(entriesOf(objs), p, now))...) - continue - } - keep := selectKept(entriesOf(objs), RetentionPolicy{}, now) // the newest good one - var del []string - for _, o := range objs { - if rootKept[o.id] || keep[o.entry.name] || !o.entry.time.Before(newestRoot) { - continue - } - del = append(del, o.entry.name) - } - primaries = append(primaries, byName(objs, del)...) + primaries = append(primaries, pick(tree.series[db], func(o pruneObject) bool { return rootKept[o.id] })...) } var newest time.Time for _, objs := range append([][]pruneObject{tree.root}, seriesValues(tree.series)...) { @@ -370,27 +383,58 @@ func seriesValues(m map[string][]pruneObject) [][]pruneObject { return out } -// CleanStaleIncoming removes pending uploads a killed run left under a prefix. -// Prune does this too; this is for a target with no retention rule. Best effort. -func (c *Client) CleanStaleIncoming(ctx context.Context, app, prefix string, s3 S3Config) { - tree, err := c.loadPruneTree(ctx, app, prefix, s3) +// CleanStaleIncoming removes pending uploads a killed run left anywhere in a +// namespace, DR bundles included, whatever its retention. Best effort, and only in +// a namespace this server alone writes to. +func (c *Client) CleanStaleIncoming(ctx context.Context, app string, s3 S3Config) { + owners, err := c.namespaceOwners(ctx, s3, app) + fp := c.fingerprint(ctx) + if err != nil || fp == "" || slices.ContainsFunc(owners, func(o string) bool { return o != fp }) { + return + } + keys, err := c.listTree(ctx, s3, app+"/") if err != nil { return } now := time.Now().UTC() - for _, o := range tree.incoming { - if now.Sub(o.entry.time) > tmpLeftoverMinutes*time.Minute { - c.exec.Run(ctx, o.loc.AWS("s3 rm "+ssh.ShellQuote(o.key))) + for _, k := range keys { + segs := strings.Split(k.Key, "/") + if !slices.Contains(segs, incomingDir) { + continue + } + if e, ok := newBackupEntry(segs[len(segs)-1]); ok && now.Sub(e.time) > tmpLeftoverMinutes*time.Minute { + c.exec.Run(ctx, s3.AWS("s3 rm "+ssh.ShellQuote("s3://"+s3.Bucket+"/"+k.Key))) } } } +// ErrClockSkew marks a prune refused because dates cannot be trusted. +var ErrClockSkew = errors.New("clock disagreement") + +// checkPruneClock refuses arithmetic on dates that cannot be right: this machine's +// clock against the store's, and any backup dated in the future. +func checkPruneClock(tree pruneTree, storeNow, now time.Time) error { + if !storeNow.IsZero() { + if d := now.Sub(storeNow); d > maxClockSkew || d < -maxClockSkew { + return fmt.Errorf("%w: this machine's clock says %s and the store's says %s, so nothing was pruned (a wrong clock deletes real backups). Fix the clock, or pass --ignore-clock", ErrClockSkew, now.Format(time.RFC3339), storeNow.Format(time.RFC3339)) + } + } + for _, objs := range append([][]pruneObject{tree.root}, seriesValues(tree.series)...) { + for _, o := range objs { + if o.entry.time.Sub(now) > maxClockSkew { + return fmt.Errorf("%w: %s is dated in the future (now %s), and would count as the newest backup, so nothing was pruned. Remove it if it is wrong, or pass --ignore-clock", ErrClockSkew, o.key, now.Format(time.RFC3339)) + } + } + } + return nil +} + // PruneBackups deletes backups under s3:///// that fall // outside the retention policy, with every sidecar they keep beside them, -// returning the deleted backups' names. A zero policy is a no-op. A namespace -// another server also writes to is refused. A refused delete (bucket locks, -// keys without delete rights) does not stop the rest: errors are returned -// together with what was deleted. +// returning the deleted backups' names. A zero policy is a no-op. A namespace this +// server cannot prove is its alone is refused, and so is a clock that cannot be +// trusted. A refused delete (bucket locks, keys without delete rights) does not +// stop the rest: errors are returned together with what was deleted. func (c *Client) PruneBackups(ctx context.Context, app, prefix string, s3 S3Config, p RetentionPolicy) ([]string, error) { if p.IsZero() { return nil, nil @@ -398,17 +442,40 @@ func (c *Client) PruneBackups(ctx context.Context, app, prefix string, s3 S3Conf if err := c.ensureAWSCLI(ctx); err != nil { return nil, err } - if err := c.checkNamespaceOwner(ctx, s3, app); err != nil { + storeNow, err := c.claimNamespace(ctx, s3, app) + if err != nil { return nil, err } - tree, err := c.loadPruneTree(ctx, app, prefix, s3) + // A previous location is checked on its own: owned means this server's old + // objects, and a marker from another server there overrides that word. + var errs []error + cfg := s3 + cfg.PreviousPrefixes = nil + for _, prev := range s3.PreviousPrefixes { + if !prev.Owned { + continue + } + loc := s3 + loc.Prefix, loc.PreviousPrefixes = prev.Prefix, nil + if _, cerr := c.claimNamespace(ctx, loc, app); cerr != nil { + errs = append(errs, cerr) + continue + } + cfg.PreviousPrefixes = append(cfg.PreviousPrefixes, prev) + } + tree, err := c.loadPruneTree(ctx, app, prefix, cfg) if err != nil { return nil, err } - primaries, sidecars := planPrune(tree, p, time.Now().UTC()) + now := time.Now().UTC() + if !p.IgnoreClock { + if err := checkPruneClock(tree, storeNow, now); err != nil { + return nil, err + } + } + primaries, sidecars := planPrune(tree, p, now) var deleted []string - var errs []error for _, o := range primaries { if _, err := c.exec.Run(ctx, o.loc.AWS("s3 rm "+ssh.ShellQuote(o.key))); err != nil { errs = append(errs, fmt.Errorf("deleting %s: %w", o.key, err)) diff --git a/internal/backup/runlock_unix.go b/internal/backup/runlock_unix.go index 8aa49f2..281bff3 100644 --- a/internal/backup/runlock_unix.go +++ b/internal/backup/runlock_unix.go @@ -52,7 +52,13 @@ func AcquireRunLock(path string, now time.Time, staleAfter time.Duration) (*RunL f.Close() return nil, holder, ErrRunLocked } - // Alive, stuck and past every deadline it had: it will never let go. + // Alive, stuck and past every deadline it had: it will never let go. Its + // dumps and uploads run in process groups of their own, which would + // outlive it into the next run. + children := childPIDs(holder.PID) + for _, child := range children { + _ = syscall.Kill(-child, syscall.SIGKILL) + } if killErr := syscall.Kill(holder.PID, syscall.SIGKILL); killErr != nil { f.Close() return nil, holder, fmt.Errorf("%w (pid %d has held it since %s and could not be stopped: %v)", ErrRunLocked, holder.PID, holder.StartedAt.Format(time.RFC3339), killErr) @@ -94,6 +100,37 @@ func readRunLockHolder(path string) RunLockHolder { return h } +// childPIDs lists a process's direct children from /proc (none where there is no +// /proc). Every child of a run leads its own group (ssh.localCommand's Setpgid). +func childPIDs(pid int) []int { + entries, err := os.ReadDir("/proc") + if err != nil { + return nil + } + var out []int + for _, e := range entries { + p, err := strconv.Atoi(e.Name()) + if err != nil { + continue + } + b, err := os.ReadFile("/proc/" + e.Name() + "/stat") + if err != nil { + continue + } + // pid (comm) state ppid ...: comm may hold spaces, so cut after the last ). + s := string(b) + i := strings.LastIndex(s, ")") + if i < 0 { + continue + } + f := strings.Fields(s[i+1:]) + if len(f) >= 2 && f[1] == strconv.Itoa(pid) { + out = append(out, p) + } + } + return out +} + // isTeployProcess guards the kill against a recycled pid. func isTeployProcess(pid int) bool { b, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/cmdline") diff --git a/internal/backup/status.go b/internal/backup/status.go index e8e786e..38ece68 100644 --- a/internal/backup/status.go +++ b/internal/backup/status.go @@ -68,7 +68,13 @@ type StatusEntry struct { Coverage string `json:"coverage"` Reason string `json:"reason,omitempty"` // Schedule: policy | own (a per-app cron line) | none. - Schedule string `json:"schedule"` + Schedule string `json:"schedule"` + // ScheduleForm is the own line's form: run, or legacy (a pre-D3 shell line, + // which uploads plaintext whatever the policy says). + ScheduleForm string `json:"schedule_form,omitempty"` + // Encrypted says this entry's new objects will be encrypted: the server's + // setting, except for a legacy shell line. + Encrypted bool `json:"encrypted"` LastSuccess *time.Time `json:"last_success,omitempty"` LastFailure *time.Time `json:"last_failure,omitempty"` // LastError is the last failure's message (event-log bounded, no secrets). @@ -110,6 +116,8 @@ type StatusPolicy struct { HeartbeatHosts []string `json:"heartbeat_hosts,omitempty"` // Recipient is the owner's recovery PUBLIC key. Recipient string `json:"recipient,omitempty"` + // AllowEmpty says a run with zero targets is not a failure. + AllowEmpty bool `json:"allow_empty,omitempty"` } // StatusLastRun is the last `backup run --all`. @@ -119,8 +127,10 @@ type StatusLastRun struct { TargetsOK int `json:"targets_ok"` TargetsFailed int `json:"targets_failed"` Omissions []string `json:"omissions,omitempty"` - Error string `json:"error,omitempty"` - Stale bool `json:"stale"` + // Warnings include prunes that were refused: backups stored, retention stopped. + Warnings []string `json:"warnings,omitempty"` + Error string `json:"error,omitempty"` + Stale bool `json:"stale"` } // FleetStatus is the whole report. @@ -162,7 +172,7 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle st.Policy = StatusPolicy{ Configured: policy.Configured(), Path: policy.Path, Interval: policy.Interval, Retention: policy.Retention, Excludes: policy.Excludes, Adopted: policy.Adopted, Volumes: policy.Volumes, VolumeMaxGB: policy.VolumeMaxGB, - HeartbeatCount: len(policy.HeartbeatURLs), Recipient: policy.Recipient, + HeartbeatCount: len(policy.HeartbeatURLs), Recipient: policy.Recipient, AllowEmpty: policy.AllowEmpty, } for _, h := range MaskedHeartbeats(policy.HeartbeatURLs) { st.Policy.HeartbeatHosts = append(st.Policy.HeartbeatHosts, strings.TrimSuffix(h, "/...")) @@ -172,20 +182,23 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle Configured: target.Bucket != "", Bucket: target.Bucket, Region: target.Region, Endpoint: target.Endpoint, Prefix: target.Prefix, HasCredentials: target.HasCredentials, PreviousPrefixes: target.PreviousPrefixes, } - if out, err := exec.Run(ctx, "command -v aws >/dev/null 2>&1 && echo aws; command -v age >/dev/null 2>&1 && echo age; true"); err == nil { + if out, err := exec.Run(ctx, "command -v aws >/dev/null 2>&1 && aws --version >/dev/null 2>&1 && echo aws; command -v age >/dev/null 2>&1 && echo age; true"); err == nil { for _, t := range splitLines(out) { st.AWSCLI = st.AWSCLI || t == "aws" st.AgeCLI = st.AgeCLI || t == "age" } } - ownCron := map[string]bool{} + ownCron := map[string]string{} if cron, err := ReadCrontab(ctx, exec); err == nil { for _, line := range cron { if m := FindCronMarker(line); m != nil { if m.Kind == KindFleet { st.Policy.CronInstalled = true } else if m.Form != CronFormCustom { - ownCron[m.App+"/"+m.Accessory] = true + ownCron[m.App+"/"+m.Accessory] = m.Form + if m.Form == CronFormLegacy { + st.Warnings = append(st.Warnings, fmt.Sprintf("cron marker %s is a pre-D3 shell line that uploads plaintext whatever the policy says: rewrite it with `teploy server backup-target set`", m.Tag)) + } } } } @@ -206,7 +219,10 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle if region == "" { region = "us-east-1" } - cfg = S3Config{Bucket: target.Bucket, Region: region, Endpoint: target.Endpoint, Prefix: target.Prefix, StoredTarget: target.Path, PreviousPrefixes: target.PreviousPrefixes} + cfg = S3Config{Bucket: target.Bucket, Region: region, Endpoint: target.Endpoint, Prefix: target.Prefix, PreviousPrefixes: target.PreviousPrefixes} + if target.HasCredentials { + cfg.StoredTarget = target.Path + } } if target.Bucket != "" && st.AWSCLI { if err := c.ProbeTarget(ctx, cfg); err != nil { @@ -254,7 +270,14 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle } stale := 2 * policy.IntervalDuration() + // Covered means a restorable object exists now and is recent, not only that a + // run once said so: a wiped namespace, or a run that stored nothing, must not + // read as covered. decide := func(e *StatusEntry, excluded bool, covered bool) { + limit := stale + if e.Schedule == "own" || limit == 0 { + limit = 48 * time.Hour + } switch { case excluded: e.Coverage = CoverageExcluded @@ -264,16 +287,20 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle e.Coverage = CoverageFailing case e.LastSuccess == nil: e.Coverage, e.Reason = CoverageStale, "no backup has succeeded yet" + case now.Sub(*e.LastSuccess) > limit: + e.Coverage, e.Reason = CoverageStale, fmt.Sprintf("the last success is older than %s", limit) + case !st.Target.Reachable: + e.Coverage, e.Reason = CoverageStale, "the target did not answer, so whether a backup exists is unknown" + case e.Newest == nil: + e.Coverage, e.Reason = CoverageStale, "a run succeeded, but the bucket holds no restorable object for it" default: - limit := stale - if e.Schedule == "own" || limit == 0 { - limit = 48 * time.Hour - } - if now.Sub(*e.LastSuccess) > limit { - e.Coverage, e.Reason = CoverageStale, fmt.Sprintf("the last success is older than %s", limit) - } else if e.Adopted { + created, err := time.Parse(time.RFC3339, e.Newest.CreatedAt) + switch { + case err != nil || now.Sub(created) > limit: + e.Coverage, e.Reason = CoverageStale, fmt.Sprintf("the newest restorable object is older than %s", limit) + case e.Adopted: e.Coverage = CoverageAdopted - } else { + default: e.Coverage = CoverageCovered } } @@ -286,10 +313,14 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle } a, isAdopted := adopted[f.Name] isAccessory := f.Role == "accessory" && f.App != "" && f.Accessory != "" && f.Name == ContainerName(f.App, f.Accessory) - if !isAdopted && !isAccessory && (f.EngineErr != nil || !f.Engine.IsDB()) { + if !isAdopted && !isAccessory && !f.DBLike() { continue } - e := StatusEntry{Name: f.Name, Engine: f.Engine.Engine, Detection: f.Engine.Source, State: f.State, Schedule: "none", Unmanaged: f.Unmanaged(), Adopted: isAdopted} + e := StatusEntry{Name: f.Name, Engine: f.Engine.Engine, Detection: f.Engine.Source, State: f.State, Schedule: "none", Unmanaged: f.Unmanaged(), Adopted: isAdopted, Encrypted: st.Encrypted} + unsupported := f.UnsupportedDatabase() + if unsupported != "" { + e.Engine, e.Detection = unsupported, "image-name" + } var app, name string switch { case isAdopted: @@ -309,8 +340,9 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle switch { case isAdopted: covered, e.Schedule = policy.Configured(), "policy" - case isAccessory && ownCron[f.App+"/"+f.Accessory]: - covered, e.Schedule = true, "own" + case isAccessory && ownCron[f.App+"/"+f.Accessory] != "": + covered, e.Schedule, e.ScheduleForm = true, "own", ownCron[f.App+"/"+f.Accessory] + e.Encrypted = st.Encrypted && e.ScheduleForm != CronFormLegacy case isAccessory && policy.Configured() && policy.Interval != "": covered, e.Schedule = true, "policy" } @@ -320,7 +352,9 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle } decide(&e, excluded, covered) if e.Coverage == CoverageUncovered && e.Reason == "" { - if e.Kind == "unmanaged" { + if unsupported != "" { + e.Reason = "teploy has no backup method for " + unsupported + ": back it up another way and exclude it" + } else if e.Kind == "unmanaged" { e.Reason = "no policy entry covers this container (adopt it or exclude it)" } else { e.Reason = "no backup policy or schedule covers this accessory" @@ -348,7 +382,7 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle found = found || f.Name == a.Container } if !found { - e := StatusEntry{Name: a.Container, Kind: TargetAdopted, Accessory: a.Container, Engine: a.Engine, Adopted: true, Unmanaged: true, Schedule: "policy", + e := StatusEntry{Name: a.Container, Kind: TargetAdopted, Accessory: a.Container, Engine: a.Engine, Adopted: true, Unmanaged: true, Schedule: "policy", Encrypted: st.Encrypted, Coverage: CoverageFailing, Reason: "adopted container does not exist on this server", State: "missing"} fillEntryEvents(&e, logNS(AdoptedApp), a.Container) fillEntryObjects(&e, listNS(AdoptedApp), a.Container) @@ -359,13 +393,14 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle st.Warnings = append(st.Warnings, err.Error()) } else { for _, tr := range trees { - e := StatusEntry{Name: tr.App + "/volumes", Kind: TargetVolumes, App: tr.App, Schedule: "none", SizeBytes: tr.KB * 1024} + e := StatusEntry{Name: tr.App + "/volumes", Kind: TargetVolumes, App: tr.App, Schedule: "none", SizeBytes: tr.KB * 1024, Encrypted: st.Encrypted} guard := policy.VolumeMaxGB > 0 && float64(e.SizeBytes) > policy.VolumeMaxGB*(1<<30) excluded := policy.Excluded(tr.App, "volumes", "") || guard covered := false switch { - case ownCron[tr.App+"/"]: - covered, e.Schedule = true, "own" + case ownCron[tr.App+"/"] != "": + covered, e.Schedule, e.ScheduleForm = true, "own", ownCron[tr.App+"/"] + e.Encrypted = st.Encrypted && e.ScheduleForm != CronFormLegacy case policy.Volumes && policy.Interval != "": covered, e.Schedule = true, "policy" } @@ -384,19 +419,14 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle continue } run := &StatusLastRun{At: ev.TS, Status: detailString(ev.Detail, "status"), TargetsOK: detailInt(ev.Detail, "targets_ok"), TargetsFailed: detailInt(ev.Detail, "targets_failed"), Error: detailString(ev.Detail, "error")} - if list, ok := ev.Detail["omissions"].([]any); ok { - for _, o := range list { - if s, ok := o.(string); ok { - run.Omissions = append(run.Omissions, s) - } - } - } + run.Omissions = detailStrings(ev.Detail, "omissions") + run.Warnings = detailStrings(ev.Detail, "warnings") st.LastRun = run } if st.LastRun != nil && stale > 0 && now.Sub(st.LastRun.At) > stale { st.LastRun.Stale = true } - if out, err := exec.Run(ctx, LeftoverWorkspacesProbe()); err == nil { + if out, err := exec.Run(ctx, LeftoverWorkspacesProbe(policy.TmpDir)); err == nil { st.LeftoverWorkspaces = splitLines(out) } sort.Strings(st.UncoveredDatabases) @@ -428,7 +458,9 @@ func (c *Client) statusUncovered(ctx context.Context, app, name string, f *Conta } // fillEntryEvents sets the success, failure and verify facts from an event log. -// name is the accessory ("" for an app's volumes). Events are oldest first. +// name is the accessory ("" for an app's volumes). Events are oldest first. Only +// runs of what covers this entry count: a pre-restore safety backup of one database +// or a manual one must not mask a failing --all. func fillEntryEvents(e *StatusEntry, evs []events.Event, name string) { for _, ev := range evs { acc := detailString(ev.Detail, "accessory") @@ -440,6 +472,12 @@ func fillEntryEvents(e *StatusEntry, evs []events.Event, name string) { if name == "" && detailString(ev.Detail, "kind") != KindVolumes { continue } + if detailString(ev.Detail, "trigger") != "" { + continue + } + if managed, _ := ev.Detail["policy_managed"].(bool); e.Schedule == "policy" && !managed { + continue + } ts := ev.TS switch ev.Outcome { case events.Succeeded: @@ -484,9 +522,10 @@ func fillEntryObjects(e *StatusEntry, rows []ListedBackup, name string) { } } sort.Slice(e.Databases, func(i, j int) bool { return e.Databases[i].Name < e.Databases[j].Name }) - // Without any event yet, a good object is still evidence of a success: the - // history may predate the event log. - if e.LastSuccess == nil && e.Newest != nil { + // With no event either way, a good object is still evidence of a success: the + // history may predate the event log. Never next to a recorded failure: a safety + // copy newer than a failing run would hide it. + if e.LastSuccess == nil && e.LastFailure == nil && e.Newest != nil { if t, err := time.Parse(time.RFC3339, e.Newest.CreatedAt); err == nil { e.LastSuccess = &t } @@ -500,6 +539,18 @@ func detailString(d map[string]any, k string) string { return "" } +func detailStrings(d map[string]any, k string) []string { + var out []string + if list, ok := d[k].([]any); ok { + for _, o := range list { + if s, ok := o.(string); ok { + out = append(out, s) + } + } + } + return out +} + func detailInt(d map[string]any, k string) int { if n, ok := d[k].(float64); ok { return int(n) diff --git a/internal/backup/store.go b/internal/backup/store.go index 222cb3a..4d69c61 100644 --- a/internal/backup/store.go +++ b/internal/backup/store.go @@ -2,13 +2,15 @@ package backup import ( "context" + "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" "net/url" "path" "regexp" - "sort" + "slices" "strconv" "strings" "time" @@ -44,8 +46,10 @@ const ageSuffix = ".age" const incomingDir = ".incoming" // readBackMax is the largest object whose bytes are read back and hashed after -// upload; above it the stored size and the checksum recorded at upload stand -// in (plus the aws CLI's own transfer checksums). Bandwidth, not safety. +// upload. Above it the object is uploaded with a provider-verified checksum +// (--checksum-algorithm SHA256: the store checks every part on receipt) and that +// is read back instead: reading a large backup back costs bandwidth, and our own +// metadata proves only what we sent. const readBackMax = 256 << 20 // agelessFloorOverhead is what age adds to a tiny object (header with two @@ -133,7 +137,6 @@ func (c *Client) encryptLocal(ctx context.Context, s3 S3Config, local string) (s } q := ssh.ShellQuote enc := local + ageSuffix - id := ageIdentityShell cmd := fmt.Sprintf("age -r %s -r %s -o %s %s", q(s3.OwnerRecipient), q(server), q(enc), q(local)) if _, err := c.exec.Run(ctx, cmd); err != nil { c.exec.Run(context.WithoutCancel(ctx), "rm -f "+q(enc)) @@ -142,7 +145,7 @@ func (c *Client) encryptLocal(ctx context.Context, s3 S3Config, local string) (s // cmp reads the decrypted stream to its end and compares it with the // plaintext: a decrypt failure shows up as a short stream, so losing age's // exit status in this pipe cannot hide anything. - check := fmt.Sprintf(`age -d -i "%s" %s | cmp -s - %s`, id, q(enc), q(local)) + check := fmt.Sprintf(`age -d %s %s | cmp -s - %s`, ageIdentityArgs, q(enc), q(local)) if _, err := c.exec.Run(ctx, check); err != nil { c.exec.Run(context.WithoutCancel(ctx), "rm -f "+q(enc)) return "", fmt.Errorf("the encrypted backup did not decrypt back to its plaintext with the server's identity (nothing was uploaded): %w", err) @@ -153,9 +156,11 @@ func (c *Client) encryptLocal(ctx context.Context, s3 S3Config, local string) (s return enc, nil } -// ageIdentityShell expands to the server's identity path (same try-order as -// ensureAgeIdentity). It is a command substitution, for use inside "...". -const ageIdentityShell = `$(if [ -r /etc/teploy/age-key ]; then printf '%s' /etc/teploy/age-key; else printf '%s' /deployments/.age-key; fi)` +// ageIdentityArgs expands to "-i " for the server's identity, same +// try-order as ensureAgeIdentity, and to nothing on a host that has none (or only +// the public-key marker left behind when the key moved). age aborts on a missing +// -i file, so a clean host recovering with --identity must not pass one. +const ageIdentityArgs = `$(for p in /etc/teploy/age-key /deployments/.age-key; do if [ -r "$p" ] && grep -q '^AGE-SECRET-KEY-' "$p" 2>/dev/null; then printf -- '-i %s' "$p"; break; fi; done)` // localFacts measures a file: size and SHA-256, in one command. func (c *Client) localFacts(ctx context.Context, local string) (int64, string, error) { @@ -185,26 +190,73 @@ func incomingKey(key string) string { type objectHead struct { Size int64 SHA256 string + // ProviderSHA256 is the store's own checksum (base64), when it kept one and + // was asked: "xxx-N" for a multipart object, whose parts it checked on receipt. + ProviderSHA256 string + // LastModified is the store's clock at the object's last write. + LastModified time.Time } // head reads an object's size and recorded checksum. func (c *Client) head(ctx context.Context, s3 S3Config, key string) (objectHead, error) { + return c.headArgs(ctx, s3, key, "") +} + +func (c *Client) headArgs(ctx context.Context, s3 S3Config, key, extra string) (objectHead, error) { bucket, k, err := splitS3URI(key) if err != nil { return objectHead{}, err } - res := ssh.RunDetailed(ctx, c.exec, s3.AWS("s3api head-object --bucket "+ssh.ShellQuote(bucket)+" --key "+ssh.ShellQuote(k))) + res := ssh.RunDetailed(ctx, c.exec, s3.AWS("s3api head-object --bucket "+ssh.ShellQuote(bucket)+" --key "+ssh.ShellQuote(k)+extra)) if res.Failed() { return objectHead{}, fmt.Errorf("reading %s: %s", key, oneLine(res.ExitErrorText())) } var h struct { - ContentLength int64 `json:"ContentLength"` - Metadata map[string]string `json:"Metadata"` + ContentLength int64 `json:"ContentLength"` + Metadata map[string]string `json:"Metadata"` + ChecksumSHA256 string `json:"ChecksumSHA256"` + LastModified string `json:"LastModified"` } if err := json.Unmarshal(res.Stdout, &h); err != nil { return objectHead{}, fmt.Errorf("reading %s: unexpected reply: %w", key, err) } - return objectHead{Size: h.ContentLength, SHA256: h.Metadata["sha256"]}, nil + out := objectHead{Size: h.ContentLength, SHA256: h.Metadata["sha256"], ProviderSHA256: h.ChecksumSHA256} + // aws CLI v2 prints RFC 3339, v1 an HTTP date. + for _, layout := range []string{time.RFC3339, time.RFC1123, "Mon, 02 Jan 2006 15:04:05 GMT"} { + if t, err := time.Parse(layout, h.LastModified); err == nil { + out.LastModified = t.UTC() + break + } + } + return out, nil +} + +func (c *Client) readBackLimit() int64 { + if c.readBack > 0 { + return c.readBack + } + return readBackMax +} + +// sha256HexToBase64 converts a sha256sum digest to the form a provider reports. +func sha256HexToBase64(h string) string { + b, err := hex.DecodeString(h) + if err != nil { + return "" + } + return base64.StdEncoding.EncodeToString(b) +} + +// checksumUnsupported recognises a CLI or provider that cannot take +// --checksum-algorithm: an old aws CLI, a store without additional checksums. +func checksumUnsupported(err error) bool { + msg := strings.ToLower(err.Error()) + for _, s := range []string{"unknown options", "notimplemented", "not implemented", "invalidargument", "invalidrequest", "unsupported", "x-amz-checksum", "x-amz-sdk-checksum"} { + if strings.Contains(msg, s) { + return true + } + } + return false } // splitS3URI splits s3://bucket/key. @@ -250,8 +302,22 @@ func (c *Client) put(ctx context.Context, s3 S3Config, local, key string) (store } pending := incomingKey(final) q := ssh.ShellQuote - if _, err := c.exec.Run(ctx, s3.AWS(fmt.Sprintf("s3 cp %s %s --metadata %s", q(src), q(pending), q("sha256="+sum)))); err != nil { - return none, fmt.Errorf("uploading: %w", err) + upload := fmt.Sprintf("s3 cp %s %s --metadata %s", q(src), q(pending), q("sha256="+sum)) + large := size > c.readBackLimit() + providerChecked := large + var uerr error + if large { + _, uerr = c.exec.Run(ctx, s3.AWS(upload+" --checksum-algorithm SHA256")) + if uerr != nil && checksumUnsupported(uerr) { + providerChecked = false + fmt.Fprintf(c.out, "Warning: this store or aws CLI has no upload checksums, so %s is verified by size alone: %v\n", path.Base(final), oneLine(uerr.Error())) + _, uerr = c.exec.Run(ctx, s3.AWS(upload)) + } + } else { + _, uerr = c.exec.Run(ctx, s3.AWS(upload)) + } + if uerr != nil { + return none, fmt.Errorf("uploading: %w", uerr) } removePending := func() { rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) @@ -275,7 +341,22 @@ func (c *Client) put(ctx context.Context, s3 S3Config, local, key string) (store removePending() return none, fmt.Errorf("verifying the upload: %w", err) } - if size <= readBackMax { + if providerChecked { + // The store checked every part on receipt; a single-part object's digest + // can be compared whole, and a store that kept none has proved nothing. + h, err := c.headArgs(ctx, s3, pending, " --checksum-mode ENABLED") + if err != nil { + removePending() + return none, fmt.Errorf("verifying the upload: %w", err) + } + if h.ProviderSHA256 != "" && !strings.Contains(h.ProviderSHA256, "-") && h.ProviderSHA256 != sha256HexToBase64(sum) { + removePending() + return none, fmt.Errorf("the store's checksum of the upload does not match what was sent (checksum mismatch)") + } + if h.ProviderSHA256 == "" { + fmt.Fprintf(c.out, "Warning: the store keeps no checksum for %s; it is verified by size alone\n", path.Base(final)) + } + } else if !large { // A pipe, but compared by value: anything short of the exact bytes // (a failed read included) produces a different hash. out, err := c.exec.Run(ctx, "{ "+s3.AWS("s3 cp "+q(pending)+" -")+"; } | sha256sum") @@ -342,7 +423,7 @@ func (c *Client) fetch(ctx context.Context, s3 S3Config, key, dest string) error if !encrypted { return nil } - ids := `-i "` + ageIdentityShell + `"` + ids := ageIdentityArgs if s3.ExtraIdentity != "" { ids += " -i " + q(s3.ExtraIdentity) } @@ -351,7 +432,7 @@ func (c *Client) fetch(ctx context.Context, s3 S3Config, key, dest string) error if errors.Is(ctx.Err(), context.Canceled) { return ctx.Err() } - return fmt.Errorf("decrypting %s with this server's identity failed (encrypted to other recipients, or corrupt; on another machine use `teploy backup fetch --identity`): %w", key, err) + return fmt.Errorf("decrypting %s failed: it is corrupt, or encrypted to other recipients than this server's identity (pass --identity , or `teploy backup fetch --identity` on another machine): %w", key, err) } c.exec.Run(ctx, "rm -f "+q(raw)) return nil @@ -417,12 +498,13 @@ func decodeDBSegment(seg string) string { // addressableDB reports whether a database name can be handed to the engine // tools as one literal shell-quoted argument. pg_dump and psql read a name with -// "=" as a connection string, and every tool reads a leading "-" as an option; +// "=" or a URI scheme as a connection string, and every tool reads a leading "-" +// as an option; // control characters cannot survive line-based listings. Everything else, // including quotes, spaces and $(...), is safe once shell-quoted. A name that // fails this is reported as not backed up, never skipped silently. func addressableDB(db string) bool { - if db == "" || len(db) > 64 || db[0] == '-' || strings.Contains(db, "=") { + if db == "" || len(db) > 64 || db[0] == '-' || strings.Contains(db, "=") || strings.Contains(db, "://") { return false } for _, r := range db { @@ -573,64 +655,90 @@ func artifactFloor(name string) int64 { // any one target, shorter than a disk can afford to wait. const tmpLeftoverMinutes = 420 -// workspaceGlobs are what the run cleans up: private run workspaces a killed -// backup can leave (dumps, possibly a credential file), and a recovery key a -// killed `dr restore --identity` left on this server. Only ever this: a killed -// restore's own files can be the only copy of something. -const workspaceGlobs = "/var/tmp/teploy-backup.* /tmp/teploy-backup.* /var/tmp/teploy-dr-identity.*" +// tmpBases are the directories backup workspaces can be under: disk, /tmp from +// older runs, and a configured TEPLOY_BACKUP_TMPDIR or policy tmp_dir, which is +// where a killed run's plaintext dump would otherwise sit forever unnoticed. +func tmpBases(extra string) []string { + bases := []string{"/var/tmp", "/tmp"} + for _, d := range []string{backupTmpDir(), extra} { + d = strings.TrimRight(strings.TrimSpace(d), "/") + if d != "" && !slices.Contains(bases, d) { + bases = append(bases, d) + } + } + return bases +} -// leftoverGlobs are what doctor and status report: the above, plus a kept or -// killed restore's and verify's scratch files. -const leftoverGlobs = workspaceGlobs + " /var/tmp/teploy-restore.* /tmp/teploy-restore.* /tmp/teploy-volume-restore.* /var/tmp/teploy-verify.*" +func tmpGlobs(extra string, names ...string) string { + var out []string + for _, base := range tmpBases(extra) { + for _, n := range names { + out = append(out, ssh.ShellQuote(base)+"/"+n) + } + } + return strings.Join(out, " ") +} -// CleanStaleWorkspaces removes run workspaces older than the leftover age and -// returns what it removed. Only called while holding the run lock, so a live -// `backup run --all` cannot be hit; the age protects a concurrent manual one. +// CleanStaleWorkspaces removes run workspaces (dumps, possibly a credential file) +// and any recovery key a killed `restore --identity` left, once they are older +// than the leftover age, and returns what it removed. Only called while holding +// the run lock, so a live `backup run --all` cannot be hit; the age protects a +// concurrent manual one. Never a killed restore's own files: those can be the only +// copy of something. func (c *Client) CleanStaleWorkspaces(ctx context.Context) []string { - out, err := c.exec.Run(ctx, fmt.Sprintf(`for d in %s; do [ -d "$d" ] || continue; if [ -n "$(find "$d" -maxdepth 0 -mmin +%d 2>/dev/null)" ]; then rm -rf -- "$d" && printf '%%s\n' "$d"; fi; done; true`, workspaceGlobs, tmpLeftoverMinutes)) + out, err := c.exec.Run(ctx, fmt.Sprintf(`for d in %s; do [ -d "$d" ] || continue; if [ -n "$(find "$d" -maxdepth 0 -mmin +%d 2>/dev/null)" ]; then rm -rf -- "$d" && printf '%%s\n' "$d"; fi; done; true`, tmpGlobs("", "teploy-backup.*", "teploy-dr-identity.*"), tmpLeftoverMinutes)) if err != nil { return nil } return splitLines(out) } -// LeftoverWorkspacesProbe lists workspaces past the leftover age, for doctor. -func LeftoverWorkspacesProbe() string { - return fmt.Sprintf(`for d in %s; do [ -d "$d" ] || continue; if [ -n "$(find "$d" -maxdepth 0 -mmin +%d 2>/dev/null)" ]; then printf '%%s\n' "$d"; fi; done; docker ps -a --filter name=teploy-backup-verify- --format 'container {{.Names}}' 2>/dev/null; true`, leftoverGlobs, tmpLeftoverMinutes) +// LeftoverWorkspacesProbe lists what doctor and status report: the above plus a +// kept or killed restore's and verify's scratch files and containers. +func LeftoverWorkspacesProbe(tmpDir string) string { + globs := tmpGlobs(tmpDir, "teploy-backup.*", "teploy-dr-identity.*", "teploy-restore.*", "teploy-volume-restore.*", "teploy-verify.*") + return fmt.Sprintf(`for d in %s; do [ -d "$d" ] || continue; if [ -n "$(find "$d" -maxdepth 0 -mmin +%d 2>/dev/null)" ]; then printf '%%s\n' "$d"; fi; done; docker ps -a --filter name=teploy-backup-verify- --format 'container {{.Names}}' 2>/dev/null; true`, globs, tmpLeftoverMinutes) } -// AbortStaleUploads aborts multipart uploads a killed run left under the base -// (they are invisible in listings and bill storage). Best effort; returns how -// many were aborted. -func (c *Client) AbortStaleUploads(ctx context.Context, s3 S3Config, now time.Time) int { +// AbortStaleUploads aborts multipart uploads a killed run left under the given +// namespaces (invisible in listings, and they bill storage). Scoped to namespaces +// only this server writes to: with no prefix a bucket-wide list would catch other +// servers' uploads. Best effort; returns how many were aborted. +func (c *Client) AbortStaleUploads(ctx context.Context, s3 S3Config, namespaces []string, now time.Time) int { q := ssh.ShellQuote - args := "s3api list-multipart-uploads --bucket " + q(s3.Bucket) + " --output json" - if s3.Prefix != "" { - args += " --prefix " + q(s3.Prefix+"/") - } - res := ssh.RunDetailed(ctx, c.exec, s3.AWS(args)) - if res.Failed() { - return 0 - } - var list struct { - Uploads []struct { - Key string `json:"Key"` - UploadID string `json:"UploadId"` - Initiated string `json:"Initiated"` - } `json:"Uploads"` - } - if json.Unmarshal(res.Stdout, &list) != nil { - return 0 - } - sort.Slice(list.Uploads, func(i, j int) bool { return list.Uploads[i].Key < list.Uploads[j].Key }) aborted := 0 - for _, u := range list.Uploads { - t, err := time.Parse(time.RFC3339, u.Initiated) - if err != nil || now.Sub(t) < tmpLeftoverMinutes*time.Minute { + for _, ns := range namespaces { + owners, err := c.namespaceOwners(ctx, s3, ns) + fp := c.fingerprint(ctx) + if err != nil || fp == "" || slices.ContainsFunc(owners, func(o string) bool { return o != fp }) { + continue + } + prefix := ns + "/" + if s3.Prefix != "" { + prefix = s3.Prefix + "/" + prefix + } + res := ssh.RunDetailed(ctx, c.exec, s3.AWS("s3api list-multipart-uploads --bucket "+q(s3.Bucket)+" --prefix "+q(prefix)+" --output json")) + if res.Failed() { continue } - if _, err := c.exec.Run(ctx, s3.AWS("s3api abort-multipart-upload --bucket "+q(s3.Bucket)+" --key "+q(u.Key)+" --upload-id "+q(u.UploadID))); err == nil { - aborted++ + var list struct { + Uploads []struct { + Key string `json:"Key"` + UploadID string `json:"UploadId"` + Initiated string `json:"Initiated"` + } `json:"Uploads"` + } + if json.Unmarshal(res.Stdout, &list) != nil { + continue + } + for _, u := range list.Uploads { + t, err := time.Parse(time.RFC3339, u.Initiated) + if err != nil || now.Sub(t) < tmpLeftoverMinutes*time.Minute { + continue + } + if _, err := c.exec.Run(ctx, s3.AWS("s3api abort-multipart-upload --bucket "+q(s3.Bucket)+" --key "+q(u.Key)+" --upload-id "+q(u.UploadID))); err == nil { + aborted++ + } } } return aborted @@ -639,88 +747,215 @@ func (c *Client) AbortStaleUploads(ctx context.Context, s3 S3Config, now time.Ti // ---- namespace ownership ---- // ownersDir holds one marker per server that has written into an app namespace: -// //.owners/. Prune refuses a namespace with another -// server's marker, so a shared bucket or a reused prefix can never make one -// server delete another's backups. +// //.owners/, a JSON body naming the server's age public +// key. Prune refuses a namespace with another server's marker, so a shared bucket +// or a reused prefix can never make one server delete another's backups. Anything +// short of proof of sole ownership refuses: a listing that fails is not "no owners". +// +// The fingerprint is a hash of /etc/machine-id and the hostname (an empty +// machine-id, or a clone's copy of one, is not enough by itself), and the body's +// recipient tells apart two hosts that share both. It is in the body and not the +// name so that an identity created later does not make a server its own stranger. +// +// Residual: an upgraded server's first prune cannot tell its own unmarked legacy +// objects from those of a server that has not upgraded yet (AUDIT_OPEN). const ownersDir = ".owners" var ownerMarkerRE = regexp.MustCompile(`^[0-9a-f]{16}$`) -// fingerprint identifies this server: a hash of its machine id, else its age -// recipient. Empty when neither can be read. +// ownerMarker is a marker's body. +type ownerMarker struct { + Recipient string `json:"recipient,omitempty"` + FirstSeen string `json:"first_seen,omitempty"` +} + +// fingerprint identifies this server. Empty when it cannot be worked out. func (c *Client) fingerprint(ctx context.Context) string { if c.serverFP != "" { return c.serverFP } - out, err := c.exec.Run(ctx, `{ cat /etc/machine-id 2>/dev/null || hostname; } | sha256sum`) + out, err := c.exec.Run(ctx, `{ cat /etc/machine-id 2>/dev/null; hostname 2>/dev/null; } | sha256sum`) if err != nil { return "" } f := strings.Fields(out) - if len(f) == 0 || len(f[0]) < 16 { + if len(f) == 0 || len(f[0]) < 16 || !ownerMarkerRE.MatchString(f[0][:16]) { return "" } - if !ownerMarkerRE.MatchString(f[0][:16]) { + // sha256 of empty input: neither source answered, and every such host agrees. + if strings.HasPrefix(f[0], "e3b0c44298fc1c14") { return "" } c.serverFP = f[0][:16] return c.serverFP } +// localRecipient is this server's age public key, "" when it has no identity. +func (c *Client) localRecipient(ctx context.Context) string { + out, err := c.exec.Run(ctx, `for p in /etc/teploy/age-key /deployments/.age-key; do if [ -r "$p" ] && grep -q '^AGE-SECRET-KEY-' "$p" 2>/dev/null; then age-keygen -y "$p" 2>/dev/null; break; fi; done; true`) + if err != nil { + return "" + } + if r := strings.TrimSpace(out); recipientRE.MatchString(r) { + return r + } + return "" +} + func namespaceOwnersPrefix(s3 S3Config, app string) string { return fmt.Sprintf("s3://%s/%s/%s/", s3.Base(), app, ownersDir) } -// namespaceOwners lists the servers that have marked this namespace. -func (c *Client) namespaceOwners(ctx context.Context, s3 S3Config, app string) []string { - out, err := c.exec.Run(ctx, s3.AWS("s3 ls "+ssh.ShellQuote(namespaceOwnersPrefix(s3, app)))) +// namespaceOwners lists the servers that have marked this namespace. An empty +// namespace is no owners; a listing that fails is an error, never "no owners". +func (c *Client) namespaceOwners(ctx context.Context, s3 S3Config, app string) ([]string, error) { + keys, err := c.listTree(ctx, s3, app+"/"+ownersDir+"/") if err != nil { - return nil + return nil, err } var owners []string - for _, line := range strings.Split(out, "\n") { - f := strings.Fields(line) - if len(f) > 0 && ownerMarkerRE.MatchString(f[len(f)-1]) { - owners = append(owners, f[len(f)-1]) + for _, k := range keys { + name := path.Base(k.Key) + if ownerMarkerRE.MatchString(name) { + owners = append(owners, name) } } - return owners + return owners, nil +} + +func (c *Client) readOwnerMarker(ctx context.Context, s3 S3Config, key string) ownerMarker { + var m ownerMarker + if res := ssh.RunDetailed(ctx, c.exec, s3.AWS("s3 cp "+ssh.ShellQuote(key)+" -")); !res.Failed() { + _ = json.Unmarshal(res.Stdout, &m) + } + return m +} + +func (c *Client) writeOwnerMarker(ctx context.Context, s3 S3Config, key string, m ownerMarker) error { + body, err := json.Marshal(m) + if err != nil { + return err + } + _, err = c.exec.Run(ctx, "printf '%s\\n' "+ssh.ShellQuote(string(body))+" | "+s3.AWS("s3 cp - "+ssh.ShellQuote(key))) + return err } // markNamespaceOwner records this server as a writer of the namespace. Best -// effort; called after a successful backup. +// effort, after a successful backup; a prune claims the namespace for itself. func (c *Client) markNamespaceOwner(ctx context.Context, s3 S3Config, app string) { fp := c.fingerprint(ctx) if fp == "" { return } - for _, o := range c.namespaceOwners(ctx, s3, app) { - if o == fp { - return - } + owners, err := c.namespaceOwners(ctx, s3, app) + if err != nil || slices.Contains(owners, fp) { + return } - key := namespaceOwnersPrefix(s3, app) + fp - c.exec.Run(ctx, "printf '%s\\n' "+ssh.ShellQuote(time.Now().UTC().Format(time.RFC3339))+" | "+s3.AWS("s3 cp - "+ssh.ShellQuote(key))) + _ = c.writeOwnerMarker(ctx, s3, namespaceOwnersPrefix(s3, app)+fp, ownerMarker{Recipient: c.localRecipient(ctx), FirstSeen: time.Now().UTC().Format(time.RFC3339)}) } -// ErrSharedNamespace marks a prune refused because another server writes there. +// ErrSharedNamespace marks a prune refused because this server cannot show it is +// the only one writing there. var ErrSharedNamespace = errors.New("shared backup namespace") -// checkNamespaceOwner refuses when another server has marked this namespace. -func (c *Client) checkNamespaceOwner(ctx context.Context, s3 S3Config, app string) error { - owners := c.namespaceOwners(ctx, s3, app) - if len(owners) == 0 { - return nil +// claimNamespace proves this server may prune the namespace, marks it (a prune +// never runs on an unmarked namespace), and returns the store's clock as read from +// the marker it just wrote: the one reference a wrong server clock cannot fool. +func (c *Client) claimNamespace(ctx context.Context, s3 S3Config, app string) (time.Time, error) { + var none time.Time + base := "s3://" + s3.Base() + owners, err := c.namespaceOwners(ctx, s3, app) + if err != nil { + return none, fmt.Errorf("%w: could not list the owners of %s/%s, so nothing was pruned: %v", ErrSharedNamespace, base, app, err) } fp := c.fingerprint(ctx) + if fp == "" { + return none, fmt.Errorf("%w: this server cannot identify itself (no machine-id or hostname), so nothing was pruned", ErrSharedNamespace) + } var others []string for _, o := range owners { if o != fp { others = append(others, o) } } - if len(others) == 0 { - return nil + if len(others) > 0 { + return none, fmt.Errorf("%w: another server (%s) also stores %s backups under %s, so pruning here could delete its copies; nothing was pruned. Give each server its own prefix (`teploy server backup-target set --prefix `). If that marker is this server before a rebuild: `teploy server backup-target owners --clear %s`", ErrSharedNamespace, strings.Join(others, ", "), app, base, strings.Join(others, ",")) + } + key := namespaceOwnersPrefix(s3, app) + fp + self := c.localRecipient(ctx) + m := ownerMarker{Recipient: self, FirstSeen: time.Now().UTC().Format(time.RFC3339)} + if slices.Contains(owners, fp) { + old := c.readOwnerMarker(ctx, s3, key) + if old.FirstSeen != "" { + m.FirstSeen = old.FirstSeen + } + if old.Recipient != "" && self != "" && old.Recipient != self { + return none, fmt.Errorf("%w: the marker for this machine under %s/%s names another age identity (%s), so this is a clone or a rebuilt server; nothing was pruned. If it is this server's own old marker: `teploy server backup-target owners --clear %s`", ErrSharedNamespace, base, app, old.Recipient, fp) + } + if self == "" { + m.Recipient = old.Recipient + } + } + if err := c.writeOwnerMarker(ctx, s3, key, m); err != nil { + return none, fmt.Errorf("%w: could not mark %s/%s as this server's before pruning, so nothing was pruned: %v", ErrSharedNamespace, base, app, err) + } + h, err := c.head(ctx, s3, key) + if err != nil { + return none, nil // no clock reading; the marker is what mattered + } + return h.LastModified, nil +} + +// NamespaceMarker is one server's marker in one namespace. +type NamespaceMarker struct { + Namespace string `json:"namespace"` + Fingerprint string `json:"fingerprint"` + Key string `json:"key"` + Self bool `json:"self"` +} + +// ListNamespaceMarkers lists every owner marker under the base. +func (c *Client) ListNamespaceMarkers(ctx context.Context, s3 S3Config) ([]NamespaceMarker, error) { + keys, err := c.listTree(ctx, s3, "") + if err != nil { + return nil, err + } + fp := c.fingerprint(ctx) + var out []NamespaceMarker + for _, k := range keys { + rel, ok := s3.relToBase(k.Key) + if !ok { + continue + } + segs := strings.Split(rel, "/") + if len(segs) == 3 && segs[1] == ownersDir && ownerMarkerRE.MatchString(segs[2]) { + out = append(out, NamespaceMarker{Namespace: segs[0], Fingerprint: segs[2], Key: "s3://" + s3.Bucket + "/" + k.Key, Self: segs[2] == fp}) + } + } + return out, nil +} + +// ClearNamespaceMarkers removes a fingerprint's markers (a server that no longer +// exists, a rebuild's old self), returning the keys removed. +func (c *Client) ClearNamespaceMarkers(ctx context.Context, s3 S3Config, fingerprint string) ([]string, error) { + if !ownerMarkerRE.MatchString(fingerprint) { + return nil, fmt.Errorf("invalid fingerprint %q", fingerprint) + } + markers, err := c.ListNamespaceMarkers(ctx, s3) + if err != nil { + return nil, err + } + var removed []string + var errs []error + for _, m := range markers { + if m.Fingerprint != fingerprint { + continue + } + if _, err := c.exec.Run(ctx, s3.AWS("s3 rm "+ssh.ShellQuote(m.Key))); err != nil { + errs = append(errs, err) + continue + } + removed = append(removed, m.Key) } - return fmt.Errorf("%w: another server (%s) also stores %s backups under s3://%s, so pruning here could delete its copies; nothing was pruned. Give each server its own prefix (`teploy server backup-target set --prefix `)", ErrSharedNamespace, strings.Join(others, ", "), app, s3.Base()) + return removed, errors.Join(errs...) } diff --git a/internal/backup/store_test.go b/internal/backup/store_test.go index 395bc7a..f3eaf21 100644 --- a/internal/backup/store_test.go +++ b/internal/backup/store_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "testing" "time" @@ -267,6 +268,7 @@ func TestPruneToleratesRefusedDeletes(t *testing.T) { func putMock(extra ...ssh.MockCommand) *ssh.MockExecutor { base := []ssh.MockCommand{ {Match: "sha256sum", Output: "abc f\n4096\n"}, + {Match: "{ cat /etc/machine-id", Output: "0123456789abcdef0123456789abcdef -\n"}, {Match: "{ ", Output: "abc -\n"}, {Match: "aws s3api head-object", Output: `{"ContentLength":4096,"Metadata":{"sha256":"abc"}}`}, {Match: "aws s3 cp", Output: "ok\n"}, @@ -400,7 +402,7 @@ func TestFetchDecryptsWithServerIdentityAndNamesTheWayOut(t *testing.T) { } dec := false for _, c := range mock.Calls { - dec = dec || strings.HasPrefix(c, "umask 077; age -d -i \"$(if [ -r /etc/teploy/age-key ]") + dec = dec || strings.HasPrefix(c, "umask 077; age -d $(for p in /etc/teploy/age-key") } if !dec { t.Fatalf("no decrypt: %v", mock.Calls) @@ -752,3 +754,293 @@ func TestBundleDownloadFindsEncryptedAndLegacyMembers(t *testing.T) { t.Fatalf("a plain member must not be decrypted: %v", legacy.Calls) } } + +// ---- review regressions ---- + +// A database failing in the last run of each day has its own history: keep_daily +// holds for it, not only for the primary's ids. +func TestPruneSelectsEachDatabaseSeriesOnItsOwn(t *testing.T) { + r := "a/accessories/db/" + var lines []string + var wantKept []string + for d := 1; d <= 9; d++ { + early := "2026010" + string(rune('0'+d)) + "-020000" + late := "2026010" + string(rune('0'+d)) + "-200000" + lines = append(lines, "d t 1000 "+r+early+".pgdump", "d t 1000 "+r+late+".pgdump", "d t 1000 "+r+"databases/x/"+early+".pgdump") + if d >= 7 { + wantKept = append(wantKept, "s3://b/"+r+"databases/x/"+early+".pgdump") + } + } + mock := treeMock(strings.Join(lines, "\n")) + if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepDaily: 3}); err != nil { + t.Fatal(err) + } + removed := map[string]bool{} + for _, k := range rmKeys(mock) { + removed[k] = true + } + for _, k := range wantKept { + if removed[k] { + t.Fatalf("keep_daily 3 must keep %s: %v", k, rmKeys(mock)) + } + } + if !removed["s3://b/"+r+"databases/x/20260106-020000.pgdump"] { + t.Fatalf("the series past its keep_daily was not pruned: %v", rmKeys(mock)) + } +} + +// A primary that reappears after a series-only period does not decide the +// databases' history, and one dropped long ago does not freeze it. +func TestPruneDatabaseSeriesIgnoresThePrimarysPeriods(t *testing.T) { + r := "a/accessories/db/" + lines := []string{"d t 1000 " + r + "20260201-000000.pgdump"} // reappeared today + for d := 10; d <= 28; d++ { + lines = append(lines, "d t 1000 "+r+"databases/x/202601"+strconv.Itoa(d)+"-000000.pgdump") + } + mock := treeMock(strings.Join(lines, "\n")) + if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 5}); err != nil { + t.Fatal(err) + } + if got := len(rmKeys(mock)); got != 14 { + t.Fatalf("19 objects with keep_last 5 must lose 14, lost %d", got) + } + // The primary long gone: the series is still selected. + lines[0] = "d t 1000 " + r + "20200101-000000.pgdump" + mock = treeMock(strings.Join(lines, "\n")) + if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 5}); err != nil { + t.Fatal(err) + } + if got := len(rmKeys(mock)); got != 14 { + t.Fatalf("an old primary froze the series: lost %d", got) + } +} + +func TestPruneOwnersFailClosed(t *testing.T) { + ctx := context.Background() + p := RetentionPolicy{KeepLast: 1} + s3 := S3Config{Bucket: "b", Region: "r"} + // A listing that fails is not "no owners". + mock := newStoreMock("h", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "aws s3 ls 's3://b/a/.owners/'", Err: errors.New("exit status 254: SlowDown")}, + ssh.MockCommand{Match: "aws s3 ls", Output: "d t 1000 a/volumes/20240101-000000.tar.gz\nd t 1000 a/volumes/20240102-000000.tar.gz\n"}, + ssh.MockCommand{Match: "aws s3 rm", Output: ""}, + ) + if _, err := NewClient(mock, io.Discard).PruneBackups(ctx, "a", "volumes", s3, p); !errors.Is(err, ErrSharedNamespace) { + t.Fatalf("err = %v", err) + } + if len(rmKeys(mock)) != 0 { + t.Fatal("deleted on a failed owners listing") + } + // Empty machine-id and hostname hash to one value on every such host. + mock = ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "{ cat /etc/machine-id", Output: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -\n"}, + ssh.MockCommand{Match: "aws s3 ls", Output: ""}, + ssh.MockCommand{Match: "aws s3 rm", Output: ""}, + ) + if _, err := NewClient(mock, io.Discard).PruneBackups(ctx, "a", "volumes", s3, p); !errors.Is(err, ErrSharedNamespace) { + t.Fatalf("an unidentifiable server: %v", err) + } + // A clone: same machine-id and hostname, another age identity. + mock = ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "{ cat /etc/machine-id", Output: "0123456789abcdef0123456789abcdef -\n"}, + ssh.MockCommand{Match: "aws s3 ls 's3://b/a/.owners/'", Output: "d t 20 a/.owners/0123456789abcdef\n"}, + ssh.MockCommand{Match: "aws s3 cp 's3://b/a/.owners/0123456789abcdef' -", Output: `{"recipient":"` + testOwnerRecipient + `"}`}, + ssh.MockCommand{Match: "for p in /etc/teploy/age-key", Output: testServerRecipient + "\n"}, + ssh.MockCommand{Match: "aws s3 rm", Output: ""}, + ) + if _, err := NewClient(mock, io.Discard).PruneBackups(ctx, "a", "volumes", s3, p); !errors.Is(err, ErrSharedNamespace) || !strings.Contains(err.Error(), "--clear") { + t.Fatalf("a clone: %v", err) + } +} + +// The marker is written before anything is deleted. +func TestPruneMarksBeforeItDeletes(t *testing.T) { + r := "a/volumes/" + mock := treeMock("d t 1000 " + r + "20240103-000000.tar.gz\nd t 1000 " + r + "20240102-000000.tar.gz\n") + if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "volumes", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1}); err != nil { + t.Fatal(err) + } + marked, deleted := -1, -1 + for i, c := range mock.Calls { + if marked < 0 && strings.Contains(c, ".owners/0123456789abcdef") && strings.HasPrefix(c, "printf") { + marked = i + } + if deleted < 0 && strings.HasPrefix(c, "aws s3 rm") { + deleted = i + } + } + if marked < 0 || deleted < 0 || marked > deleted { + t.Fatalf("marked at %d, deleted at %d: %v", marked, deleted, mock.Calls) + } +} + +func TestPruneRefusesAWrongClock(t *testing.T) { + ctx := context.Background() + r := "a/volumes/" + listing := "d t 1000 " + r + "20240103-000000.tar.gz\nd t 1000 " + r + "20240102-000000.tar.gz\n" + // The store says it is a year from now: this machine is slow, or the store fast. + future := time.Now().UTC().AddDate(1, 0, 0).Format(time.RFC3339) + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "{ cat /etc/machine-id", Output: "0123456789abcdef0123456789abcdef -\n"}, + ssh.MockCommand{Match: "printf", Output: ""}, + ssh.MockCommand{Match: "for p in /etc/teploy/age-key", Output: ""}, + ssh.MockCommand{Match: "aws s3api head-object", Output: `{"ContentLength":1,"LastModified":"` + future + `"}`}, + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "aws s3 ls", Output: listing}, + ssh.MockCommand{Match: "aws s3 rm", Output: ""}, + ) + _, err := NewClient(mock, io.Discard).PruneBackups(ctx, "a", "volumes", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1}) + if !errors.Is(err, ErrClockSkew) || len(rmKeys(mock)) != 0 { + t.Fatalf("err = %v, removed %v", err, rmKeys(mock)) + } + if _, err := NewClient(mock, io.Discard).PruneBackups(ctx, "a", "volumes", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1, IgnoreClock: true}); err != nil { + t.Fatalf("--ignore-clock: %v", err) + } + // A backup dated in the future would count as the newest: keep_last 1 would + // then delete today's real one. + mock = treeMock("d t 1000 " + r + time.Now().UTC().AddDate(1, 0, 0).Format("20060102-150405") + ".tar.gz\nd t 1000 " + r + time.Now().UTC().Format("20060102-150405") + ".tar.gz\n") + _, err = NewClient(mock, io.Discard).PruneBackups(ctx, "a", "volumes", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1}) + if !errors.Is(err, ErrClockSkew) || len(rmKeys(mock)) != 0 { + t.Fatalf("future-dated: err = %v, removed %v", err, rmKeys(mock)) + } +} + +func TestPrunePendingUploadsAtAnyDepth(t *testing.T) { + r := "a/accessories/db/" + listing := strings.Join([]string{ + "d t 1000 " + r + "20240103-000000.pgdump", + "d t 1000 " + r + "databases/x/.incoming/20240101-000000.pgdump", + "d t 1000 " + r + "globals/.incoming/20240101-000000.sql.gz", + "d t 1000 " + r + "meta/.incoming/20240101-000000.json", + "d t 1000 " + r + ".incoming/20240101-000000.pgdump", + }, "\n") + mock := treeMock(listing) + if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1}); err != nil { + t.Fatal(err) + } + if got := rmKeys(mock); len(got) != 4 { + t.Fatalf("removed %v", got) + } +} + +// A destructive operation's safety copy outlasts retention for its window. +func TestPruneProtectsSafetyCopies(t *testing.T) { + r := "a/accessories/db/" + recent := time.Now().UTC().Add(-time.Hour).Format("20060102-150405") + newer := time.Now().UTC().Add(-time.Minute).Format("20060102-150405") + old := time.Now().UTC().AddDate(0, 0, -30).Format("20060102-150405") + listing := strings.Join([]string{ + "d t 1000 " + r + newer + ".pgdump", + "d t 1000 " + r + recent + ".pgdump", + "d t 8 " + r + "safety/" + recent + ".safety", + "d t 1000 " + r + old + ".pgdump", + "d t 8 " + r + "safety/" + old + ".safety", + }, "\n") + mock := treeMock(listing) + if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1}); err != nil { + t.Fatal(err) + } + got := strings.Join(rmKeys(mock), ",") + if strings.Contains(got, recent) { + t.Fatalf("a recent safety copy was pruned: %s", got) + } + if !strings.Contains(got, old+".pgdump") || !strings.Contains(got, "safety/"+old) { + t.Fatalf("a safety copy past its window, with its marker, must go: %s", got) + } +} + +func TestPutLargeObjectUsesAProviderChecksum(t *testing.T) { + ctx := context.Background() + big := func(extra ...ssh.MockCommand) *ssh.MockExecutor { + return putMock(append(extra, ssh.MockCommand{Match: "aws s3api head-object", Output: `{"ContentLength":4096,"Metadata":{"sha256":"abc"},"ChecksumSHA256":"AAA=-3"}`})...) + } + c := func(m *ssh.MockExecutor) *Client { cl := NewClient(m, io.Discard); cl.readBack = 1; return cl } + mock := big() + if _, err := c(mock).put(ctx, S3Config{Bucket: "b", Region: "r"}, "/w/x", "s3://b/k/1.pgdump"); err != nil { + t.Fatal(err) + } + joined := strings.Join(mock.Calls, "\n") + if !strings.Contains(joined, "--checksum-algorithm SHA256") || strings.Contains(joined, "{ aws s3 cp") { + t.Fatalf("calls: %s", joined) + } + // A whole-object digest that differs refuses. + mock = putMock( + ssh.MockCommand{Match: "aws s3api head-object", Output: `{"ContentLength":4096,"Metadata":{"sha256":"abc"},"ChecksumSHA256":"bm9wZQ=="}`}, + ) + if _, err := c(mock).put(ctx, S3Config{Bucket: "b", Region: "r"}, "/w/x", "s3://b/k/1.pgdump"); err == nil || !strings.Contains(err.Error(), "checksum mismatch") { + t.Fatalf("err = %v", err) + } + // A store or CLI without it falls back, and says so. + var out strings.Builder + mock = big(ssh.MockCommand{Match: "aws s3 cp '/w/x' 's3://b/k/.incoming/1.pgdump' --metadata 'sha256=abc' --checksum-algorithm", Err: errors.New("exit status 2: Unknown options: --checksum-algorithm")}) + cl := NewClient(mock, &out) + cl.readBack = 1 + if _, err := cl.put(ctx, S3Config{Bucket: "b", Region: "r"}, "/w/x", "s3://b/k/1.pgdump"); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "verified by size alone") { + t.Fatalf("out: %s", out.String()) + } +} + +func TestFetchPassesNoMissingIdentity(t *testing.T) { + mock := putMock(ssh.MockCommand{Match: "umask 077; age -d", Output: ""}) + if err := NewClient(mock, io.Discard).fetch(context.Background(), S3Config{Bucket: "b", Region: "r", ExtraIdentity: "/tmp/owner"}, "s3://b/k/1.pgdump.age", "/w/x"); err != nil { + t.Fatal(err) + } + var dec string + for _, c := range mock.Calls { + if strings.HasPrefix(c, "umask 077; age -d") { + dec = c + } + } + if !strings.Contains(dec, `[ -r "$p" ]`) || !strings.Contains(dec, "-i '/tmp/owner'") || strings.Contains(dec, `-i "$(`) { + t.Fatalf("%s", dec) + } +} + +func TestAddressableDBRefusesAConnectionURI(t *testing.T) { + if addressableDB("postgresql://host/x") || addressableDB("postgres://u@h/x") { + t.Fatal("pg_dump reads these as conninfo") + } +} + +func TestStaleWorkspacesLookInTheConfiguredDir(t *testing.T) { + t.Setenv("TEPLOY_BACKUP_TMPDIR", "/data/tmp") + if !strings.Contains(tmpGlobs("", "teploy-backup.*"), "'/data/tmp'/teploy-backup.*") { + t.Fatal(tmpGlobs("", "teploy-backup.*")) + } + t.Setenv("TEPLOY_BACKUP_TMPDIR", "") + if !strings.Contains(LeftoverWorkspacesProbe("/mnt/big"), "'/mnt/big'/teploy-backup.*") { + t.Fatal(LeftoverWorkspacesProbe("/mnt/big")) + } +} + +// A bucket-wide multipart listing would catch other servers' uploads. +func TestAbortStaleUploadsIsScopedToOwnedNamespaces(t *testing.T) { + old := time.Now().UTC().AddDate(0, 0, -2).Format(time.RFC3339) + mock := newStoreMock("h", + ssh.MockCommand{Match: "aws s3 ls 's3://b/mine/.owners/'", Output: "d t 20 mine/.owners/0123456789abcdef\n"}, + ssh.MockCommand{Match: "aws s3 ls 's3://b/shared/.owners/'", Output: "d t 20 shared/.owners/aaaaaaaaaaaaaaaa\n"}, + ssh.MockCommand{Match: "aws s3api list-multipart-uploads", Output: `{"Uploads":[{"Key":"mine/x","UploadId":"u1","Initiated":"` + old + `"}]}`}, + ssh.MockCommand{Match: "aws s3api abort-multipart-upload", Output: ""}, + ) + if n := NewClient(mock, io.Discard).AbortStaleUploads(context.Background(), S3Config{Bucket: "b", Region: "r"}, []string{"mine", "shared"}, time.Now().UTC()); n != 1 { + t.Fatalf("aborted %d", n) + } + lists := 0 + for _, c := range mock.Calls { + if strings.HasPrefix(c, "aws s3api list-multipart-uploads") { + lists++ + if !strings.Contains(c, "--prefix 'mine/'") { + t.Fatalf("unscoped: %s", c) + } + } + } + if lists != 1 { + t.Fatalf("lists = %d", lists) + } +} diff --git a/internal/backup/store_test_helpers_test.go b/internal/backup/store_test_helpers_test.go index 7cae720..3824842 100644 --- a/internal/backup/store_test_helpers_test.go +++ b/internal/backup/store_test_helpers_test.go @@ -12,7 +12,10 @@ func newStoreMock(host string, cmds ...ssh.MockCommand) *ssh.MockExecutor { var storeStubs = []ssh.MockCommand{ {Match: "sha256sum", Output: "abc f\n4096\n"}, + {Match: "{ cat /etc/machine-id", Output: "0123456789abcdef0123456789abcdef -\n"}, {Match: "{ ", Output: "abc -\n"}, + {Match: "printf '%s\\n'", Output: ""}, + {Match: "for p in /etc/teploy/age-key", Output: ""}, {Match: "aws s3api head-object", Output: `{"ContentLength":4096,"Metadata":{"sha256":"abc"}}`}, } diff --git a/internal/cli/accessory.go b/internal/cli/accessory.go index cd151a6..157c23e 100644 --- a/internal/cli/accessory.go +++ b/internal/cli/accessory.go @@ -368,7 +368,7 @@ func runAccessoryUpgrade(flags *Flags, name, newImage string, noBackup bool) err if err != nil { return err } - id, err := preOperationBackup(ctx, backup.NewClient(executor, os.Stdout), executor, appCfg.App, name, "", cfg, "this upgrade") + id, err := preOperationBackup(ctx, backup.NewClient(executor, os.Stdout), executor, appCfg.App, name, cfg, "this upgrade", preBackupScope{All: true}) if err != nil { return err } @@ -580,6 +580,8 @@ func newAccessoryRestoreCmd(flags *Flags) *cobra.Command { yes bool ignore bool noBackup bool + identity string + deadline time.Duration ) cmd := &cobra.Command{ @@ -587,7 +589,7 @@ func newAccessoryRestoreCmd(flags *Flags) *cobra.Command { Short: "Restore an accessory from backup (stops the app while it runs)", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - return runAccessoryRestore(flags, args[0], args[1], bucket, region, endpoint, prefix, yes, ignore, noBackup) + return runAccessoryRestore(flags, args[0], args[1], bucket, region, endpoint, prefix, identity, yes, ignore, noBackup, deadline) }, } @@ -598,11 +600,13 @@ func newAccessoryRestoreCmd(flags *Flags) *cobra.Command { cmd.Flags().BoolVar(&yes, "yes", false, "proceed without interactive confirmation (the app is stopped while the restore runs)") cmd.Flags().BoolVar(&ignore, "ignore-space-check", false, "skip the free-space preflight") cmd.Flags().BoolVar(&noBackup, "no-backup", false, "skip the automatic backup taken before the restore") + cmd.Flags().StringVar(&identity, "identity", "", "recovery key file, to decrypt a backup this host's identity cannot") + cmd.Flags().DurationVar(&deadline, "deadline", 0, "deadline of the backup taken first, and of the restore (default 6h each)") return cmd } -func runAccessoryRestore(flags *Flags, name, date, bucket, region, endpoint, prefix string, yes, ignoreSpace, noBackup bool) error { +func runAccessoryRestore(flags *Flags, name, date, bucket, region, endpoint, prefix, identity string, yes, ignoreSpace, noBackup bool, deadline time.Duration) error { appCfg, err := loadAppCfgForAccessory() if err != nil { return err @@ -616,7 +620,7 @@ func runAccessoryRestore(flags *Flags, name, date, bucket, region, endpoint, pre // the live database with the app still writing). Stopping the app is // an effect it never had, so it now asks first (--yes skips it, and is // required without a terminal) — the same contract as backup restore. - return runBackupRestoreWith(flags, "", name, false, date, &backupTargetFlags{bucket: bucket, region: region, endpoint: endpoint, prefix: prefix}, yes, backup.RestoreOptions{IgnoreSpaceCheck: ignoreSpace, NoPreBackup: noBackup}) + return runBackupRestoreWith(flags, "", name, false, date, &backupTargetFlags{bucket: bucket, region: region, endpoint: endpoint, prefix: prefix}, yes, backup.RestoreOptions{IgnoreSpaceCheck: ignoreSpace, NoPreBackup: noBackup, IdentityFile: identity, Deadline: deadline}) } // verifyEventErr is the backup.verify outcome: an operational error, or a @@ -676,6 +680,9 @@ func newAccessoryVerifyBackupCmd(flags *Flags) *cobra.Command { local bool database string adopted bool + identity string + deadline time.Duration + prefix string ) cmd := &cobra.Command{ @@ -695,21 +702,24 @@ func newAccessoryVerifyBackupCmd(flags *Flags) *cobra.Command { " teploy accessory verify-backup wordpress-shared-mysql --adopted --database wp_blog --host prod --json", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return runAccessoryVerifyBackup(flags, appName, args[0], bucket, region, date, endpoint, database, local, adopted) + return runAccessoryVerifyBackup(flags, appName, args[0], bucket, region, date, endpoint, prefix, database, identity, local, adopted, deadline) }, } cmd.Flags().StringVar(&database, "database", "", "verify this database's own backups instead of the primary") cmd.Flags().BoolVar(&adopted, "adopted", false, " is an adopted container's docker name (see `server backup-policy`)") + cmd.Flags().StringVar(&identity, "identity", "", "recovery key file, to decrypt a backup this host's identity cannot") + cmd.Flags().DurationVar(&deadline, "deadline", 0, "deadline of this verification (default 6h)") cmd.Flags().StringVar(&appName, "app", "", "app name — act on server state instead of teploy.yml (requires --host)") cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name") cmd.Flags().StringVar(®ion, "region", "us-east-1", "AWS region") cmd.Flags().StringVar(&endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2); creds from TEPLOY_S3_ACCESS_KEY/SECRET_KEY or AWS_* env") cmd.Flags().StringVar(&date, "date", "", "backup timestamp to verify (default: latest)") + cmd.Flags().StringVar(&prefix, "prefix", "", "key prefix under --bucket (default: the stored target's prefix)") cmd.Flags().BoolVar(&local, "local", false, "run against the local docker host (the mode scheduled cron jobs use; requires --app)") return cmd } -func runAccessoryVerifyBackup(flags *Flags, appName, name, bucket, region, date, endpoint, database string, local, adopted bool) error { +func runAccessoryVerifyBackup(flags *Flags, appName, name, bucket, region, date, endpoint, prefix, database, identity string, local, adopted bool, deadline time.Duration) error { if date != "" { if err := backup.ValidateDate(date); err != nil { return err @@ -751,10 +761,16 @@ func runAccessoryVerifyBackup(flags *Flags, appName, name, bucket, region, date, } defer executor.Close() - s3Cfg, err := resolveBackupS3(ctx, executor, &backupTargetFlags{bucket: bucket, region: region, endpoint: endpoint}) + s3Cfg, err := resolveBackupS3(ctx, executor, &backupTargetFlags{bucket: bucket, region: region, endpoint: endpoint, prefix: prefix}) + if err != nil { + return err + } + identityPath, cleanupIdentity, err := placeIdentity(ctx, executor, identity) if err != nil { return err } + defer cleanupIdentity() + s3Cfg.ExtraIdentity = identityPath out := io.Writer(os.Stdout) if flags.JSON { @@ -762,6 +778,7 @@ func runAccessoryVerifyBackup(flags *Flags, appName, name, bucket, region, date, out = os.Stderr } client := backup.NewClient(executor, out) + client.SetDeadline(deadline) if err := applyAdoptedEngine(ctx, client, executor, appCfg.App, name); err != nil { return err } diff --git a/internal/cli/backup.go b/internal/cli/backup.go index 3f78c93..0dc244e 100644 --- a/internal/cli/backup.go +++ b/internal/cli/backup.go @@ -9,6 +9,7 @@ import ( "os" "os/signal" "strings" + "time" "github.com/spf13/cobra" "github.com/useteploy/teploy/internal/backup" @@ -126,15 +127,19 @@ func TargetS3(t backup.Target) backup.S3Config { if region == "" { region = "us-east-1" } - return backup.S3Config{ - Bucket: t.Bucket, - Region: region, - Endpoint: t.Endpoint, - Prefix: t.Prefix, - StoredTarget: t.Path, - + cfg := backup.S3Config{ + Bucket: t.Bucket, + Region: region, + Endpoint: t.Endpoint, + Prefix: t.Prefix, PreviousPrefixes: t.PreviousPrefixes, } + // Only with credentials: the stored path decrypts them, and a --no-credentials + // target has none, so every command through it failed. + if t.HasCredentials { + cfg.StoredTarget = t.Path + } + return cfg } // s3Config assembles the legacy flag-driven target. --endpoint switches the @@ -218,6 +223,7 @@ func newBackupCreateCmd(flags *Flags) *cobra.Command { tf backupTargetFlags keepLast int maxAgeDays int + deadline time.Duration ) cmd := &cobra.Command{ @@ -225,7 +231,7 @@ func newBackupCreateCmd(flags *Flags) *cobra.Command { Short: "Create a volume or accessory backup", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - return runBackupCreate(flags, appName, accessory, database, adopted, &tf, keepLast, maxAgeDays) + return runBackupCreate(flags, appName, accessory, database, adopted, &tf, keepLast, maxAgeDays, deadline) }, } @@ -233,13 +239,14 @@ func newBackupCreateCmd(flags *Flags) *cobra.Command { cmd.Flags().StringVar(&accessory, "accessory", "", "back up this accessory instead of the app's volumes") cmd.Flags().StringVar(&database, "database", "", "back up only this database of the accessory") cmd.Flags().BoolVar(&adopted, "adopted", false, "--accessory names an adopted container (see `server backup-policy`), addressed by its docker name (requires --host)") + cmd.Flags().DurationVar(&deadline, "deadline", 0, "deadline of this backup (default 6h)") addBackupTargetFlags(cmd, &tf) addRetentionFlags(cmd, &keepLast, &maxAgeDays) return cmd } -func runBackupCreate(flags *Flags, appName, accessory, database string, adopted bool, tf *backupTargetFlags, keepLast, maxAgeDays int) error { +func runBackupCreate(flags *Flags, appName, accessory, database string, adopted bool, tf *backupTargetFlags, keepLast, maxAgeDays int, deadline time.Duration) error { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() if adopted && accessory == "" { @@ -261,6 +268,7 @@ func runBackupCreate(flags *Flags, appName, accessory, database string, adopted } client := backup.NewClient(executor, backupProgressOut(flags)) + client.SetDeadline(deadline) var res backup.BackupResult var kind string @@ -404,19 +412,24 @@ type backupScheduleDTO struct { func newBackupListCmd(flags *Flags) *cobra.Command { var ( - appName string - adopted bool - tf backupTargetFlags + appName string + adopted bool + databases bool + tf backupTargetFlags ) cmd := &cobra.Command{ Use: "list", Short: "List available backups and schedules", - Args: cobra.NoArgs, + Long: "--databases also lists each database's own objects. They are off by default:\n" + + "a row carries a database that a caller must pass as `restore --database`, and\n" + + "a caller that restores by id alone would replace the primary with the wrong one.", + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - return runBackupList(flags, appName, adopted, &tf) + return runBackupList(flags, appName, adopted, databases, &tf) }, } + cmd.Flags().BoolVar(&databases, "databases", false, "also list each database's own objects (their rows carry \"database\")") cmd.Flags().StringVar(&appName, "app", "", "app name — act on server state instead of teploy.yml (requires --host)") cmd.Flags().BoolVar(&adopted, "adopted", false, "list adopted containers' backups instead of an app's (requires --host)") @@ -425,7 +438,7 @@ func newBackupListCmd(flags *Flags) *cobra.Command { return cmd } -func runBackupList(flags *Flags, appName string, adopted bool, tf *backupTargetFlags) error { +func runBackupList(flags *Flags, appName string, adopted, databases bool, tf *backupTargetFlags) error { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() @@ -445,9 +458,8 @@ func runBackupList(flags *Flags, appName string, adopted bool, tf *backupTargetF if err != nil { return err } - if rows == nil { - rows = []backup.ListedBackup{} - } + // An adopted container's backups are all per-database rows. + rows, hidden := filterDatabaseRows(rows, databases || adopted) schedules, err := discoverBackupSchedules(ctx, executor, appCfg.App) if err != nil { return err @@ -478,6 +490,9 @@ func runBackupList(flags *Flags, appName string, adopted bool, tf *backupTargetF }) } + if hidden > 0 { + fmt.Printf("%d per-database object(s) not shown: --databases lists them\n", hidden) + } if len(rows) == 0 && len(schedules) == 0 { fmt.Println("No backups or schedules found") return nil @@ -509,6 +524,20 @@ func runBackupList(flags *Flags, appName string, adopted bool, tf *backupTargetF return nil } +// filterDatabaseRows drops per-database rows unless asked for, reporting how many. +func filterDatabaseRows(rows []backup.ListedBackup, keep bool) ([]backup.ListedBackup, int) { + out := []backup.ListedBackup{} + hidden := 0 + for _, r := range rows { + if r.Database != "" && !keep { + hidden++ + continue + } + out = append(out, r) + } + return out, hidden +} + // discoverBackupSchedules reads the crontab's teploy backup markers and // joins each with the retention stored in the app's schedule store. func discoverBackupSchedules(ctx context.Context, exec ssh.Executor, app string) ([]backupScheduleDTO, error) { @@ -593,7 +622,10 @@ func newBackupRestoreCmd(flags *Flags) *cobra.Command { "Before anything changes a fresh backup of what is about to be replaced is\n" + "taken; if that fails the restore is refused, unless --no-backup.\n\n" + "--adopted addresses a container by its docker name (no app, so nothing is\n" + - "stopped for you: stop what uses it first), one database at a time.", + "stopped for you: stop what uses it first), one database at a time.\n\n" + + "--identity is the recovery key file, for a host without the identity that\n" + + "encrypted the backup. It is placed on the server privately for this restore\n" + + "and removed afterwards.", Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { date := "" @@ -625,6 +657,8 @@ func newBackupRestoreCmd(flags *Flags) *cobra.Command { cmd.Flags().BoolVar(&opts.ignoreSpace, "ignore-space-check", false, "skip the free-space preflight") cmd.Flags().BoolVar(&opts.noBackup, "no-backup", false, "skip the automatic backup taken before the restore") cmd.Flags().BoolVar(&adopted, "adopted", false, "--accessory names an adopted container (requires --host)") + cmd.Flags().StringVar(&opts.identity, "identity", "", "recovery key file, to decrypt a backup this host's identity cannot") + cmd.Flags().DurationVar(&opts.deadline, "deadline", 0, "deadline of the backup taken first, and of the restore (default 6h each)") return cmd } @@ -632,11 +666,12 @@ func newBackupRestoreCmd(flags *Flags) *cobra.Command { // restoreFlags are the accessory-restore options of `backup restore`. type restoreFlags struct { release, rollback, ignoreSpace, force, noBackup bool - recoveryDatabase, database string + recoveryDatabase, database, identity string + deadline time.Duration } func (o *restoreFlags) restoreOptions() backup.RestoreOptions { - return backup.RestoreOptions{IgnoreSpaceCheck: o.ignoreSpace, Database: o.database, NoPreBackup: o.noBackup} + return backup.RestoreOptions{IgnoreSpaceCheck: o.ignoreSpace, Database: o.database, NoPreBackup: o.noBackup, IdentityFile: o.identity, Deadline: o.deadline} } func (o *restoreFlags) validate(date, accessory string) error { @@ -653,6 +688,8 @@ func (o *restoreFlags) validate(date, accessory string) error { return fmt.Errorf("--ignore-space-check applies to a restore, not to --release/--rollback") case action && o.noBackup: return fmt.Errorf("--no-backup applies to a restore, not to --release/--rollback") + case action && o.identity != "": + return fmt.Errorf("--identity applies to a restore, not to --release/--rollback") case o.force && !o.release: return fmt.Errorf("--force applies to --release") case !action && date == "": @@ -681,6 +718,12 @@ func runBackupRestoreWith(flags *Flags, appName, accessory string, adopted bool, if err != nil { return err } + identityPath, cleanupIdentity, err := placeIdentity(ctx, executor, restoreOpts.IdentityFile) + if err != nil { + return err + } + defer cleanupIdentity() + cfg.ExtraIdentity = identityPath if err := backup.ValidateDate(date); err != nil { return err } @@ -717,6 +760,7 @@ func runBackupRestoreWith(flags *Flags, appName, accessory string, adopted bool, progress := backupProgressOut(flags) client := backup.NewClient(executor, progress) + client.SetDeadline(restoreOpts.Deadline) if err := applyAdoptedEngine(ctx, client, executor, appCfg.App, accessory); err != nil { return err } @@ -762,7 +806,7 @@ func runBackupRestoreWith(flags *Flags, appName, accessory string, adopted bool, } // A fresh copy of what is about to be replaced, before anything stops. if !restoreOpts.NoPreBackup { - id, pErr := preOperationBackup(ctx, client, executor, appCfg.App, accessory, restoreOpts.Database, cfg, "this restore") + id, pErr := preOperationBackup(ctx, client, executor, appCfg.App, accessory, cfg, "this restore", preBackupScope{Database: restoreOpts.Database, IgnoreSpace: restoreOpts.IgnoreSpaceCheck}) if pErr != nil { return finishRestoreFailure(flags, ctx, executor, lc, lk, dto, false, nil, pErr) } @@ -1062,13 +1106,14 @@ func restorePrompt(app, accessory, database, date string) string { func newBackupPruneCmd(flags *Flags) *cobra.Command { var ( - appName string - adopted bool - tf backupTargetFlags - accessory string - keepLast int - maxAgeDays int - gfs backup.FleetRetention + appName string + adopted bool + tf backupTargetFlags + accessory string + keepLast int + maxAgeDays int + gfs backup.FleetRetention + ignoreClock bool ) cmd := &cobra.Command{ @@ -1082,7 +1127,9 @@ func newBackupPruneCmd(flags *Flags) *cobra.Command { Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { gfs.KeepLast, gfs.MaxAgeDays = keepLast, maxAgeDays - return runBackupPrune(flags, appName, adopted, &tf, accessory, gfs.Policy()) + policy := gfs.Policy() + policy.IgnoreClock = ignoreClock + return runBackupPrune(flags, appName, adopted, &tf, accessory, policy) }, } @@ -1091,6 +1138,7 @@ func newBackupPruneCmd(flags *Flags) *cobra.Command { cmd.Flags().IntVar(&gfs.KeepDaily, "keep-daily", 0, "keep the newest backup of the last N days that have one") cmd.Flags().IntVar(&gfs.KeepWeekly, "keep-weekly", 0, "keep the newest backup of the last N ISO weeks that have one") cmd.Flags().IntVar(&gfs.KeepMonthly, "keep-monthly", 0, "keep the newest backup of the last N months that have one") + cmd.Flags().BoolVar(&ignoreClock, "ignore-clock", false, "prune even though this machine's clock disagrees with the store's, or a backup is dated in the future") addBackupTargetFlags(cmd, &tf) cmd.Flags().StringVar(&accessory, "accessory", "", "Prune an accessory's backups instead of volumes (accessory name)") addRetentionFlags(cmd, &keepLast, &maxAgeDays) diff --git a/internal/cli/backup_cron_test.go b/internal/cli/backup_cron_test.go index 39f8d0d..1f75654 100644 --- a/internal/cli/backup_cron_test.go +++ b/internal/cli/backup_cron_test.go @@ -151,7 +151,7 @@ func TestDiscoverBackupSchedules(t *testing.T) { func TestResolveBackupS3(t *testing.T) { ctx := context.Background() stored := ssh.NewMockExecutor("srv", - ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "absent\n"}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "absent\n"}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"kept","region":"eu-west-1","endpoint":"http://127.0.0.1:9100","credentials":"AGE"}`}, ) cfg, err := resolveBackupS3(ctx, stored, &backupTargetFlags{}) diff --git a/internal/cli/backup_destructive_test.go b/internal/cli/backup_destructive_test.go index ced8aa8..013a7fb 100644 --- a/internal/cli/backup_destructive_test.go +++ b/internal/cli/backup_destructive_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/useteploy/teploy/internal/backup" "github.com/useteploy/teploy/internal/config" "github.com/useteploy/teploy/internal/ssh" ) @@ -84,3 +85,68 @@ func TestBackupBeforeDeployHook(t *testing.T) { t.Fatalf("err = %v", err) } } + +// A docker blip is not "no container": a purge must not go ahead unprotected. +func TestPreOperationBackupInspectBlipRefuses(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "docker inspect -f", Err: errors.New("exit status 1: Cannot connect to the Docker daemon")}) + _, err := preOperationBackup(context.Background(), backup.NewClient(mock, io.Discard), mock, "a", "db", backup.S3Config{Bucket: "b", Region: "r"}, "remove --purge", preBackupScope{All: true}) + if err == nil || !strings.Contains(err.Error(), "--no-backup") { + t.Fatalf("err = %v", err) + } +} + +// An unreadable volumes tree is not an empty one. +func TestPreOperationBackupUnreadableVolumesRefuse(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "if [ ! -e '/deployments/a/volumes'", Err: errors.New("exit status 1: ls: permission denied")}) + _, err := preOperationBackup(context.Background(), backup.NewClient(mock, io.Discard), mock, "a", "", backup.S3Config{Bucket: "b", Region: "r"}, "this restore", preBackupScope{}) + if err == nil || !strings.Contains(err.Error(), "--no-backup") { + t.Fatalf("err = %v", err) + } +} + +// Purge and upgrade destroy every database, so every one must be stored; a name +// the tools cannot address refuses. Mock mysql with one such schema. +func TestPreOperationBackupAllCoversEveryDatabase(t *testing.T) { + calls := func(extra ...ssh.MockCommand) *ssh.MockExecutor { + return ssh.NewMockExecutor("h", append(extra, + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "docker inspect -f '{{.Config.Image}}'", Output: "mysql:8\n"}, + ssh.MockCommand{Match: "docker inspect -f '{{range .Config.Env}}", Output: "MYSQL_VERSION=8\nMYSQL_DATABASE=app\nMYSQL_ROOT_PASSWORD=pw\n"}, + ssh.MockCommand{Match: "docker inspect -f", Output: "{}\n[]\n[]\n"}, + ssh.MockCommand{Match: "umask 077", Output: "/var/tmp/teploy-backup.t\n"}, + ssh.MockCommand{Match: "docker exec 'a-db' sh -c 'MYSQL_PWD=${MARIADB_ROOT_PASSWORD:-${MYSQL_ROOT_PASSWORD:-${MYSQL_PASSWORD:-}}}; export MYSQL_PWD; exec mysql -u root -N -B -e '\"'\"'SELECT COUNT(*)", Output: "1\n"}, + ssh.MockCommand{Match: "docker exec", Output: ""}, + ssh.MockCommand{Match: "sha256sum", Output: "abc f\n4096\n"}, + ssh.MockCommand{Match: "{ cat /etc/machine-id", Output: "0123456789abcdef0123456789abcdef -\n"}, + ssh.MockCommand{Match: "{ ", Output: "abc -\n"}, + ssh.MockCommand{Match: "aws s3api head-object", Output: `{"ContentLength":4096,"Metadata":{"sha256":"abc"}}`}, + ssh.MockCommand{Match: "aws s3", Output: ""}, + ssh.MockCommand{Match: "mkdir -p", Output: ""}, + ssh.MockCommand{Match: "rm -rf", Output: ""}, + ssh.MockCommand{Match: "wc -c", Output: "4096\n"}, + ssh.MockCommand{Match: "df -Pk", Output: "Filesystem 1K-blocks Used Available Use% Mounted\n/dev/x 1 1 99999999 1% /\n"}, + ssh.MockCommand{Match: "printf", Output: ""}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq":1,"ts":"2026-01-01T00:00:00Z"}`}, + )...) + } + schemas := func(rows string) ssh.MockCommand { + return ssh.MockCommand{Match: "docker exec 'a-db' sh -c 'MYSQL_PWD=${MARIADB_ROOT_PASSWORD:-${MYSQL_ROOT_PASSWORD:-${MYSQL_PASSWORD:-}}}; export MYSQL_PWD; exec mysql -u root -N -B -e '\"'\"'SELECT s.schema_name", Output: rows} + } + cfg := backup.S3Config{Bucket: "b", Region: "r"} + mock := calls(schemas("app\t0\nanalytics\t0\n")) + id, err := preOperationBackup(context.Background(), backup.NewClient(mock, io.Discard), mock, "a", "db", cfg, "remove --purge", preBackupScope{All: true}) + if err != nil || id == "" { + t.Fatalf("%q %v", id, err) + } + joined := strings.Join(mock.Calls, "\n") + if !strings.Contains(joined, "databases/analytics/.incoming/") { + t.Fatalf("analytics was not stored: %s", joined) + } + if !strings.Contains(joined, "/safety/"+id+".safety") { + t.Fatalf("no safety marker: %s", joined) + } + mock = calls(schemas("app\t0\nit's\t0\n")) + if _, err := preOperationBackup(context.Background(), backup.NewClient(mock, io.Discard), mock, "a", "db", cfg, "remove --purge", preBackupScope{All: true}); err == nil || !strings.Contains(err.Error(), "it's") || !strings.Contains(err.Error(), "--no-backup") { + t.Fatalf("err = %v", err) + } +} diff --git a/internal/cli/backup_fleet.go b/internal/cli/backup_fleet.go index 618746b..a6ab586 100644 --- a/internal/cli/backup_fleet.go +++ b/internal/cli/backup_fleet.go @@ -16,6 +16,7 @@ import ( "filippo.io/age" "github.com/spf13/cobra" + "github.com/spf13/pflag" "github.com/useteploy/teploy/internal/backup" "github.com/useteploy/teploy/internal/config" "github.com/useteploy/teploy/internal/events" @@ -113,6 +114,9 @@ func runFleet(ctx context.Context, exec ssh.Executor, out io.Writer, ignoreSpace return res, fleetFailed(res) } res.Encrypted = policy.Recipient != "" + if policy.TmpDir != "" { + os.Setenv("TEPLOY_BACKUP_TMPDIR", policy.TmpDir) + } lock, holder, lerr := backup.AcquireRunLock(backup.RunLockPath(policy.Path), started, fleetRunDeadline+time.Hour) if lerr != nil { @@ -137,6 +141,11 @@ func runFleet(ctx context.Context, exec ssh.Executor, out io.Writer, ignoreSpace cfg := TargetS3(target) cfg.OwnerRecipient = policy.Recipient client := backup.NewClient(exec, out) + if policy.TargetDeadline != "" { + if d, derr := time.ParseDuration(policy.TargetDeadline); derr == nil { + client.SetDeadline(d) + } + } if err := client.EnsureAWSCLIPublic(ctx); err != nil { return finishFleet(ctx, exec, out, res, policy, started, err, hc) } @@ -147,20 +156,35 @@ func runFleet(ctx context.Context, exec ssh.Executor, out io.Writer, ignoreSpace res.Warnings = append(res.Warnings, "no recovery key is configured, so objects are stored unencrypted and app .env files are left out of volume backups (teploy backup keygen; teploy server backup-policy set --recipient)") } - // A killed run's leftovers, now that this run owns the lock. Bounded: a wedged - // janitor must not cost the backups. - jctx, jcancel := context.WithTimeout(ctx, 5*time.Minute) - res.CleanedWorkspaces = len(client.CleanStaleWorkspaces(jctx)) - res.AbortedUploads = client.AbortStaleUploads(jctx, cfg, started) - jcancel() - ectx, ecancel := context.WithTimeout(ctx, 10*time.Minute) - targets, uncovered, err := client.EnumerateFleet(ectx, policy) + targets, uncovered, enumWarnings, err := client.EnumerateFleet(ectx, policy) ecancel() if err != nil { return finishFleet(ctx, exec, out, res, policy, started, err, hc) } res.UncoveredContainers = uncovered + res.Warnings = append(res.Warnings, enumWarnings...) + // Zero targets with a policy: a lost accessory and an empty docker daemon look + // exactly like this, and reporting ok would hide them. A server meant to have + // nothing yet says so. + if len(targets) == 0 && !policy.AllowEmpty { + return finishFleet(ctx, exec, out, res, policy, started, fmt.Errorf("the policy found nothing to back up on this server: no accessory, app volume or adopted container. If that is right, set allow_empty (teploy server backup-policy set --allow-empty)"), hc) + } + + // A killed run's leftovers, now that this run owns the lock. Bounded: a wedged + // janitor must not cost the backups. + jctx, jcancel := context.WithTimeout(ctx, 5*time.Minute) + res.CleanedWorkspaces = len(client.CleanStaleWorkspaces(jctx)) + nsSeen := map[string]bool{} + var namespaces []string + for _, t := range targets { + if t.App != "" && !nsSeen[t.App] { + nsSeen[t.App] = true + namespaces = append(namespaces, t.App) + } + } + res.AbortedUploads = client.AbortStaleUploads(jctx, cfg, namespaces, started) + jcancel() for _, t := range targets { if ctx.Err() != nil { @@ -172,6 +196,11 @@ func runFleet(ctx context.Context, exec ssh.Executor, out io.Writer, ignoreSpace r := client.RunFleetTarget(ctx, t, cfg, backup.BackupOptions{IgnoreSpace: ignoreSpace}) res.Targets = append(res.Targets, r) recordFleetTargetEvent(ctx, exec, r) + // A refused prune is a stored backup and a retention that has stopped: not + // something to find only in one target's JSON. + for _, w := range r.Warnings { + res.Warnings = append(res.Warnings, fleetLabel(r)+": "+w) + } switch r.Status { case backup.TargetOK: fmt.Fprintf(out, "OK %s (%d bytes)\n", fleetLabel(r), r.SizeBytes) @@ -205,6 +234,9 @@ func recordFleetTargetEvent(ctx context.Context, exec ssh.Executor, r backup.Fle detail := backupRunDetail(r.Result, kind, r.Accessory) detail["policy_managed"] = true detail["pruned"] = r.Pruned + if len(r.Warnings) > 0 { + detail["warnings"] = r.Warnings + } var dbs []string for _, o := range r.Objects { if o.Database != "" { @@ -314,6 +346,9 @@ type backupPolicyDTO struct { HeartbeatHosts []string `json:"heartbeat_hosts,omitempty"` Recipient string `json:"recipient,omitempty"` Encrypted bool `json:"encrypted"` + AllowEmpty bool `json:"allow_empty,omitempty"` + TmpDir string `json:"tmp_dir,omitempty"` + TargetDeadline string `json:"target_deadline,omitempty"` } func newServerBackupPolicyCmd(flags *Flags) *cobra.Command { @@ -334,11 +369,11 @@ func newServerBackupPolicyCmd(flags *Flags) *cobra.Command { } type policySetFlags struct { - stdin, heartbeatStdin, noRecipient, clearHeartbeats, volumes bool - interval, recipient string - keepLast, keepDaily, keepWeekly, keepMonthly, maxAgeDays int - volumeMaxGB float64 - excludes, adopts, removeAdopted []string + stdin, heartbeatStdin, noRecipient, clearHeartbeats, volumes, allowEmpty bool + interval, recipient, tmpDir, targetDeadline string + keepLast, keepDaily, keepWeekly, keepMonthly, maxAgeDays int + volumeMaxGB float64 + excludes, adopts, removeAdopted []string } func newServerBackupPolicySetCmd(flags *Flags) *cobra.Command { @@ -381,6 +416,9 @@ func newServerBackupPolicySetCmd(flags *Flags) *cobra.Command { cmd.Flags().BoolVar(&f.noRecipient, "no-recipient", false, "stop encrypting new objects") cmd.Flags().BoolVar(&f.heartbeatStdin, "heartbeat-stdin", false, "read heartbeat URLs from stdin, one per line (replaces the list)") cmd.Flags().BoolVar(&f.clearHeartbeats, "clear-heartbeats", false, "remove every heartbeat URL") + cmd.Flags().BoolVar(&f.allowEmpty, "allow-empty", false, "a run with nothing to back up is not a failure (a server with nothing on it yet)") + cmd.Flags().StringVar(&f.tmpDir, "tmp-dir", "", "build backups on this disk instead of /var/tmp") + cmd.Flags().StringVar(&f.targetDeadline, "target-deadline", "", "deadline of one target (default 6h)") return cmd } @@ -430,6 +468,15 @@ func applyPolicyFlags(cmd *cobra.Command, p *backup.FleetPolicy, f *policySetFla if ch("volume-max-gb") { p.VolumeMaxGB = f.volumeMaxGB } + if ch("allow-empty") { + p.AllowEmpty = f.allowEmpty + } + if ch("tmp-dir") { + p.TmpDir = f.tmpDir + } + if ch("target-deadline") { + p.TargetDeadline = f.targetDeadline + } if ch("recipient") && ch("no-recipient") { return fmt.Errorf("--recipient and --no-recipient are mutually exclusive") } @@ -486,11 +533,20 @@ func readHeartbeatLines(r io.Reader) ([]string, error) { func runServerBackupPolicySet(cmd *cobra.Command, flags *Flags, f *policySetFlags) error { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer cancel() + var err error if f.stdin && f.heartbeatStdin { return fmt.Errorf("--stdin and --heartbeat-stdin both read stdin: put heartbeat_urls in the JSON") } - if f.stdin && cmd.Flags().NFlag() > 1 { - return fmt.Errorf("--stdin takes the whole policy: combine it with no other policy flags") + if f.stdin { + // LocalNonPersistentFlags: --host and friends are not policy flags. + cmd.LocalNonPersistentFlags().Visit(func(fl *pflag.Flag) { + if fl.Name != "stdin" && fl.Name != "no-recipient" && err == nil { + err = fmt.Errorf("--stdin takes the whole policy: combine it with no policy flag but --no-recipient") + } + }) + if err != nil { + return err + } } var incoming *backup.FleetPolicy var heartbeats []string @@ -529,12 +585,23 @@ func runServerBackupPolicySet(cmd *cobra.Command, flags *Flags, f *policySetFlag func setBackupPolicyOn(ctx context.Context, cmd *cobra.Command, executor ssh.Executor, flags *Flags, f *policySetFlags, incoming *backup.FleetPolicy, heartbeats []string) error { stored, err := backup.ReadFleetPolicy(ctx, executor) + if errors.Is(err, backup.ErrPolicyHidden) { + // This user cannot see /etc/teploy, and is setting a policy of its own in + // ~/.config: that is the way out of the refusal, not another refusal. + fmt.Fprintf(os.Stderr, "Warning: %v\nThis user's own policy is stored in ~/.config/teploy and applies to its own runs.\n", err) + stored, err = backup.FleetPolicy{}, nil + } if err != nil { return err } p := stored if incoming != nil { p = *incoming + // A machine-written policy that omits the key must not switch encryption + // off as a side effect. + if stored.Recipient != "" && p.Recipient == "" && !f.noRecipient { + return fmt.Errorf("this policy has no recipient, which would stop encrypting new backups on a server that encrypts them; nothing was changed. Send the recipient, or pass --no-recipient if that is the intent") + } } else { if err := applyPolicyFlags(cmd, &p, f); err != nil { return err @@ -547,6 +614,9 @@ func setBackupPolicyOn(ctx context.Context, cmd *cobra.Command, executor ssh.Exe if err := p.Validate(); err != nil { return err } + if stored.Recipient != "" && p.Recipient == "" { + fmt.Fprintln(os.Stderr, "Warning: encryption is now OFF: new backups are stored in plaintext, and app .env files are left out of volume backups") + } if p.Retention.Policy().IsZero() && p.Interval != "" { fmt.Fprintln(os.Stderr, "Warning: this policy has no retention rule, so backups are never pruned") } @@ -617,6 +687,7 @@ func policyDTO(host string, p backup.FleetPolicy, cronSpec string) backupPolicyD Interval: p.Interval, Cron: cronSpec, Retention: p.Retention, Excludes: p.Excludes, Adopted: p.Adopted, Volumes: p.Volumes, VolumeMaxGB: p.VolumeMaxGB, HeartbeatCount: len(p.HeartbeatURLs), Recipient: p.Recipient, Encrypted: p.Recipient != "", + AllowEmpty: p.AllowEmpty, TmpDir: p.TmpDir, TargetDeadline: p.TargetDeadline, } for _, h := range backup.MaskedHeartbeats(p.HeartbeatURLs) { d.HeartbeatHosts = append(d.HeartbeatHosts, strings.TrimSuffix(h, "/...")) @@ -642,6 +713,15 @@ func printPolicy(p backup.FleetPolicy, cronSpec string) { fmt.Printf(" (cap %.2f GB per archive)", p.VolumeMaxGB) } fmt.Println() + if p.AllowEmpty { + fmt.Println(" Nothing to back up: allowed (--allow-empty)") + } + if p.TmpDir != "" { + fmt.Printf(" Builds backups in: %s\n", p.TmpDir) + } + if p.TargetDeadline != "" { + fmt.Printf(" Deadline per target: %s\n", p.TargetDeadline) + } for _, a := range p.Adopted { dbs := "all" if !a.Databases.All { @@ -1071,43 +1151,80 @@ func applyAdoptedEngine(ctx context.Context, client *backup.Client, exec ssh.Exe // ---- backup before destructive operations ---- +// preBackupScope says what a safety backup must cover. +type preBackupScope struct { + // Database scopes a restore's backup to what it replaces ("" = the primary). + Database string + // All is for an operation that destroys the whole accessory: purge and upgrade. + // Every database must be in the backup, or the operation is refused. + All bool + // IgnoreSpace carries --ignore-space-check: a restore is sometimes how a full + // disk gets fixed. + IgnoreSpace bool +} + +const noBackupHint = "Fix it, or pass --no-backup to go ahead without one (a restore over corrupt data, where the dump cannot succeed, is what that flag is for)" + +// noSuchContainer is the one inspect failure that means there is nothing there. A +// docker blip must not read as nothing to protect. +func noSuchContainer(err error) bool { + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "no such object") || strings.Contains(msg, "no such container") +} + // preOperationBackup stores a copy of what op is about to destroy. A failure // refuses the operation and says exactly what is at stake; --no-backup is the // operator's explicit way past it (a restore is sometimes the fix for what is -// breaking the backup, and the local safety copy still exists). Nothing to -// protect (no such container) is not a failure. -func preOperationBackup(ctx context.Context, client *backup.Client, exec ssh.Executor, app, accessory, database string, cfg backup.S3Config, op string) (string, error) { +// breaking the backup, and the local safety copy still exists). Nothing to protect +// (no such container, a database already gone) is not a failure. +func preOperationBackup(ctx context.Context, client *backup.Client, exec ssh.Executor, app, accessory string, cfg backup.S3Config, op string, scope preBackupScope) (string, error) { var res backup.BackupResult var err error if accessory == "" { // An empty or missing tree has nothing to protect, and restoring into one - // is exactly what must not be refused. + // is exactly what must not be refused. An unreadable one is not empty. d := ssh.ShellQuote("/deployments/" + app + "/volumes") - if out, lsErr := exec.Run(ctx, "if [ -n \"$(ls -A "+d+" 2>/dev/null)\" ]; then echo yes; fi"); lsErr == nil && strings.TrimSpace(out) != "yes" { + out, lsErr := exec.Run(ctx, "if [ ! -e "+d+" ]; then echo empty; elif x=$(ls -A "+d+"); then if [ -n \"$x\" ]; then echo yes; else echo empty; fi; else exit 1; fi") + if lsErr != nil { + return "", fmt.Errorf("could not read %s's volumes before %s, so nothing was changed: %w\n%s", app, op, lsErr, noBackupHint) + } + if strings.TrimSpace(out) != "yes" { return "", nil } - res, err = client.BackupVolumesWith(ctx, app, cfg, backup.BackupOptions{}) + res, err = client.BackupVolumesWith(ctx, app, cfg, backup.BackupOptions{IgnoreSpace: scope.IgnoreSpace, SafetyCopy: true}) recordPreBackupEvent(ctx, exec, app, accessory, res, err, op) if err != nil { - return "", fmt.Errorf("the automatic backup before %s failed, so nothing was changed: %w\nFix it, or pass --no-backup to go ahead without one", op, err) + return "", fmt.Errorf("the automatic backup before %s failed, so nothing was changed: %w\n%s", op, err, noBackupHint) } return res.ID, nil } image, env, inspErr := client.InspectAccessory(ctx, app, accessory) if inspErr != nil { - return "", nil + if noSuchContainer(inspErr) { + return "", nil + } + return "", fmt.Errorf("could not inspect %s before %s, so nothing was changed: %w\n%s", backup.ContainerName(app, accessory), op, inspErr, noBackupHint) } - // Only what op will replace, by name: a database that is already gone has - // nothing to protect, and restoring it is exactly what must not be refused. - if database == "" { - database = client.PrimaryDatabase(ctx, app, accessory, image, env) + opts := backup.BackupOptions{IgnoreSpace: scope.IgnoreSpace, SafetyCopy: true} + if scope.All { + opts.AllDatabases = true + } else { + // Only what op will replace, by name: a database that is already gone has + // nothing to protect, and restoring it is exactly what must not be refused. + opts.SingleDatabase = scope.Database + if opts.SingleDatabase == "" { + opts.SingleDatabase = client.PrimaryDatabase(ctx, app, accessory, image, env) + } } - res, err = client.AccessoryBackupWith(ctx, app, accessory, image, env, cfg, backup.BackupOptions{SingleDatabase: database}) + res, err = client.AccessoryBackupWith(ctx, app, accessory, image, env, cfg, opts) if err != nil || len(res.Objects) > 0 { recordPreBackupEvent(ctx, exec, app, accessory, res, err, op) } if err != nil { - return "", fmt.Errorf("the automatic backup of %s before %s failed, so nothing was changed: %w\nFix it, or pass --no-backup to go ahead without one", backup.ContainerName(app, accessory), op, err) + return "", fmt.Errorf("the automatic backup of %s before %s failed, so nothing was changed: %w\n%s", backup.ContainerName(app, accessory), op, err, noBackupHint) + } + if scope.All && len(res.UncoveredDatabases) > 0 { + return "", fmt.Errorf("the automatic backup of %s before %s does not cover %s (names the dump tools cannot address), so nothing was changed\n%s", backup.ContainerName(app, accessory), op, strings.Join(res.UncoveredDatabases, ", "), noBackupHint) } if len(res.Objects) == 0 { return "", nil @@ -1125,6 +1242,37 @@ func recordPreBackupEvent(ctx context.Context, exec ssh.Executor, app, accessory recordAppEvent(ctx, exec, app, events.KindBackupRun, err, detail) } +// placeIdentity puts a local identity file on the server for one operation, in a +// private directory removed by the returned cleanup. A backup encrypted by another +// host can be read no other way; that is the one time the recovery key leaves the +// owner's machine, and only when asked for by flag. The janitor and doctor know the +// directory, if a kill leaves it behind. +func placeIdentity(ctx context.Context, exec ssh.Executor, identity string) (string, func(), error) { + none := func() {} + if identity == "" { + return "", none, nil + } + body, err := os.ReadFile(identity) + if err != nil { + return "", none, fmt.Errorf("reading the identity file: %w", err) + } + if _, err := age.ParseIdentities(strings.NewReader(string(body))); err != nil { + return "", none, fmt.Errorf("%s is not an age identity file: %w", identity, err) + } + out, err := exec.Run(ctx, "umask 077; mktemp -d /var/tmp/teploy-dr-identity.XXXXXXXX") + if err != nil { + return "", none, fmt.Errorf("creating a private directory for the identity: %w", err) + } + dir := strings.TrimSpace(out) + cleanup := func() { exec.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(dir)) } + path := dir + "/identity" + if err := exec.Upload(ctx, strings.NewReader(string(body)), path, "0600"); err != nil { + cleanup() + return "", none, fmt.Errorf("placing the identity: %w", err) + } + return path, cleanup, nil +} + // resolvePreBackupTarget is the S3 config the safety backups use: the server's // stored target. No target is a refusal naming the way out. func resolvePreBackupTarget(ctx context.Context, exec ssh.Executor, op string) (backup.S3Config, error) { @@ -1154,7 +1302,10 @@ func backupBeforeDeployHook(ctx context.Context, exec ssh.Executor, appCfg *conf for _, name := range sortedAccessoryNames(appCfg.Accessories) { image, env, err := client.InspectAccessory(ctx, appCfg.App, name) if err != nil { - continue + if noSuchContainer(err) { + continue + } + return fmt.Errorf("could not inspect %s before this deploy's pre_deploy hook: %w\n%s", backup.ContainerName(appCfg.App, name), err, noBackupHint) } det, err := client.ResolveEngine(ctx, appCfg.App, name, image, env) if err != nil || !det.IsDB() { @@ -1166,11 +1317,13 @@ func backupBeforeDeployHook(ctx context.Context, exec ssh.Executor, appCfg *conf } haveCfg = true } - id, err := preOperationBackup(ctx, client, exec, appCfg.App, name, "", cfg, "this deploy's pre_deploy hook") + id, err := preOperationBackup(ctx, client, exec, appCfg.App, name, cfg, "this deploy's pre_deploy hook", preBackupScope{All: true}) if err != nil { return err } - fmt.Fprintf(out, "Backed up %s before the pre_deploy hook: %s\n", backup.ContainerName(appCfg.App, name), id) + if id != "" { + fmt.Fprintf(out, "Backed up %s before the pre_deploy hook: %s\n", backup.ContainerName(appCfg.App, name), id) + } } return nil } diff --git a/internal/cli/backup_fleet_test.go b/internal/cli/backup_fleet_test.go index e1d9910..5a23639 100644 --- a/internal/cli/backup_fleet_test.go +++ b/internal/cli/backup_fleet_test.go @@ -90,28 +90,38 @@ func TestRestorePromptNamesWhatIsReplaced(t *testing.T) { } func TestMergePreviousPrefixes(t *testing.T) { - old := backup.Target{Bucket: "b", Endpoint: "e", Prefix: "", PreviousPrefixes: []backup.PreviousPrefix{{Prefix: "older"}}} - prev, lost := mergePreviousPrefixes(old, "b", "e", "srv1", false, false) - if lost || len(prev) != 2 || prev[0].Prefix != "older" || prev[1].Prefix != "" || prev[1].Owned { - t.Fatalf("%+v %v", prev, lost) + old := backup.Target{Bucket: "b", Endpoint: "e", Prefix: "mine", PreviousPrefixes: []backup.PreviousPrefix{{Prefix: ""}, {Prefix: "older", Owned: true}}} + // owned applies only to the prefix being left, never to the shared root. + prev, lost, err := mergePreviousPrefixes(old, "b", "e", "srv1", true, false, nil) + if err != nil || lost || len(prev) != 3 { + t.Fatalf("%+v %v %v", prev, lost, err) + } + for _, p := range prev { + want := p.Prefix == "mine" || p.Prefix == "older" + if p.Owned != want { + t.Fatalf("%q owned=%v: %+v", p.Prefix, p.Owned, prev) + } } - // Moving back to an old prefix takes it off the list. - prev, _ = mergePreviousPrefixes(backup.Target{Bucket: "b", Endpoint: "e", Prefix: "srv1", PreviousPrefixes: []backup.PreviousPrefix{{Prefix: ""}}}, "b", "e", "", false, false) - if len(prev) != 1 || prev[0].Prefix != "srv1" { - t.Fatalf("%+v", prev) + // An earlier one is claimed by name, and only one that exists. + prev, _, err = mergePreviousPrefixes(old, "b", "e", "mine", false, false, []string{""}) + if err != nil || len(prev) != 2 || !prev[0].Owned { + t.Fatalf("%+v %v", prev, err) + } + if _, _, err := mergePreviousPrefixes(old, "b", "e", "mine", false, false, []string{"never"}); err == nil { + t.Fatal("claiming an unknown prefix must be refused") } - // Same prefix: no new history, and ownership can be declared. - prev, _ = mergePreviousPrefixes(backup.Target{Bucket: "b", Endpoint: "e", Prefix: "srv1", PreviousPrefixes: []backup.PreviousPrefix{{Prefix: ""}}}, "b", "e", "srv1", true, false) - if len(prev) != 1 || !prev[0].Owned { + // Moving back takes it off the list. + prev, _, _ = mergePreviousPrefixes(backup.Target{Bucket: "b", Endpoint: "e", Prefix: "srv1", PreviousPrefixes: []backup.PreviousPrefix{{Prefix: ""}}}, "b", "e", "", false, false, nil) + if len(prev) != 1 || prev[0].Prefix != "srv1" { t.Fatalf("%+v", prev) } - if prev, lost := mergePreviousPrefixes(old, "other", "e", "x", false, false); !lost || prev != nil { + if prev, lost, _ := mergePreviousPrefixes(old, "other", "e", "x", false, false, nil); !lost || prev != nil { t.Fatalf("a new bucket starts fresh: %+v %v", prev, lost) } - if prev, lost := mergePreviousPrefixes(old, "b", "e", "x", false, true); lost || prev != nil { + if prev, lost, _ := mergePreviousPrefixes(old, "b", "e", "x", false, true, nil); lost || prev != nil { t.Fatalf("forget: %+v %v", prev, lost) } - if prev, lost := mergePreviousPrefixes(backup.Target{}, "b", "e", "x", false, false); lost || prev != nil { + if prev, lost, _ := mergePreviousPrefixes(backup.Target{}, "b", "e", "x", false, false, nil); lost || prev != nil { t.Fatalf("first target: %+v %v", prev, lost) } } @@ -266,7 +276,7 @@ func TestRunFleetNoTargetHeartbeatsDown(t *testing.T) { defer srv.Close() dir := t.TempDir() mock := ssh.NewMockExecutor("h", - ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","interval":"1h","heartbeat_urls":["` + srv.URL + `/api/push/SECRET"]}`}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","interval":"1h","heartbeat_urls":["` + srv.URL + `/api/push/SECRET"]}`}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "absent\n"}, ssh.MockCommand{Match: "python3 -c", Output: `{"seq":1,"ts":"2026-01-01T00:00:00Z"}`}, ) @@ -289,7 +299,7 @@ func TestRunFleetNoTargetHeartbeatsDown(t *testing.T) { } func TestRunFleetWithNoPolicyFails(t *testing.T) { - mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "absent\n"}) + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "absent\n"}) res, err := runFleet(context.Background(), mock, io.Discard, false, nil) if err == nil || res.Status != backup.RunFailed || !strings.Contains(res.Error, "backup-policy set") { t.Fatalf("%v %+v", err, res) @@ -308,7 +318,7 @@ func TestRunFleetStandsDownOnOverlap(t *testing.T) { } defer lock.Release() mock := ssh.NewMockExecutor("h", - ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","heartbeat_urls":["` + srv.URL + `/hb"]}`}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","heartbeat_urls":["` + srv.URL + `/hb"]}`}, ) res, err := runFleet(context.Background(), mock, io.Discard, false, nil) if err != nil || res.Status != "skipped" { @@ -332,7 +342,7 @@ func TestRunFleetPartialFailure(t *testing.T) { return string(b) } mock := ssh.NewMockExecutor("h", - ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","interval":"1h","retention":{"keep_last":1},"heartbeat_urls":["` + srv.URL + `/hb"]}`}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","interval":"1h","retention":{"keep_last":1},"heartbeat_urls":["` + srv.URL + `/hb"]}`}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"b","region":"r"}`}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "for d in", Output: ""}, @@ -381,7 +391,7 @@ func TestPreOperationBackupRefuses(t *testing.T) { ssh.MockCommand{Match: "python3 -c", Output: `{"seq":1,"ts":"2026-01-01T00:00:00Z"}`}, ) client := backup.NewClient(mock, io.Discard) - _, err := preOperationBackup(context.Background(), client, mock, "a", "db", "", backup.S3Config{Bucket: "b", Region: "r"}, "this restore") + _, err := preOperationBackup(context.Background(), client, mock, "a", "db", backup.S3Config{Bucket: "b", Region: "r"}, "this restore", preBackupScope{}) if err == nil || !strings.Contains(err.Error(), "--no-backup") || !strings.Contains(err.Error(), "nothing was changed") { t.Fatalf("err = %v", err) } @@ -389,8 +399,8 @@ func TestPreOperationBackupRefuses(t *testing.T) { // No such container: nothing to protect, not a refusal. func TestPreOperationBackupNothingToProtect(t *testing.T) { - mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "docker inspect -f", Err: errors.New("exit status 1: no such object")}) - id, err := preOperationBackup(context.Background(), backup.NewClient(mock, io.Discard), mock, "a", "db", "", backup.S3Config{Bucket: "b", Region: "r"}, "this restore") + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "docker inspect -f", Err: errors.New("exit status 1: Error: No such object: a-db")}) + id, err := preOperationBackup(context.Background(), backup.NewClient(mock, io.Discard), mock, "a", "db", backup.S3Config{Bucket: "b", Region: "r"}, "this restore", preBackupScope{}) if err != nil || id != "" { t.Fatalf("%q %v", id, err) } @@ -414,7 +424,7 @@ func TestPreOperationBackupOfADroppedDatabaseIsNotARefusal(t *testing.T) { ssh.MockCommand{Match: "aws s3api", Output: `{"ContentLength":4096,"Metadata":{"sha256":"abc"}}`}, ssh.MockCommand{Match: "python3 -c", Output: `{"seq":1,"ts":"2026-01-01T00:00:00Z"}`}, ) - id, err := preOperationBackup(context.Background(), backup.NewClient(mock, io.Discard), mock, "a", "db", "", backup.S3Config{Bucket: "b", Region: "r"}, "this restore") + id, err := preOperationBackup(context.Background(), backup.NewClient(mock, io.Discard), mock, "a", "db", backup.S3Config{Bucket: "b", Region: "r"}, "this restore", preBackupScope{}) if err != nil || id != "" { t.Fatalf("a dropped database must not refuse the restore that brings it back: %q %v", id, err) } @@ -448,7 +458,7 @@ func TestEnsureServerPathCompletesCronPath(t *testing.T) { func TestSetBackupPolicyFromStdin(t *testing.T) { t.Setenv("TEPLOY_SERVER_BINARY", "/opt/teploy") mock := ssh.NewMockExecutor("srv", - ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "absent\n"}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "absent\n"}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"b","region":"r"}`}, ssh.MockCommand{Match: "sched_daemon=no", Output: "daemon=yes\nenabled=enabled\ncrontab=yes\nteploy_entries=0\n"}, ssh.MockCommand{Match: "which age", Output: "/usr/bin/age"}, @@ -492,7 +502,7 @@ func TestSetBackupPolicyFromStdin(t *testing.T) { // Refused before anything is stored. bad := backup.FleetPolicy{Adopted: []backup.AdoptedEntry{{Container: "x'; rm -rf /", Databases: backup.DatabaseSelector{All: true}}}} - refused := ssh.NewMockExecutor("srv", ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "absent\n"}) + refused := ssh.NewMockExecutor("srv", ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "absent\n"}) if err := setBackupPolicyOn(context.Background(), &cobra.Command{}, refused, &Flags{}, &policySetFlags{stdin: true}, &bad, nil); err == nil { t.Fatal("a hostile container name must be refused") } @@ -500,3 +510,60 @@ func TestSetBackupPolicyFromStdin(t *testing.T) { t.Fatalf("refused policy touched the server: %v", refused.Calls) } } + +// Per-database rows reach Dash only when asked for: it restores a row by id and +// accessory alone, and restoring a database row that way replaces the primary. +func TestFilterDatabaseRows(t *testing.T) { + rows := []backup.ListedBackup{{ID: "1"}, {ID: "1", Database: "analytics"}} + got, hidden := filterDatabaseRows(rows, false) + if len(got) != 1 || hidden != 1 || got[0].Database != "" { + t.Fatalf("%+v %d", got, hidden) + } + got, hidden = filterDatabaseRows(rows, true) + if len(got) != 2 || hidden != 0 { + t.Fatalf("%+v %d", got, hidden) + } + if got, _ := filterDatabaseRows(nil, false); got == nil { + t.Fatal("must encode as []") + } +} + +// A --no-credentials target has nothing to decrypt. +func TestTargetS3StoredOnlyWithCredentials(t *testing.T) { + if cfg := TargetS3(backup.Target{Bucket: "b", Path: "/etc/teploy/backup-target.json"}); cfg.StoredTarget != "" { + t.Fatalf("%+v", cfg) + } + if cfg := TargetS3(backup.Target{Bucket: "b", Path: "/p", HasCredentials: true}); cfg.StoredTarget != "/p" { + t.Fatalf("%+v", cfg) + } +} + +// Zero targets is a failure unless the policy allows it. +func TestRunFleetZeroTargets(t *testing.T) { + var got []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { got = append(got, r.URL.RequestURI()) })) + defer srv.Close() + dir := t.TempDir() + mk := func(extra string) *ssh.MockExecutor { + return ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","interval":"1h"` + extra + `,"heartbeat_urls":["` + srv.URL + `/hb"]}`}, + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"b","region":"r"}`}, + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "for d in", Output: ""}, + ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: ""}, + ssh.MockCommand{Match: "crontab -l", Output: ""}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq":1,"ts":"2026-01-01T00:00:00Z"}`}, + ) + } + res, err := runFleet(context.Background(), mk(""), io.Discard, false, nil) + if err == nil || res.Status != backup.RunFailed || !strings.Contains(res.Error, "allow_empty") { + t.Fatalf("%v %+v", err, res) + } + if !strings.Contains(got[len(got)-1], "status=down") { + t.Fatalf("%v", got) + } + res, err = runFleet(context.Background(), mk(`,"allow_empty":true`), io.Discard, false, nil) + if err != nil || res.Status != backup.RunOK { + t.Fatalf("%v %+v", err, res) + } +} diff --git a/internal/cli/backup_target.go b/internal/cli/backup_target.go index 6d6bce0..6c4a770 100644 --- a/internal/cli/backup_target.go +++ b/internal/cli/backup_target.go @@ -43,6 +43,85 @@ func newServerBackupTargetCmd(flags *Flags) *cobra.Command { cmd.AddCommand(newServerBackupTargetSetCmd(flags)) cmd.AddCommand(newServerBackupTargetShowCmd(flags)) cmd.AddCommand(newServerBackupTargetUnsetCmd(flags)) + cmd.AddCommand(newServerBackupTargetOwnersCmd(flags)) + return cmd +} + +// newServerBackupTargetOwnersCmd shows and clears namespace markers: a prune +// refuses a namespace holding another server's, so a rebuilt server's old self +// would stop retention there forever, and growth is the only symptom. +func newServerBackupTargetOwnersCmd(flags *Flags) *cobra.Command { + var clear string + cmd := &cobra.Command{ + Use: "owners", + Short: "List which servers have written under this target; --clear removes one's markers", + Long: "Each server leaves a marker in every namespace it writes to, and a prune\n" + + "refuses a namespace holding another server's. --clear removes a\n" + + "server's markers, under the current and previous prefixes: for a server that\n" + + "no longer exists, or this one before a rebuild. Backups are not touched. Do not\n" + + "clear a server that still writes here: give it its own --prefix.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + host, user, key, err := configResolveServerForTarget(flags) + if err != nil { + return err + } + executor, err := ssh.Connect(ctx, ssh.ConnectConfig{Host: host, User: user, KeyPath: key}) + if err != nil { + return err + } + defer executor.Close() + target, err := backup.ReadTarget(ctx, executor) + if err != nil { + return err + } + if !TargetConfigured(target) { + return fmt.Errorf("no backup target is configured for this server") + } + client := backup.NewClient(executor, io.Discard) + cfg := TargetS3(target) + locs := []backup.S3Config{cfg} + for _, p := range target.PreviousPrefixes { + l := cfg + l.Prefix, l.PreviousPrefixes = p.Prefix, nil + locs = append(locs, l) + } + if clear != "" { + total := 0 + for _, l := range locs { + removed, err := client.ClearNamespaceMarkers(ctx, l, clear) + total += len(removed) + if err != nil { + return err + } + } + fmt.Printf("Removed %d marker(s) of %s\n", total, clear) + return nil + } + markers := []backup.NamespaceMarker{} + for _, l := range locs { + m, err := client.ListNamespaceMarkers(ctx, l) + if err != nil { + return err + } + markers = append(markers, m...) + } + if flags.JSON { + return json.NewEncoder(os.Stdout).Encode(map[string]any{"machine_interface": MachineInterface, "host": executor.Host(), "markers": markers}) + } + for _, m := range markers { + who := "another server" + if m.Self { + who = "this server" + } + fmt.Printf("%-24s %s (%s) %s\n", m.Namespace, m.Fingerprint, who, m.Key) + } + return nil + }, + } + cmd.Flags().StringVar(&clear, "clear", "", "remove this fingerprint's markers") return cmd } @@ -72,6 +151,7 @@ func newServerBackupTargetSetCmd(flags *Flags) *cobra.Command { noCredentials bool prevOwned bool dropPrevious bool + claimPrevious []string ) cmd := &cobra.Command{ Use: "set", @@ -85,11 +165,13 @@ func newServerBackupTargetSetCmd(flags *Flags) *cobra.Command { "Give each server its own --prefix when servers share a bucket. Changing the\n" + "prefix keeps the old one on record: list, verify and restore still find the\n" + "objects there. They are not pruned, because they may be another server's;\n" + - "--previous-prefix-owned says only this server ever wrote there, and lets them\n" + - "age out under the policy's retention.", + "--previous-prefix-owned says only this server ever wrote under the prefix\n" + + "being left, and --claim-previous-prefix says it of an earlier one, which lets\n" + + "them age out under the policy's retention (never where another server's\n" + + "marker says otherwise).", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - return runServerBackupTargetSet(flags, bucket, region, endpoint, prefix, noCredentials, prevOwned, dropPrevious) + return runServerBackupTargetSet(flags, bucket, region, endpoint, prefix, noCredentials, prevOwned, dropPrevious, claimPrevious) }, } cmd.Flags().StringVar(&bucket, "bucket", "", "S3 bucket name") @@ -97,7 +179,8 @@ func newServerBackupTargetSetCmd(flags *Flags) *cobra.Command { cmd.Flags().StringVar(&endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2)") cmd.Flags().StringVar(&prefix, "prefix", "", "key prefix under the bucket (optional)") cmd.Flags().BoolVar(&noCredentials, "no-credentials", false, "store no credentials — use the server's ambient AWS credential chain") - cmd.Flags().BoolVar(&prevOwned, "previous-prefix-owned", false, "the previous prefix holds only this server's objects: let retention age them out") + cmd.Flags().BoolVar(&prevOwned, "previous-prefix-owned", false, "the prefix being left holds only this server's objects: let retention age them out") + cmd.Flags().StringArrayVar(&claimPrevious, "claim-previous-prefix", nil, "an earlier prefix (from `backup-target show`) that holds only this server's objects") cmd.Flags().BoolVar(&dropPrevious, "forget-previous-prefixes", false, "stop looking under previous prefixes (their objects are left in the bucket)") return cmd } @@ -106,29 +189,48 @@ func newServerBackupTargetSetCmd(flags *Flags) *cobra.Command { // prefix being left joins the list, the prefix being entered leaves it. A new // bucket or endpoint starts fresh, since the old location is not reachable from // the new config. -func mergePreviousPrefixes(old backup.Target, bucket, endpoint, prefix string, owned, drop bool) (prev []backup.PreviousPrefix, lost bool) { +// +// owned applies only to the prefix being left: earlier ones, the shared root +// among them, may hold other servers' objects, and one word about one move must +// not make them deletable. An earlier one is claimed by name. +func mergePreviousPrefixes(old backup.Target, bucket, endpoint, prefix string, owned, drop bool, claim []string) (prev []backup.PreviousPrefix, lost bool, err error) { if !TargetConfigured(old) { - return nil, false + if len(claim) > 0 { + return nil, false, fmt.Errorf("--claim-previous-prefix: this server has no previous prefixes") + } + return nil, false, nil } if old.Bucket != bucket || old.Endpoint != endpoint { - return nil, true + return nil, true, nil } if drop { - return nil, false + return nil, false, nil } for _, p := range old.PreviousPrefixes { if p.Prefix != prefix && p.Prefix != old.Prefix { - p.Owned = p.Owned || owned prev = append(prev, p) } } if old.Prefix != prefix { prev = append(prev, backup.PreviousPrefix{Prefix: old.Prefix, Owned: owned}) + } else if owned { + fmt.Fprintln(os.Stderr, "Warning: --previous-prefix-owned applies to the prefix being left and this set leaves none; use --claim-previous-prefix for an earlier one") + } + for _, c := range claim { + found := false + for i := range prev { + if prev[i].Prefix == c { + prev[i].Owned, found = true, true + } + } + if !found { + return nil, false, fmt.Errorf("--claim-previous-prefix %q is not one of this target's previous prefixes", c) + } } - return prev, false + return prev, false, nil } -func runServerBackupTargetSet(flags *Flags, bucket, region, endpoint, prefix string, noCredentials, prevOwned, dropPrevious bool) error { +func runServerBackupTargetSet(flags *Flags, bucket, region, endpoint, prefix string, noCredentials, prevOwned, dropPrevious bool, claimPrevious []string) error { if bucket == "" { return fmt.Errorf("--bucket is required") } @@ -185,7 +287,10 @@ func runServerBackupTargetSet(flags *Flags, bucket, region, endpoint, prefix str } t := backup.Target{Bucket: bucket, Region: region, Endpoint: endpoint, Prefix: prefix, HasCredentials: credsArmor != ""} var lostOld bool - t.PreviousPrefixes, lostOld = mergePreviousPrefixes(old, bucket, endpoint, prefix, prevOwned, dropPrevious) + t.PreviousPrefixes, lostOld, err = mergePreviousPrefixes(old, bucket, endpoint, prefix, prevOwned, dropPrevious, claimPrevious) + if err != nil { + return err + } path, fallback, err := backup.WriteTarget(ctx, executor, t, credsArmor) if err != nil { return err diff --git a/internal/cli/contracts_golden_test.go b/internal/cli/contracts_golden_test.go index 0bd82ba..c879a7f 100644 --- a/internal/cli/contracts_golden_test.go +++ b/internal/cli/contracts_golden_test.go @@ -640,15 +640,18 @@ func TestContractsBackupFleetGolden(t *testing.T) { Policy: backup.StatusPolicy{Configured: true, Path: policy.Path, Interval: "1h", CronInstalled: true, Retention: policy.Retention, Excludes: policy.Excludes, Adopted: policy.Adopted, Volumes: true, VolumeMaxGB: 20, HeartbeatCount: 1, HeartbeatHosts: []string{"https://dash.example.internal"}, Recipient: policy.Recipient}, Encrypted: true, AWSCLI: true, AgeCLI: true, - LastRun: &backup.StatusLastRun{At: at, Status: backup.RunPartial, TargetsOK: 3, TargetsFailed: 1, Omissions: []string{"redis-cache: a database container no policy entry covers (adopt or exclude it)"}}, + LastRun: &backup.StatusLastRun{At: at, Status: backup.RunPartial, TargetsOK: 3, TargetsFailed: 1, Omissions: []string{"redis-cache: a database container no policy entry covers (adopt or exclude it)"}, Warnings: []string{"blog-db: prune: shared backup namespace: another server (aaaaaaaaaaaaaaaa) also stores blog backups"}}, Entries: []backup.StatusEntry{ - {Name: "blog-db", Kind: backup.TargetAccessory, App: "blog", Accessory: "db", Engine: "postgres", Detection: "env", State: "running", Coverage: backup.CoverageCovered, Schedule: "policy", LastSuccess: &at, Newest: &backup.StatusObject{ID: "20261104-031704-1f2e3d4c5b6a7988", Key: "s3://teploy-backups/prod/blog/accessories/db/20261104-031704-1f2e3d4c5b6a7988.pgdump.age", SizeBytes: 1048576, CreatedAt: "2026-11-04T03:17:04Z", Encrypted: true}, LastVerify: &backup.StatusVerify{At: at, OK: true, ID: "20261104-031704-1f2e3d4c5b6a7988", Metric: "tables=42"}}, - {Name: "wordpress-shared-mysql", Kind: backup.TargetAdopted, Accessory: "wordpress-shared-mysql", Engine: "mysql", Detection: "policy", State: "running", Unmanaged: true, Adopted: true, Coverage: backup.CoverageAdopted, Schedule: "policy", LastSuccess: &at, Newest: newest, Databases: []backup.StatusDatabase{{Name: "wp_blog", Newest: newest}}}, - {Name: "omni_postgres", Kind: backup.TargetAdopted, Accessory: "omni_postgres", Engine: "postgres", Detection: "env", State: "running", Unmanaged: true, Adopted: true, Coverage: backup.CoverageFailing, Schedule: "policy", LastFailure: &failedAt, LastError: "dumping omni: exit status 1", UncoveredDatabases: []string{"reports"}}, + {Name: "blog-db", Kind: backup.TargetAccessory, App: "blog", Accessory: "db", Engine: "postgres", Detection: "env", State: "running", Coverage: backup.CoverageCovered, Schedule: "policy", Encrypted: true, LastSuccess: &at, Newest: &backup.StatusObject{ID: "20261104-031704-1f2e3d4c5b6a7988", Key: "s3://teploy-backups/prod/blog/accessories/db/20261104-031704-1f2e3d4c5b6a7988.pgdump.age", SizeBytes: 1048576, CreatedAt: "2026-11-04T03:17:04Z", Encrypted: true}, LastVerify: &backup.StatusVerify{At: at, OK: true, ID: "20261104-031704-1f2e3d4c5b6a7988", Metric: "tables=42"}}, + {Name: "wordpress-shared-mysql", Kind: backup.TargetAdopted, Accessory: "wordpress-shared-mysql", Engine: "mysql", Detection: "policy", State: "running", Unmanaged: true, Adopted: true, Coverage: backup.CoverageAdopted, Schedule: "policy", Encrypted: true, LastSuccess: &at, Newest: newest, Databases: []backup.StatusDatabase{{Name: "wp_blog", Newest: newest}}}, + {Name: "omni_postgres", Kind: backup.TargetAdopted, Accessory: "omni_postgres", Engine: "postgres", Detection: "env", State: "running", Unmanaged: true, Adopted: true, Coverage: backup.CoverageFailing, Schedule: "policy", Encrypted: true, LastFailure: &failedAt, LastError: "dumping omni: exit status 1", UncoveredDatabases: []string{"reports"}}, {Name: "redis-cache", Kind: "unmanaged", Engine: "redis", Detection: "env", State: "running", Unmanaged: true, Coverage: backup.CoverageUncovered, Reason: "no policy entry covers this container (adopt or exclude it)", Schedule: "none"}, - {Name: "blog/volumes", Kind: backup.TargetVolumes, App: "blog", Coverage: backup.CoverageExcluded, Schedule: "policy", SizeBytes: 53687091200, Reason: "50.00 GB is over the policy's volume_max_gb 20.00"}, + {Name: "shop-db", Kind: backup.TargetAccessory, App: "shop", Accessory: "db", Engine: "postgres", Detection: "env", State: "running", Coverage: backup.CoverageStale, Reason: "a run succeeded, but the bucket holds no restorable object for it", Schedule: "own", ScheduleForm: "legacy", LastSuccess: &at}, + {Name: "analytics-ch", Kind: "unmanaged", Engine: "clickhouse-server", Detection: "image-name", State: "running", Unmanaged: true, Coverage: backup.CoverageUncovered, Reason: "teploy has no backup method for clickhouse-server: back it up another way and exclude it", Schedule: "none"}, + {Name: "blog/volumes", Kind: backup.TargetVolumes, App: "blog", Coverage: backup.CoverageExcluded, Schedule: "policy", Encrypted: true, SizeBytes: 53687091200, Reason: "50.00 GB is over the policy's volume_max_gb 20.00"}, }, - UncoveredDatabases: []string{"omni_postgres/reports", "redis-cache"}, + UncoveredDatabases: []string{"analytics-ch", "omni_postgres/reports", "redis-cache"}, + Warnings: []string{"cron marker teploy-accessory-backup:shop:db is a pre-D3 shell line that uploads plaintext whatever the policy says: rewrite it with `teploy server backup-target set`"}, }} writeFixture(t, "backup-status-envelope/valid/fleet.json", status) writeFixture(t, "backup-status-envelope/valid/unconfigured.json", backupStatusDTO{MachineInterface: MachineInterface, Host: "192.0.2.10", ObservedAt: at, FleetStatus: backup.FleetStatus{ diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index 5770692..40bed98 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -987,6 +987,9 @@ func doctorBackupPolicyCheck(ctx context.Context, exec ssh.Executor, now time.Ti sctx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() st, err := backup.ReadFleetStatus(sctx, exec, now) + if errors.Is(err, backup.ErrPolicyHidden) { + return doctorCheck{Name: "backup-policy", Result: doctorFail, Detail: err.Error(), Remediation: "connect as root, give this user passwordless sudo, or run on the server: chmod 711 /etc/teploy. Until then every backup through this user is refused rather than risk plaintext"} + } if err != nil { return doctorCheck{Name: "backup-policy", Result: doctorWarn, Detail: "could not read the backup status: " + err.Error()} } @@ -1032,6 +1035,15 @@ func doctorBackupPolicyCheck(ctx context.Context, exec ssh.Executor, now time.Ti remedies = append(remedies, "printf '\\n' | teploy server backup-policy set --host --heartbeat-stdin") } } + if st.LastRun != nil && len(st.LastRun.Warnings) > 0 { + warns = append(warns, "the last backup run warned: "+strings.Join(st.LastRun.Warnings, "; ")) + remedies = append(remedies, "a refused prune means retention has stopped and storage keeps growing: `teploy server backup-target owners` for a stale marker") + } + for _, w := range st.Warnings { + if strings.Contains(w, "pre-D3") { + warns = append(warns, w) + } + } if st.Target.Configured && st.AWSCLI && !st.Target.Reachable { fails = append(fails, "the backup target did not answer: "+st.Target.Error) remedies = append(remedies, "check the bucket, endpoint and credentials (teploy server backup-target set)") diff --git a/internal/cli/doctor_backup_test.go b/internal/cli/doctor_backup_test.go index 677e274..b4fe2b2 100644 --- a/internal/cli/doctor_backup_test.go +++ b/internal/cli/doctor_backup_test.go @@ -11,16 +11,17 @@ import ( func doctorPolicyMock(policy, target string, tools string, cron, events string) *ssh.MockExecutor { return ssh.NewMockExecutor("h", - ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: policy}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: policy}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: target}, ssh.MockCommand{Match: "command -v aws >/dev/null", Output: tools}, ssh.MockCommand{Match: "crontab -l", Output: cron}, ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: ""}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "( c=$(python3", Output: ""}, + ssh.MockCommand{Match: "aws s3 ls", Output: ""}, ssh.MockCommand{Match: "python3 -c", Output: events}, ssh.MockCommand{Match: "for d in /deployments/*/volumes", Output: ""}, - ssh.MockCommand{Match: "for d in /var/tmp/teploy-backup.", Output: ""}, + ssh.MockCommand{Match: "for d in '/var/tmp'/teploy-backup.", Output: ""}, ) } diff --git a/internal/cli/doctor_test.go b/internal/cli/doctor_test.go index 93a5ba8..bca08a5 100644 --- a/internal/cli/doctor_test.go +++ b/internal/cli/doctor_test.go @@ -41,12 +41,12 @@ func doctorHappyMock() *ssh.MockExecutor { ssh.MockCommand{Match: "sched_daemon=no", Output: "daemon=yes\nenabled=enabled\ncrontab=yes\nteploy_entries=0\n"}, ssh.MockCommand{Match: "for d in /var/tmp/teploy-pre-restore.*", Output: ""}, ssh.MockCommand{Match: "docker ps --filter label=teploy.role=accessory", Output: ""}, - ssh.MockCommand{Match: "for p in /etc/teploy/backup-policy.json", Output: "absent\n"}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "absent\n"}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "absent\n"}, ssh.MockCommand{Match: "command -v aws >/dev/null", Output: "aws\nage\n"}, ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: ""}, ssh.MockCommand{Match: "for d in /deployments/*/volumes", Output: ""}, - ssh.MockCommand{Match: "for d in /var/tmp/teploy-backup", Output: ""}, + ssh.MockCommand{Match: "for d in '/var/tmp'/teploy-backup", Output: ""}, ssh.MockCommand{Match: "python3 -c", Output: ""}, ) } @@ -206,10 +206,11 @@ func TestDoctorHumanTable(t *testing.T) { ssh.MockCommand{Match: "for d in /var/tmp/teploy-pre-restore.*", Output: ""}, ssh.MockCommand{Match: "docker ps --filter label=teploy.role=accessory", Output: ""}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-", Output: "absent\n"}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "absent\n"}, ssh.MockCommand{Match: "command -v aws >/dev/null", Output: "aws\nage\n"}, ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: ""}, ssh.MockCommand{Match: "for d in /deployments/*/volumes", Output: ""}, - ssh.MockCommand{Match: "for d in /var/tmp/teploy-backup", Output: ""}, + ssh.MockCommand{Match: "for d in '/var/tmp'/teploy-backup", Output: ""}, ssh.MockCommand{Match: "python3 -c", Output: ""}, ) report, _ := doctorRun(context.Background(), doctorTestDeps(mock), &Flags{}, "", doctorTestApp(), nil) @@ -726,10 +727,11 @@ func assertDoctorReadOnlyCalls(t *testing.T, calls []string) { "for d in /var/tmp/teploy-pre-restore.*", "docker ps --filter label=teploy.role=accessory", "for p in /etc/teploy/backup-", + "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", "command -v aws >/dev/null", "ids=$(docker ps -aq", "for d in /deployments/*/volumes", - "for d in /var/tmp/teploy-backup", + "for d in '/var/tmp'/teploy-backup", "python3 -c", } for _, call := range calls { diff --git a/internal/cli/dr.go b/internal/cli/dr.go index 3ec4291..1eb04b8 100644 --- a/internal/cli/dr.go +++ b/internal/cli/dr.go @@ -8,7 +8,6 @@ import ( "io" "os" "os/signal" - "strings" "github.com/spf13/cobra" "github.com/useteploy/teploy/internal/backup" @@ -53,12 +52,19 @@ func drStoreFlags(cmd *cobra.Command, bucket, region, endpoint, dir *string) { cmd.Flags().StringVar(region, "region", "", "AWS region (default: the stored target's region, else us-east-1)") cmd.Flags().StringVar(endpoint, "endpoint", "", "S3-compatible endpoint URL (MinIO/B2/R2); creds from TEPLOY_S3_ACCESS_KEY/SECRET_KEY or AWS_* env") cmd.Flags().StringVar(dir, "dir", "", "store bundles in a directory on the server (offline bundles) instead of S3") + cmd.Flags().StringVar(&drPrefix, "prefix", "", "key prefix under --bucket (a stored target has its own)") } // drStore builds the BundleStore from the flags; with neither --bucket nor // --dir the server's stored backup target is used (stored credentials and // all — no bucket or credentials need to travel with the command). func drStore(ctx context.Context, exec ssh.Executor, bucket, region, endpoint, dir string) (backup.BundleStore, error) { + if drPrefix != "" && (bucket == "" || dir != "") { + return nil, fmt.Errorf("--prefix goes with --bucket: the stored target already has its own") + } + if err := backup.ValidatePrefix(drPrefix); err != nil { + return nil, err + } switch { case dir != "" && bucket != "": return nil, fmt.Errorf("choose one bundle target: --bucket (S3) or --dir (server directory), not both") @@ -76,6 +82,7 @@ func drStore(ctx context.Context, exec ssh.Executor, bucket, region, endpoint, d return nil, err } cfg := s3Config(bucket, r, endpoint) + cfg.Prefix = drPrefix if err := backup.ApplyPolicyEncryption(ctx, exec, &cfg); err != nil { return nil, err } @@ -95,32 +102,17 @@ func drStore(ctx context.Context, exec ssh.Executor, bucket, region, endpoint, d return backup.S3BundleStore{S3: cfg}, nil } -// withBundleIdentity puts a local identity file on the server for this restore -// only, in a private directory removed by the returned cleanup. An encrypted -// bundle restored on a host that did not make it can be read no other way; that -// is the one time the recovery key leaves the owner's machine, and only when -// asked for by flag. +// drPrefix is the shared --prefix of the dr commands. +var drPrefix string + +// withBundleIdentity places the recovery key for a bundle restore (placeIdentity): +// an encrypted bundle on a host that did not make it can be read no other way. func withBundleIdentity(ctx context.Context, exec ssh.Executor, store backup.BundleStore, identity string) (backup.BundleStore, string, func(), error) { - none := func() {} - if identity == "" { - return store, "", none, nil - } - body, err := os.ReadFile(identity) - if err != nil { - return nil, "", none, fmt.Errorf("reading the identity file: %w", err) - } - out, err := exec.Run(ctx, "umask 077; mktemp -d /var/tmp/teploy-dr-identity.XXXXXXXX") + path, cleanup, err := placeIdentity(ctx, exec, identity) if err != nil { - return nil, "", none, fmt.Errorf("creating a private directory for the identity: %w", err) - } - dir := strings.TrimSpace(out) - cleanup := func() { exec.Run(context.WithoutCancel(ctx), "rm -rf "+ssh.ShellQuote(dir)) } - path := dir + "/identity" - if err := exec.Upload(ctx, strings.NewReader(string(body)), path, "0600"); err != nil { - cleanup() - return nil, "", none, fmt.Errorf("placing the identity: %w", err) + return nil, "", cleanup, err } - if s3, ok := store.(backup.S3BundleStore); ok { + if s3, ok := store.(backup.S3BundleStore); ok && path != "" { s3.S3.ExtraIdentity = path store = s3 } diff --git a/internal/cli/remove.go b/internal/cli/remove.go index 219ab7a..1b508d3 100644 --- a/internal/cli/remove.go +++ b/internal/cli/remove.go @@ -98,7 +98,8 @@ func purgeInventory(ctx context.Context, exec ssh.Executor, app string) (accesso } } slices.Sort(accessoriesNames) - out, err := exec.Run(ctx, "d="+ssh.ShellQuote("/deployments/"+app+"/volumes")+`; if [ -d "$d" ] && [ -n "$(ls -A "$d" 2>/dev/null)" ]; then echo yes; fi; true`) + // An unreadable tree is an error, not an empty one. + out, err := exec.Run(ctx, "d="+ssh.ShellQuote("/deployments/"+app+"/volumes")+`; if [ ! -e "$d" ]; then echo empty; elif x=$(ls -A "$d"); then if [ -n "$x" ]; then echo yes; else echo empty; fi; else exit 1; fi`) if err != nil { return nil, false, fmt.Errorf("checking volumes: %w", err) } @@ -116,14 +117,19 @@ func backupBeforePurge(ctx context.Context, exec ssh.Executor, app string, accs } client := backup.NewClient(exec, out) for _, acc := range accs { - id, err := preOperationBackup(ctx, client, exec, app, acc, "", cfg, "remove --purge") + // Every database: a purge destroys them all, not just the primary. + id, err := preOperationBackup(ctx, client, exec, app, acc, cfg, "remove --purge", preBackupScope{All: true}) if err != nil { return err } - fmt.Fprintf(out, "Backed up %s first: %s\n", backup.ContainerName(app, acc), id) + if id == "" { + fmt.Fprintf(out, "%s has nothing to back up\n", backup.ContainerName(app, acc)) + } else { + fmt.Fprintf(out, "Backed up every database of %s first: %s\n", backup.ContainerName(app, acc), id) + } } if volumes { - id, err := preOperationBackup(ctx, client, exec, app, "", "", cfg, "remove --purge") + id, err := preOperationBackup(ctx, client, exec, app, "", cfg, "remove --purge", preBackupScope{}) if err != nil { return err } diff --git a/internal/cli/setup.go b/internal/cli/setup.go index fd02083..d563fef 100644 --- a/internal/cli/setup.go +++ b/internal/cli/setup.go @@ -700,7 +700,7 @@ func setupStages() []setupStage { affects: "aws CLI (apt, else AWS's own bundle) and age (apt, if missing) — the backup transport and payload encryption; a failure here warns, since nothing but backups needs them", run: func(ctx context.Context, exec ssh.Executor, w io.Writer, env *setupEnv) error { fmt.Fprintln(w, "Checking aws CLI and age...") - if _, err := exec.Run(ctx, "command -v aws"); err == nil { + if _, err := exec.Run(ctx, "command -v aws >/dev/null && aws --version >/dev/null 2>&1"); err == nil { fmt.Fprintln(w, " aws CLI already installed") } else { fmt.Fprintln(w, " Installing the aws CLI...") From 9dbb45954a3368450269bb408068b9e200fbd28c Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:40:30 -0700 Subject: [PATCH 09/17] test(backup): review regressions on real multi-database accessories Purge and upgrade across databases; a stale schedule store; every database dropped; each series' own retention in a real bucket; the real --previous-prefix-owned and a foreign marker under it; a wiped namespace; owners --clear; a real cron firing with cron's PATH; a non-root user behind a 0700 /etc/teploy, with sudo and without; zero targets; a stuck holder's children; a target over the read-back limit; policy --stdin; clean-host recovery through `backup restore --identity` and `dr restore --identity --bucket --prefix` with no server identity; AWS's installer, live, with a wrong pin. Co-Authored-By: Claude Opus 5.5 --- .../backup/awsinstall_integration_test.go | 45 ++ internal/backup/engines_integration_test.go | 1 + internal/cli/fleet_integration_test.go | 402 +++++++++++++++++- 3 files changed, 432 insertions(+), 16 deletions(-) create mode 100644 internal/backup/awsinstall_integration_test.go diff --git a/internal/backup/awsinstall_integration_test.go b/internal/backup/awsinstall_integration_test.go new file mode 100644 index 0000000..393ccf0 --- /dev/null +++ b/internal/backup/awsinstall_integration_test.go @@ -0,0 +1,45 @@ +//go:build integration + +package backup + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/useteploy/teploy/internal/ssh" +) + +// TestAWSCLIInstallScriptLive runs setup's installer on the fixture for real: AWS's +// bundle, its real signature, a real install, with the pinned key and without it. +// It replaces the VM's aws, so it needs TEPLOY_IT_INSTALL_AWS=1. +func TestAWSCLIInstallScriptLive(t *testing.T) { + if os.Getenv("TEPLOY_IT_INSTALL_AWS") == "" { + t.Skip("set TEPLOY_IT_INSTALL_AWS=1: this reinstalls the fixture's aws CLI") + } + exec := drFixtureEnv(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + exec.Run(ctx, "sudo -n rm -rf /usr/local/aws-cli /usr/local/bin/aws /usr/local/bin/aws_completer; true") + // A key that is not the pin: the real signature must not be accepted. + tampered := strings.ReplaceAll(AWSCLIInstallScript, awsCLIKeyFingerprint, "0000000000000000000000000000000000000000") + res := ssh.RunDetailed(ctx, exec, "sudo -n sh -c "+ssh.ShellQuote(tampered)) + if !res.Failed() || !strings.Contains(string(res.Stderr), "did not verify") { + t.Fatalf("a wrong pin must refuse: exit %d\n%s%s", res.ExitCode, res.Stdout, res.Stderr) + } + if out, _ := exec.Run(ctx, "command -v aws; true"); strings.TrimSpace(out) != "" { + t.Fatalf("installed after a failed verification: %s", out) + } + res = ssh.RunDetailed(ctx, exec, "sudo -n sh -c "+ssh.ShellQuote(AWSCLIInstallScript)) + if res.Failed() { + t.Fatalf("install: exit %d\n%s%s", res.ExitCode, res.Stdout, res.Stderr) + } + // As a user other than root: cron's case, and where u+x modes used to fail. + out, err := exec.Run(ctx, "aws --version") + if err != nil { + t.Fatalf("aws does not run: %v %s", err, out) + } + t.Log(strings.TrimSpace(out)) +} diff --git a/internal/backup/engines_integration_test.go b/internal/backup/engines_integration_test.go index 408abcf..2886268 100644 --- a/internal/backup/engines_integration_test.go +++ b/internal/backup/engines_integration_test.go @@ -36,6 +36,7 @@ import ( const fakeAWS = `#!/bin/sh set -eu root=__ROOT__ +[ "${1:-}" = --version ] && { echo "fake aws"; exit 0; } if [ "$1" = s3api ]; then # head-object --bucket B --key K [--region R] [--endpoint-url E] [ "$2" = head-object ] || exit 1 diff --git a/internal/cli/fleet_integration_test.go b/internal/cli/fleet_integration_test.go index a254eca..aaff9c9 100644 --- a/internal/cli/fleet_integration_test.go +++ b/internal/cli/fleet_integration_test.go @@ -45,16 +45,17 @@ const ( ) type fleetIT struct { - t *testing.T - exec ssh.Executor - ctx context.Context - s3 backup.S3Config - owner string // owner identity file on the VM - pub string // owner recipient - local string // darwin/linux binary for the --host commands - host string - user string - key string + t *testing.T + exec ssh.Executor + ctx context.Context + s3 backup.S3Config + owner string // owner identity file on the VM + ownerLocal string // the same key on this machine, for --identity + pub string // owner recipient + local string // darwin/linux binary for the --host commands + host string + user string + key string } func (f *fleetIT) sh(cmd string) string { @@ -173,11 +174,11 @@ func TestFleetIntegration(t *testing.T) { t.Cleanup(func() { vm.Close() }) f := &fleetIT{t: t, exec: vm, ctx: ctx, host: host, user: user, key: keyPath} - containers := []string{"teploy-it-minio", "teployit-db", "teployit-cache", "teployit2-db", "wordpress-shared-mysql", "omni_postgres", "teploy-it-hook", "teploy-it-scratch"} + containers := []string{"teploy-it-minio", "teployit-db", "teployit-cache", "teployit2-db", "teployit3-db", "wordpress-shared-mysql", "omni_postgres", "teploy-it-hook", "teploy-it-scratch"} cleanup := func() { c := context.Background() - vm.Run(c, "docker rm -fv "+strings.Join(containers, " ")+" >/dev/null 2>&1; docker ps -aq --filter name=teploy-backup-verify- | xargs -r docker rm -fv >/dev/null 2>&1; pkill -f teploy-it-hook.py; true") - vm.Run(c, "rm -rf "+itDir+" /deployments/teployit /deployments/teployit2 /deployments/_adopted /deployments/.teploy/apps/teployit /deployments/.teploy/apps/teployit2 /deployments/.teploy/apps/_adopted /deployments/.teploy/apps/_fleet /var/tmp/teploy-backup.* /var/tmp/teploy-pre-restore.* /var/tmp/teploy-verify.* /var/tmp/teploy-restore.* /var/tmp/teploy-dr /tmp/teploy-pwned; crontab -r 2>/dev/null; rm -f \"$HOME\"/.config/teploy/backup-policy.json \"$HOME\"/.config/teploy/backup-target.json \"$HOME\"/.config/teploy/backup-run.lock; true") + vm.Run(c, "docker rm -fv "+strings.Join(containers, " ")+" >/dev/null 2>&1; docker ps -aq --filter name=teploy-backup-verify- | xargs -r docker rm -fv >/dev/null 2>&1; pkill -f teploy-it-hook.py; pkill -9 -f '[s]leep 600'; true") + vm.Run(c, "rm -rf "+itDir+" /deployments/teployit /deployments/teployit2 /deployments/teployit3 /deployments/_adopted /deployments/.teploy/apps/teployit /deployments/.teploy/apps/teployit2 /deployments/.teploy/apps/teployit3 /deployments/.teploy/apps/_adopted /deployments/.teploy/apps/_fleet /var/tmp/teploy-backup.* /var/tmp/teploy-pre-restore.* /var/tmp/teploy-verify.* /var/tmp/teploy-restore.* /var/tmp/teploy-dr /tmp/teploy-pwned; crontab -r 2>/dev/null; sudo -n rm -rf /etc/teploy/backup-policy.json; sudo -n chmod 755 /etc/teploy 2>/dev/null; sudo -n userdel -rf teployit 2>/dev/null; rm -f \"$HOME\"/.config/teploy/backup-policy.json \"$HOME\"/.config/teploy/backup-target.json \"$HOME\"/.config/teploy/backup-run.lock; true") } cleanup() t.Cleanup(cleanup) @@ -223,6 +224,10 @@ func TestFleetIntegration(t *testing.T) { if err := vm.Upload(ctx, strings.NewReader(string(b)), file, "0600"); err != nil { t.Fatal(err) } + f.ownerLocal = filepath.Join(tmp, "owner.txt") + if err := os.WriteFile(f.ownerLocal, b, 0o600); err != nil { + t.Fatal(err) + } for _, line := range strings.Split(string(b), "\n") { if v, ok := strings.CutPrefix(line, "# public key: "); ok { return v @@ -622,6 +627,9 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() } f.awsIT("s3 rm s3://itfleet/" + bad + " --only-show-errors") f.awsIT("s3 rm s3://itfleet/" + partial + " --only-show-errors") + // An external kill -9 cannot reach the runs' own process groups: clear what + // those two left, so what follows measures the stale-holder path alone. + f.sh("pkill -9 -f '[s]leep 600'; true") // A holder alive and stuck past every deadline it had: the next run ends it. startKilled(itDir + "/publishing") holderPID := strings.TrimSpace(f.sh("cat " + itDir + "/pid")) @@ -633,6 +641,11 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() if alive := ssh.RunDetailed(ctx, vm, "kill -0 "+holderPID); !alive.Failed() { t.Fatalf("the stuck holder %s is still alive", holderPID) } + // Its upload ran in a process group of its own: that must not survive + // into the next run either. + if left := strings.TrimSpace(f.sh("pgrep -fa '[s]leep 600' || true")); left != "" { + t.Fatalf("the stuck holder's children survived: %s", left) + } f.sh("kill -9 -- -" + holderPID + " 2>/dev/null; rm -rf /var/tmp/teploy-backup.* " + itShims + "/aws") }) @@ -719,6 +732,107 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() if len(rest) != 4*len(want) { t.Fatalf("%d objects left for %d backups: %v", len(rest), len(want), rest) } + + // A database whose last run of each day fails: it has its own history, and + // keep_daily holds for it, not only for the primary's ids. + var primary, reports []string + seed2 := "export AWS_ACCESS_KEY_ID=" + itAK + " AWS_SECRET_ACCESS_KEY=" + itSK + " AWS_DEFAULT_REGION=us-east-1; s() { aws --endpoint-url " + itEndpoint + " s3 cp --only-show-errors " + itDir + "/blob \"s3://itfleet/seriesapp/accessories/db/$1\"; }; " + for d := 0; d < 12; d++ { + day := time.Date(2026, 10, 10+d, 0, 0, 0, 0, time.UTC) + early, late := day.Add(2*time.Hour).Format("20060102-150405"), day.Add(20*time.Hour).Format("20060102-150405") + primary = append(primary, early, late) + reports = append(reports, early) + seed2 += "s " + early + ".pgdump.age; s " + late + ".pgdump.age; s databases/reports/" + early + ".pgdump.age; " + } + f.sh(seed2) + daily := backup.RetentionPolicy{KeepDaily: 4} + if _, err := client.PruneBackups(ctx, "seriesapp", "accessories/db", f.s3, daily); err != nil { + t.Fatalf("prune: %v", err) + } + gotReports := map[string]bool{} + for _, k := range f.keys("seriesapp/accessories/db/databases/reports/") { + parts := strings.Split(k, "/") + gotReports[strings.Split(parts[len(parts)-1], ".")[0]] = true + } + wantReports := map[string]bool{} + for _, id := range resticForget(reports, daily) { + wantReports[id] = true + } + if fmt.Sprint(sortedKeysOf(gotReports)) != fmt.Sprint(sortedKeysOf(wantReports)) { + t.Fatalf("reports kept %v, want %v: a database must get its own keep_daily", sortedKeysOf(gotReports), sortedKeysOf(wantReports)) + } + _ = primary + }) + + t.Run("a leftover schedule store does not decide retention", func(t *testing.T) { + f.t = t + before := len(f.keys("teployit/accessories/db/")) + if err := backup.WritePolicy(ctx, vm, "teployit", backup.AppPolicy{Accessories: map[string]*backup.KindPolicy{"db": {KeepLast: 1}}}); err != nil { + t.Fatal(err) + } + res, code, out := f.runAll("") + f.sh("rm -f " + backup.PolicyPath("teployit")) + if code != 0 { + t.Fatalf("exit %d: %s", code, out) + } + if !strings.Contains(strings.Join(res.Warnings, "\n"), "no cron line runs it") { + t.Fatalf("no warning about the leftover: %v", res.Warnings) + } + var primaries int + for _, k := range f.keys("teployit/accessories/db/") { + if strings.Count(strings.TrimPrefix(k, "teployit/accessories/db/"), "/") == 0 { + primaries++ + } + } + if primaries < 3 { + t.Fatalf("keep_last 1 from a stale store pruned the history: %d primaries left (of %d keys before)", primaries, before) + } + }) + + t.Run("every database dropped: failure and heartbeat down, nothing pruned", func(t *testing.T) { + f.t = t + f.sh("docker run -d --name teployit3-db --memory 128m --label teploy.role=accessory --label teploy.app=teployit3 --label teploy.accessory=db -e POSTGRES_DB=three -e POSTGRES_USER=appuser -e POSTGRES_PASSWORD=" + itDBPW + " postgres:17-alpine >/dev/null") + f.waitFor("postgres", "docker exec teployit3-db pg_isready -h 127.0.0.1 -U appuser") + f.psql("teployit3-db", "three", "CREATE TABLE t(id int)") + defer f.sh("docker rm -fv teployit3-db >/dev/null 2>&1; true") + res, code, out := f.runAll("") + if code != 0 || f.target(res, "teployit3-db").Status != backup.TargetOK { + t.Fatalf("first run: %d %s", code, out) + } + before := f.keys("teployit3/accessories/db/") + f.sh("docker exec teployit3-db psql -X -q -U appuser -d postgres -c 'DROP DATABASE three'") + res, code, out = f.runAll("") + if code == 0 || f.target(res, "teployit3-db").Status != backup.TargetFailed { + t.Fatalf("a target that had backups and now holds nothing must fail: exit %d %+v\n%s", code, f.target(res, "teployit3-db"), out) + } + if got := f.keys("teployit3/accessories/db/"); strings.Join(got, ",") != strings.Join(before, ",") { + t.Fatalf("pruned on a run that stored nothing:\n%v\n%v", before, got) + } + if log := f.sh("cat " + itDir + "/hook.log"); !strings.Contains(lastLine(log), "status=down") { + t.Fatalf("heartbeat must be down: %s", lastLine(log)) + } + }) + + t.Run("a target over 256 MiB gets a checksum the store verified", func(t *testing.T) { + f.t = t + f.sh("dd if=/dev/urandom of=/deployments/teployit/volumes/data/big bs=1M count=258 2>/dev/null") + res, code, out := f.runAll("") + f.sh("rm -f /deployments/teployit/volumes/data/big") + if code != 0 { + t.Fatalf("exit %d: %s", code, out) + } + vol := f.target(res, "teployit/volumes") + if vol.SizeBytes < 256<<20 { + t.Fatalf("not over the read-back limit: %+v", vol) + } + bucket, key, _ := strings.Cut(strings.TrimPrefix(vol.Objects[0].Key, "s3://"), "/") + head := f.awsIT("s3api head-object --checksum-mode ENABLED --bucket " + bucket + " --key " + key) + if !strings.Contains(head, "sha256") { + t.Fatalf("no recorded sha256: %s", head) + } + if left := f.keys("teployit/volumes/.incoming/"); len(left) != 0 { + t.Fatalf("pending upload left: %v", left) + } }) t.Run("never prunes across servers", func(t *testing.T) { @@ -735,6 +849,17 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() if got := f.keys("gfsapp/accessories/db/"); len(got) != len(before) { t.Fatalf("pruned across servers: %v", got) } + // A rebuilt server's old marker: visible, and clearable by name. + out, err := f.cli("", "--json", "server", "backup-target", "owners") + if err != nil || !strings.Contains(out, "aaaaaaaaaaaaaaaa") { + t.Fatalf("owners: %v\n%s", err, out) + } + if out, err := f.cli("", "server", "backup-target", "owners", "--clear", "aaaaaaaaaaaaaaaa"); err != nil || !strings.Contains(out, "Removed 1 marker") { + t.Fatalf("clear: %v\n%s", err, out) + } + if _, err := client.PruneBackups(ctx, "gfsapp", "accessories/db", f.s3, backup.RetentionPolicy{KeepLast: 1}); err != nil { + t.Fatalf("prune after clearing: %v", err) + } // Per-server prefixes: a sibling prefix is invisible to this one's prune. f.sh("head -c 2000 /dev/urandom > " + itDir + "/blob") f.awsIT("s3 cp " + itDir + "/blob s3://itfleet/server-b/shared/accessories/db/20200101-000000.pgdump --only-show-errors") @@ -746,10 +871,10 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() if _, err := client.PruneBackups(ctx, "shared", "accessories/db", a, backup.RetentionPolicy{KeepLast: 1}); err != nil { t.Fatal(err) } - if got := f.keys("server-a/"); len(got) != 1 { + if got := f.keys("server-a/shared/accessories/"); len(got) != 1 { t.Fatalf("server-a kept %v", got) } - if got := f.keys("server-b/"); len(got) != 2 { + if got := f.keys("server-b/shared/accessories/"); len(got) != 2 { t.Fatalf("server-a's prune reached server-b: %v", got) } }) @@ -820,6 +945,49 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() if got := f.keys("teployit/accessories/db/"); len(got) != len(before) { t.Fatalf("pruned an unowned previous prefix: %d -> %d", len(before), len(got)) } + // The real flag: owned applies only to the prefix being left. A second run + // first, so the prefix being left has history to age out. + if _, code, runOut := f.runAll(""); code != 0 { + t.Fatalf("second run: %s", runOut) + } + srv1Before := len(f.keys("srv1/teployit/accessories/db/")) + rootBefore := len(f.keys("teployit/accessories/db/")) + out, err = f.cli(cred, "server", "backup-target", "set", "--bucket", "itfleet", "--endpoint", itEndpoint, "--prefix", "srv2", "--previous-prefix-owned") + if err != nil { + t.Fatalf("target set --previous-prefix-owned: %v\n%s", err, out) + } + moved, err := backup.ReadTarget(ctx, vm) + if err != nil { + t.Fatal(err) + } + for _, pp := range moved.PreviousPrefixes { + if want := pp.Prefix == "srv1"; pp.Owned != want { + t.Fatalf("%q owned=%v: %+v", pp.Prefix, pp.Owned, moved.PreviousPrefixes) + } + } + if _, err := client.PruneBackups(ctx, "teployit", "accessories/db", TargetS3(moved), backup.RetentionPolicy{KeepLast: 1}); err != nil { + t.Fatal(err) + } + if got := len(f.keys("teployit/accessories/db/")); got != rootBefore { + t.Fatalf("--previous-prefix-owned pruned the shared root: %d -> %d", rootBefore, got) + } + if got := len(f.keys("srv1/teployit/accessories/db/")); got >= srv1Before { + t.Fatalf("the owned prefix did not age out: %d -> %d", srv1Before, got) + } + // A marker from another server in the owned prefix overrides the word. + f.sh("printf x > " + itDir + "/o") + f.awsIT("s3 cp " + itDir + "/o s3://itfleet/srv1/teployit/.owners/bbbbbbbbbbbbbbbb --only-show-errors") + srv1Now := len(f.keys("srv1/teployit/accessories/db/")) + if _, err := client.PruneBackups(ctx, "teployit", "accessories/db", TargetS3(moved), backup.RetentionPolicy{KeepLast: 1}); err == nil || !strings.Contains(err.Error(), "shared backup namespace") { + t.Fatalf("a foreign marker in an owned prefix: %v", err) + } + if got := len(f.keys("srv1/teployit/accessories/db/")); got != srv1Now { + t.Fatalf("pruned a prefix with a foreign marker: %d -> %d", srv1Now, got) + } + f.awsIT("s3 rm s3://itfleet/srv1/teployit/.owners/bbbbbbbbbbbbbbbb --only-show-errors") + if out, err := f.cli(cred, "server", "backup-target", "set", "--bucket", "itfleet", "--endpoint", itEndpoint, "--prefix", "srv1"); err != nil { + t.Fatalf("moving back: %v\n%s", err, out) + } // Owned: it ages out under retention, and the newest of all survives. owned := cfg owned.PreviousPrefixes = []backup.PreviousPrefix{{Prefix: "", Owned: true}} @@ -894,13 +1062,58 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() if !sawUncovered { t.Fatalf("uncovered_databases: %v", st.UncoveredDatabases) } + + // Dash restores a row by id and accessory alone: per-database rows only on + // request, or restoring one would replace the primary. + listOut, err := f.cli("", "--json", "backup", "list", "--app", "teployit") + if err != nil { + t.Fatalf("list: %v\n%s", err, listOut) + } + if strings.Contains(listOut, `"database"`) { + t.Fatalf("per-database rows without --databases: %s", listOut) + } + listOut, err = f.cli("", "--json", "backup", "list", "--app", "teployit", "--databases") + if err != nil || !strings.Contains(listOut, `"database":"reports"`) { + t.Fatalf("--databases: %v\n%s", err, listOut) + } + + // A wiped namespace is not covered, whatever the last run said. + f.awsIT("s3 rm --recursive --only-show-errors s3://itfleet/srv1/_adopted/accessories/wordpress-shared-mysql/") + f.awsIT("s3 rm --recursive --only-show-errors s3://itfleet/_adopted/accessories/wordpress-shared-mysql/") + out, err = f.cli("", "--json", "backup", "status") + if err != nil { + t.Fatalf("status: %v\n%s", err, out) + } + st = backupStatusDTO{} + if err := json.Unmarshal(lastJSONLine([]byte(out)), &st); err != nil { + t.Fatal(err) + } + if e := entry("wordpress-shared-mysql"); e.Coverage != backup.CoverageStale || e.Newest != nil { + t.Fatalf("a wiped namespace: %+v", e) + } + res, code, runOut := f.runAll("") + if code != 0 { + t.Fatalf("run: %d %s", code, runOut) + } + _ = res }) t.Run("adopting by name: an underscore, no labels", func(t *testing.T) { f.t = t f.psql("omni_postgres", "postgres", "CREATE DATABASE omni") f.psql("omni_postgres", "omni", "CREATE TABLE o(id int); INSERT INTO o VALUES (1)") - out, err := f.cli("", "server", "backup-policy", "set", "--adopt", "omni_postgres:postgres:all", "--volume-max-gb", "0.000001") + // The machine path, end to end. --host must not count as a policy flag. + policyJSON, _ := json.Marshal(backup.FleetPolicy{Interval: "1h", Retention: retention, Volumes: true, Recipient: f.pub, Adopted: base.Adopted, HeartbeatURLs: base.HeartbeatURLs}) + out, err := f.cli(string(policyJSON), "--json", "server", "backup-policy", "set", "--stdin") + if err != nil || strings.Contains(out, itToken) { + t.Fatalf("policy set --stdin: %v\n%s", err, out) + } + // A machine-written policy that drops the key does not silently stop encrypting. + noKey, _ := json.Marshal(backup.FleetPolicy{Interval: "1h", Retention: retention, Volumes: true}) + if out, err := f.cli(string(noKey), "server", "backup-policy", "set", "--stdin"); err == nil || !strings.Contains(out, "--no-recipient") { + t.Fatalf("dropping the recipient on stdin: %v\n%s", err, out) + } + out, err = f.cli("", "server", "backup-policy", "set", "--adopt", "omni_postgres:postgres:all", "--volume-max-gb", "0.000001") if err != nil { t.Fatalf("policy set: %v\n%s", err, out) } @@ -983,6 +1196,8 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() f.sh("docker run -d --name teployit2-db --memory 128m --label teploy.role=accessory --label teploy.app=teployit2 --label teploy.accessory=db -e POSTGRES_DB=two -e POSTGRES_USER=appuser -e POSTGRES_PASSWORD=" + itDBPW + " postgres:17-alpine >/dev/null") f.waitFor("postgres", "docker exec teployit2-db pg_isready -h 127.0.0.1 -U appuser") f.psql("teployit2-db", "two", "CREATE TABLE z(id int)") + f.sh("docker exec teployit2-db psql -X -q -U appuser -d postgres -c 'CREATE DATABASE analytics'") + f.psql("teployit2-db", "analytics", "CREATE TABLE a(id int); INSERT INTO a VALUES (41),(42)") // Upgrade, from a folder with teploy.yml. dir := t.TempDir() yml := "app: teployit2\nserver: " + f.host + "\ndomain: teployit2.example.test\nimage: alpine:3\naccessories:\n db:\n image: postgres:17-alpine\n env:\n POSTGRES_DB: two\n POSTGRES_USER: appuser\n POSTGRES_PASSWORD: " + itDBPW + "\n" @@ -1024,8 +1239,16 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() if len(f.keys(base+"teployit2/accessories/db/")) <= keysBefore { t.Fatal("no backup object from the upgrade") } + if len(f.keys(base+"teployit2/accessories/db/databases/analytics/")) == 0 { + t.Fatalf("the upgrade's backup left out a database: %v", f.keys(base+"teployit2/accessories/db/")) + } f.waitFor("postgres", "docker exec teployit2-db pg_isready -h 127.0.0.1 -U appuser") f.psql("teployit2-db", "two", "CREATE TABLE IF NOT EXISTS z(id int)") + // The accessory has no volume in this fixture, so the data did not survive + // the recreate: seed the second database again for the purge. + f.sh("docker exec teployit2-db psql -X -q -U appuser -d postgres -c 'CREATE DATABASE analytics'") + f.psql("teployit2-db", "analytics", "CREATE TABLE a(id int); INSERT INTO a VALUES (41),(42)") + analyticsBefore := len(f.keys(base + "teployit2/accessories/db/databases/analytics/")) // Healthy: the purge backs up the accessory and the volumes, then deletes. keysBefore = len(f.keys(base + "teployit2/accessories/db/")) out, err = f.cli("", "remove", "--app", "teployit2", "--purge", "--yes") @@ -1035,6 +1258,17 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() if len(f.keys(base+"teployit2/accessories/db/")) <= keysBefore || len(f.keys(base+"teployit2/volumes/")) == 0 { t.Fatalf("no backups from the purge: %v", f.keys(base+"teployit2/")) } + if !strings.Contains(out, "Backed up every database") { + t.Fatalf("purge output: %s", out) + } + // What the purge destroyed is in the bucket: every database, not the primary alone. + if got := len(f.keys(base + "teployit2/accessories/db/databases/analytics/")); got <= analyticsBefore { + t.Fatalf("the purge's backup left out a database: %d -> %d", analyticsBefore, got) + } + // Safety copies are marked: retention leaves them for a week. + if len(f.keys(base+"teployit2/accessories/db/safety/")) == 0 { + t.Fatalf("no safety markers: %v", f.keys(base+"teployit2/accessories/db/")) + } if res := ssh.RunDetailed(ctx, vm, "docker inspect teployit2-db >/dev/null 2>&1"); !res.Failed() { t.Fatal("the purge left the accessory") } @@ -1105,6 +1339,142 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() t.Fatalf("receipt: %+v", receipt) } vm.Run(ctx, "docker ps -aq --filter name=teploy-dr- | xargs -r docker rm -fv >/dev/null 2>&1; rm -rf /var/tmp/teploy-dr/teployit") + + // Clean-host recovery through the normal commands: no server identity, no + // policy, no stored target; only the bucket credentials and the owner's key. + cleanHost := "mkdir -p " + itDir + "/hidden && mv /deployments/.age-key " + itDir + "/hidden/age-key && for f in backup-policy.json backup-target.json; do [ -e \"$HOME\"/.config/teploy/$f ] && mv \"$HOME\"/.config/teploy/$f " + itDir + "/hidden/$f; done; true" + restoreHost := "mv " + itDir + "/hidden/age-key /deployments/.age-key; for f in backup-policy.json backup-target.json; do [ -e " + itDir + "/hidden/$f ] && mv " + itDir + "/hidden/$f \"$HOME\"/.config/teploy/$f; done; true" + f.sh(cleanHost) + defer f.sh(restoreHost) + dir := t.TempDir() + yml := "app: teployit\nserver: " + f.host + "\ndomain: teployit.example.test\nimage: alpine:3\nvolumes:\n data: /app/data\naccessories:\n db:\n image: postgres:17-alpine\n env:\n POSTGRES_DB: appdb\n POSTGRES_USER: appuser\n POSTGRES_PASSWORD: " + itDBPW + "\n" + if err := os.WriteFile(filepath.Join(dir, "teploy.yml"), []byte(yml), 0o600); err != nil { + t.Fatal(err) + } + creds := []string{"TEPLOY_S3_ACCESS_KEY=" + itAK, "TEPLOY_S3_SECRET_KEY=" + itSK} + run := func(args ...string) (string, error) { + c := exec.CommandContext(ctx, f.local, append([]string{"--host", f.host, "--user", f.user, "--key", f.key}, args...)...) + c.Dir = dir + c.Env = append(os.Environ(), creds...) + o, e := c.CombinedOutput() + return string(o), e + } + s3Flags := []string{"--bucket", "itfleet", "--endpoint", itEndpoint, "--prefix", "srv1"} + // Without the key: refused, naming the flag. + out, err := run(append([]string{"dr", "restore", m.ID}, s3Flags...)...) + if err == nil || !strings.Contains(out, "--identity") { + t.Fatalf("dr restore with no key on a clean host: %v\n%s", err, out) + } + f.sh("rm -rf /var/tmp/teploy-dr/teployit") + out, err = run(append(append([]string{"dr", "restore", m.ID}, s3Flags...), "--identity", f.ownerLocal)...) + if err != nil { + t.Fatalf("dr restore --identity --bucket --prefix on a clean host: %v\n%s", err, out) + } + vm.Run(ctx, "docker ps -aq --filter name=teploy-dr- | xargs -r docker rm -fv >/dev/null 2>&1; rm -rf /var/tmp/teploy-dr/teployit") + if left := strings.TrimSpace(f.sh("ls -d /var/tmp/teploy-dr-identity.* 2>/dev/null; true")); left != "" { + t.Fatalf("the owner key was left on the server: %s", left) + } + // A database and one schema, encrypted, restored with only the owner's key. + rows := f.keys("srv1/teployit/accessories/db/") + id := "" + for _, k := range rows { + if rel := strings.TrimPrefix(k, "srv1/teployit/accessories/db/"); !strings.Contains(rel, "/") && strings.HasSuffix(rel, ".pgdump.age") { + id = strings.Split(rel, ".")[0] + } + } + if id == "" { + t.Fatalf("no encrypted primary: %v", rows) + } + f.psql("teployit-db", "appdb", "INSERT INTO t VALUES (777)") + // The server identity alone is absent: without --identity it says what to do. + out, err = run(append([]string{"backup", "restore", id, "--app", "teployit", "--accessory", "db", "--yes", "--no-backup"}, s3Flags...)...) + if err == nil || !strings.Contains(out, "--identity") { + t.Fatalf("restore with no key on a clean host: %v\n%s", err, out) + } + out, err = run(append(append([]string{"backup", "restore", id, "--app", "teployit", "--accessory", "db", "--yes", "--no-backup"}, s3Flags...), "--identity", f.ownerLocal)...) + if err != nil { + t.Fatalf("backup restore --identity on a clean host: %v\n%s", err, out) + } + if got := f.psql("teployit-db", "appdb", "SELECT COUNT(*) FROM t WHERE id = 777"); got != "0" { + t.Fatalf("the restore did not replace the data: %s", got) + } + if left := strings.TrimSpace(f.sh("ls -d /var/tmp/teploy-dr-identity.* 2>/dev/null; true")); left != "" { + t.Fatalf("the owner key was left on the server: %s", left) + } + }) + + t.Run("a real cron firing, with cron's own PATH", func(t *testing.T) { + f.t = t + seq := f.sh("wc -l < /deployments/.teploy/apps/_fleet/events.jsonl") + f.sh(`crontab -l | sed -E 's|^([^ ]+ +){5}|* * * * * |' | crontab -`) + line := f.sh("crontab -l") + if !strings.HasPrefix(line, "* * * * * ") { + t.Fatalf("crontab: %s", line) + } + fired := false + for i := 0; i < 240 && !fired; i++ { + if strings.TrimSpace(f.sh("wc -l < /deployments/.teploy/apps/_fleet/events.jsonl")) != strings.TrimSpace(seq) { + fired = true + break + } + time.Sleep(time.Second) + } + f.sh(`crontab -l | sed -E 's|^([^ ]+ +){5}|7 3 * * * |' | crontab -`) + f.waitFor("the cron run to finish", "! pgrep -f '[b]ackup run --all' >/dev/null") + if !fired { + t.Fatal("cron never ran it") + } + last := f.sh("tail -n 1 /deployments/.teploy/apps/_fleet/events.jsonl") + if !strings.Contains(last, `"kind":"backup.all"`) || strings.Contains(last, "aws CLI not found") { + t.Fatalf("cron's run: %s", last) + } + }) + + t.Run("a non-root user who cannot read /etc/teploy never uploads plaintext", func(t *testing.T) { + f.t = t + policy := `{"v":"1","interval":"1h","recipient":"` + f.pub + `"}` + f.sh("sudo -n mkdir -p /etc/teploy && printf '%s' " + ssh.ShellQuote(policy) + " | sudo -n tee /etc/teploy/backup-policy.json >/dev/null && sudo -n chmod 600 /etc/teploy/backup-policy.json && sudo -n chmod 700 /etc/teploy") + defer f.sh("sudo -n rm -f /etc/teploy/backup-policy.json; sudo -n chmod 755 /etc/teploy; sudo -n userdel -rf teployit 2>/dev/null; true") + // This user has passwordless sudo, as deploy users do: the policy is read. + got, err := backup.ReadFleetPolicy(ctx, vm) + if err != nil || got.Recipient != f.pub { + t.Fatalf("through sudo: %+v %v", got, err) + } + // One without sudo: refused, not "no policy". A target of its own, so the + // policy read is what stands between it and a plaintext upload. + f.sh("sudo -n useradd -m teployit 2>/dev/null; printf '%s' " + ssh.ShellQuote(`{"v":"1","bucket":"itfleet","region":"us-east-1","endpoint":"`+itEndpoint+`"}`) + " | sudo -n -u teployit -H sh -c 'mkdir -p ~/.config/teploy && cat > ~/.config/teploy/backup-target.json'; sudo -n chmod 755 " + itDir + " " + itBin) + before := strings.Join(f.keys("srv1/teployit/volumes/"), ",") + res := ssh.RunDetailed(ctx, vm, "sudo -n -u teployit -H env AWS_ACCESS_KEY_ID="+itAK+" AWS_SECRET_ACCESS_KEY="+itSK+" "+itBin+" backup run --app teployit") + if !res.Failed() || !strings.Contains(string(res.Stdout)+string(res.Stderr), "cannot tell whether") { + t.Fatalf("a hidden policy must refuse: exit %d\n%s%s", res.ExitCode, res.Stdout, res.Stderr) + } + if strings.Join(f.keys("srv1/teployit/volumes/"), ",") != before { + t.Fatal("something was uploaded") + } + res = ssh.RunDetailed(ctx, vm, "sudo -n -u teployit -H "+itBin+" backup run --all") + if !res.Failed() || !strings.Contains(string(res.Stdout)+string(res.Stderr), "cannot tell whether") { + t.Fatalf("--all: exit %d\n%s%s", res.ExitCode, res.Stdout, res.Stderr) + } + }) + + t.Run("zero targets is a failure unless the policy allows it", func(t *testing.T) { + f.t = t + f.sh("docker rm -fv teployit-db teployit-cache wordpress-shared-mysql omni_postgres >/dev/null 2>&1; true") + p := backup.FleetPolicy{V: "1", Interval: "1h", Retention: retention, Recipient: f.pub, HeartbeatURLs: base.HeartbeatURLs} + f.setPolicy(p) + res, code, out := f.runAll("") + if code == 0 || res.Status != backup.RunFailed || !strings.Contains(res.Error, "allow_empty") { + t.Fatalf("exit %d %s %s\n%s", code, res.Status, res.Error, out) + } + if log := f.sh("cat " + itDir + "/hook.log"); !strings.Contains(lastLine(log), "status=down") { + t.Fatalf("heartbeat: %s", lastLine(log)) + } + p.AllowEmpty = true + f.setPolicy(p) + res, code, out = f.runAll("") + if code != 0 || res.Status != backup.RunOK { + t.Fatalf("allowed: exit %d %s\n%s", code, res.Status, out) + } }) t.Run("doctor flags a policy with no target and no recipient", func(t *testing.T) { From 31348879e003ac80563cd3e972bade2f6135d56e Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:40:30 -0700 Subject: [PATCH 10/17] docs: fleet engine review fixes in the README, CHANGELOG and AUDIT_OPEN Co-Authored-By: Claude Opus 5.5 --- AUDIT_OPEN.md | 44 +++++++++++++++++++++++++++++--- CHANGELOG.md | 10 +++++++- README.md | 70 +++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 100 insertions(+), 24 deletions(-) diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index f737fb9..64c3d98 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -21,14 +21,50 @@ Decisions worth reviewing: - `.owners/` markers refuse a prune in a namespace another server wrote to, because a reused prefix or a same-named app is the failure a prefix alone cannot catch. +Independent review (2026-11-05, same day) reproduced real holes and all are fixed on the +branch: purge and upgrade backed up only the primary (now every database, refusing what +it cannot cover); a leftover backup-schedule.json decided a policy run's retention; a run +that stored nothing still pruned and reported ok; a database's objects followed the +primary's ids (each series is selected on its own now); the .owners guard failed open on +a listing error, an empty machine-id and a late marker; status never looked at the +bucket; owner-key recovery could not work through normal commands; a non-root user reading +a 0700 /etc/teploy saw "no policy" and uploaded plaintext; per-database rows reached Dash, +whose restore by id would have replaced the primary. Medium/low: refused prunes invisible, +pre-restore not forwarding --ignore-space-check, zero targets reading as ok, .incoming only +at one depth, a docker blip reading as no container, a wrong clock deleting real backups, +--previous-prefix-owned marking every earlier prefix, AWS's bundle unverified, a 6h deadline +with no override, legacy lines reported encrypted, ClickHouse and friends in no report, a +--no-credentials target never working, --stdin dropping the recipient silently, bucket-wide +multipart abort, TEPLOY_BACKUP_TMPDIR workspaces invisible, postgresql:// names, a stale +holder's children, self-referential checksums above 256 MiB, safety copies prunable. + Verified live (colima, MinIO via the real aws CLI, real age, the real linux binary): -`go test -tags integration -run TestFleetIntegration ./internal/cli`, 17 scenarios. Only -unit-tested: the deploy hook, healthchecks.io's /fail form, the janitors' age thresholds and -multipart abort, setup's aws install, the space preflights' refusal, a bucket refusing deletes. +`go test -tags integration -run TestFleetIntegration ./internal/cli`, 22 scenarios, on real +multi-database accessories; `TEPLOY_IT_INSTALL_AWS=1 go test -tags integration -run +TestAWSCLIInstallScriptLive ./internal/backup` installs AWS's bundle for real and refuses a wrong +pin. Only unit-tested: the deploy hook, healthchecks.io's /fail form, the janitors' age thresholds +and multipart abort, the space preflights' refusal, a bucket refusing deletes, a store without +upload checksums, a musl host, the clock checks and an empty machine-id, setup's stage itself +(the script it runs is the live one), the /etc/teploy root path (the fixture is non-root; the +non-root paths, both with sudo and without, ran live). Regressions: `TEPLOY_IT_ENGINES=postgres,mysql,redis go test -tags integration ./internal/backup`. -The other Slice 2 engines were not re-run. The DR fixture still needs a 022 umask (open below). +The Engine matrix's other Slice 2 engines were not re-run. The DR fixture still needs a 022 umask. Open / deferred: +- Legacy objects from a server that has not upgraded yet carry no marker. A first prune in + a shared root by an upgraded server cannot tell them from its own: only markers from the + moment each server upgrades protect it. Per-server prefixes close this; the set command + warns when there is none. Attributing unmarked objects would stop a single server's own + legacy history ever ageing out. +- A dropped database in an accessory whose other databases are fine keeps its series (its + newest good object forever) and raises nothing. It reads as history, not as a problem. +- An external kill -9 of a run leaves its children (own process groups) running until they + finish; they cannot block the next run, and a stale holder's are killed. A Pdeathsig + would reach only the shell. +- AWS's signing key is pinned (expires 2027-07-01; old signatures stay valid). A rotated + key makes setup refuse the bundle until the pin is updated: the safe way to fail. +- Unsupported databases are recognised by image name only; a renamed image reads as generic + (and a labelled accessory still hits the live-data guard). - A schema name holding `'` or `\`: `mysqldump` mangles it in its own `USE` (seen live on mysql:8), so it is reported as not backed up in omissions and status, never silently. A Postgres or MySQL name with `=`, a leading `-`, a control character or over 64 bytes diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f3f6b5..35d3820 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,7 +38,13 @@ All notable changes to teploy are documented here. Format follows [Keep a Change before a deploy with a `pre_deploy` hook (`backup_before_hooks: true`). - Volume size guard: `volume_max_gb` warns and excludes, visible in status and events. - `teploy doctor` check `backup-policy`; `teploy setup` installs the aws CLI (AWS's - bundle where the distribution has none, as on Ubuntu 24.04) and age. + bundle with its PGP signature checked, where the distribution has none, as on Ubuntu + 24.04) and age. +- `backup restore`, `accessory restore` and `accessory verify-backup` take `--identity`, + `dr` commands take `--prefix`: a rebuilt host recovers with only the owner key and the + bucket credentials, through the normal commands. `--deadline` on manual backups, + restores and verifies; `--tmp-dir`, `--target-deadline`, `--allow-empty` on the policy. +- `teploy server backup-target owners [--clear ]`. ### Changed @@ -54,6 +60,8 @@ All notable changes to teploy are documented here. Format follows [Keep a Change and a reserve, measured against what the backup will occupy. - `backup list` and status no longer treat below-floor objects as restorable, and a newest good backup is never pruned even with only `--max-age-days` set. +- `backup list` shows per-database rows only with `--databases`: a caller restoring a + row by id and accessory alone would replace the primary with another database's dump. ### Fixed diff --git a/README.md b/README.md index 33094b3..6444f6c 100644 --- a/README.md +++ b/README.md @@ -696,15 +696,20 @@ Targets run one at a time. One failing does not stop the next, and nothing of a failed target is pruned. The exit status is nonzero unless every target succeeded, and `backup.all` (under the pseudo-app `_fleet`) and `--json` both report `ok`, `partial` or `failed` plus `omissions`: everything left out (exclusions, size-guard -hits, a DB-like container nothing adopts, a schema the tools cannot address). A -clean status with omissions is not a complete backup. A run that finds another +hits, a DB-like container nothing adopts, including engines teploy has no method for +such as ClickHouse or Qdrant, a schema the tools cannot address). A clean status with +omissions is not a complete backup. Zero targets is a failure unless the policy says +`--allow-empty`: a lost accessory and an empty docker daemon look the same. A target +that has earlier backups and now holds nothing to back up (its databases dropped) is +a failure too. `--tmp-dir` builds on another disk and `--target-deadline` replaces the +6h per-target deadline; manual commands take `--deadline`. A run that finds another running stands down; the lock is an `flock`, so a killed run cannot block the next, and a holder alive past the 23h run deadline is killed. Each target has a 6h deadline. A killed run's leftovers are invisible to later runs and visible to doctor: a final key only ever holds a verified object (the upload goes to a `.incoming/` key, its -size and SHA-256 are checked, up to 256 MiB its bytes are read back, and only then -is it copied into place), sidecars are stored before the primary object that commits +size and SHA-256 are checked, up to 256 MiB its bytes are read back, above that the +store's own `--checksum-algorithm SHA256` is used, and only then is it copied into place), sidecars are stored before the primary object that commits a backup, and objects below their engine's plausibility floor are never a good copy. **Retention** is a union of rules over the good backups, restic `forget` style: @@ -712,8 +717,14 @@ a backup, and objects below their engine's plausibility floor are never a good c N UTC days, ISO weeks and months that have one (an empty period does not use one up); `--max-age-days D` keeps everything younger than D days (restic's `keep-within`), and anything older goes unless another rule keeps it. The newest good backup is never -deleted. A pruned backup takes its globals, meta and per-database objects with it. -The same flags work on `teploy backup prune`. +deleted, and a safety copy a destructive operation took outlasts every rule for 7 days. +The primary and each database's series are selected on their own, so a database that +fails in the last run of each day still gets its `keep_daily`; ids the primary keeps +are also kept for the databases. A pruned backup takes its sidecars with it. The same +flags work on `teploy backup prune`. A run that stored nothing prunes nothing. A prune +refuses (a warning in the run's output, event and doctor: the backup is stored, retention +has stopped) when this machine's clock is over an hour from the store's or a backup is +dated in the future (`backup prune --ignore-clock`). **Heartbeats**: after every run the server GETs each URL, adding `?status=down` on anything but `ok` (`/fail` for healthchecks.io). A failing receiver is reported and @@ -725,7 +736,8 @@ still does. under the `_adopted` namespace: `teploy backup restore --adopted --accessory wordpress-shared-mysql --database wp_blog --host prod`, `teploy accessory verify-backup wordpress-shared-mysql --adopted --database wp_blog --host prod`, -`teploy backup list --adopted --host prod`. Nothing that uses an adopted container +`teploy backup list --adopted --host prod`. `backup list` shows per-database rows only +with `--databases`. Nothing that uses an adopted container is stopped for you, and the prompt says so. MySQL and MariaDB get one object per schema, so one WordPress site restores alone. Postgres clusters get one per database plus the roles (`pg_dumpall --globals-only`). @@ -751,33 +763,53 @@ objects still need the old key. Without a recipient backups keep working unencry (`encrypted: false` everywhere, so Dash can warn), and `.env`, which holds resolved secrets, is left out of volume archives with a warning (restoring such a backup keeps the server's current `.env`). `dr create --include-age-key` needs a recipient and -wraps the server's private key to it alone; `dr restore --identity ` unwraps it, -placing the recovery key on the target privately for that restore. +wraps the server's private key to it alone. + +On a host with no server identity (a rebuilt server), the normal commands recover with +the owner key alone, placing it on the host privately for that one command: +`teploy backup restore --app A --accessory db --identity recovery.txt --bucket B +--endpoint E --prefix P --host H`, likewise `accessory restore` and `accessory +verify-backup`, and `teploy dr restore --identity recovery.txt --bucket B --endpoint +E --prefix P`. + +A non-root SSH user usually cannot see into `/etc/teploy` (0700 once root has used it), +where a policy would be invisible rather than absent. Teploy asks through passwordless +sudo when it can; a user with its own `~/.config/teploy/backup-policy.json` follows that; +anyone else is refused rather than risk uploading plaintext on a server configured to +encrypt (`chmod 711 /etc/teploy` lets the refusal go away, and leaves the files 0600). #### Shared buckets Give each server its own `--prefix` on `server backup-target set`. Changing the prefix keeps the old one on record: list, verify and restore still find objects under it (`previous_prefix: true`). Its objects are never pruned, since they may be another -server's, unless `--previous-prefix-owned` says only this server wrote there; then -they age out under the policy's retention. As a second guard every server leaves a -marker at `//.owners/`, and a namespace holding another -server's marker is never pruned. +server's, unless `--previous-prefix-owned` says only this server wrote under the prefix +being left (`--claim-previous-prefix` for an earlier one); then they age out under the +policy's retention. As a second guard every server leaves a marker at +`//.owners/` before its first prune, and a namespace holding +another server's marker, or one whose owners cannot be listed, is never pruned. `teploy +server backup-target owners` lists the markers; `--clear ` removes those of +a server that is gone, or this one before a rebuild. #### Before anything destructive `backup restore`, `accessory restore`, `accessory upgrade` and `remove --purge` first -store a fresh backup of what they are about to replace, through the same path. If -that fails, or a purge or upgrade has no target to save to, the operation is refused -with what is at stake named; `--no-backup` goes ahead without one. A database or -container that no longer exists has nothing to protect and is not a refusal. With +store a fresh backup of what they are about to replace, through the same path: the one +database a restore replaces, and every database of the accessory for a purge or upgrade, +refusing if any cannot be stored. If that fails, or a purge or upgrade has no target to +save to, the operation is refused with what is at stake named; `--no-backup` goes ahead +without one (a restore over corrupt data, where the dump cannot succeed, is what it is +for). A database or container that no longer exists has nothing to protect and is not a +refusal; a docker that does not answer is. With `backup_before_hooks: true` in `teploy.yml`, a deploy whose `hooks.pre_deploy` is set does the same for each running database accessory (default off; `deploy --no-backup`). `teploy doctor` flags a policy with no target, no running cron line, no recipient or stale runs, no heartbeat, a missing `aws` or `age`, a target that does not answer, -leftover workspaces and uploads, and databases nothing backs up. `teploy setup` -installs `aws` (AWS's bundle where the distribution has no package) and `age`. +leftover workspaces and uploads, a refused prune, a legacy shell cron line (which +uploads plaintext whatever the policy says), and databases nothing backs up. `teploy +setup` installs `aws` (AWS's bundle, signature checked, where the distribution has no +package) and `age`. Accessory backups are engine-aware. The engine is detected from the container — the accessory's `engine:` in `teploy.yml` (set on the container From a81afe0986f3fc358ceb101a1b7dffe6d60f8ea8 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 00:45:39 -0700 Subject: [PATCH 11/17] feat(doctor): flag a policy with no prefix in a bucket other servers write to Per-server prefixes are the answer to objects no marker can attribute, and Dash always sets one. backup status reports other_servers when there is no prefix, from the markers at the bucket root; doctor warns with the exact remedy. set keeps its warning. Servers that have not upgraded leave no marker, so they stay invisible here. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 ++- internal/backup/status.go | 15 +++++++++++++++ internal/cli/doctor.go | 4 ++++ internal/cli/doctor_backup_test.go | 27 +++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 72bb741..95cab31 100644 --- a/README.md +++ b/README.md @@ -815,7 +815,8 @@ policy's retention. As a second guard every server leaves a marker at `//.owners/` before its first prune, and a namespace holding another server's marker, or one whose owners cannot be listed, is never pruned. `teploy server backup-target owners` lists the markers; `--clear ` removes those of -a server that is gone, or this one before a rebuild. +a server that is gone, or this one before a rebuild. A policy with no prefix in a bucket another server +writes to is flagged by `teploy doctor`. #### Before anything destructive diff --git a/internal/backup/status.go b/internal/backup/status.go index 38ece68..4e3dbd1 100644 --- a/internal/backup/status.go +++ b/internal/backup/status.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "slices" "sort" "strings" "time" @@ -99,6 +100,9 @@ type StatusTarget struct { // Reachable says the listing this report needed worked. Reachable bool `json:"reachable"` Error string `json:"error,omitempty"` + // OtherServers are fingerprints of servers that also write at this target's + // base. Only looked for with no prefix, where same-named apps would collide. + OtherServers []string `json:"other_servers,omitempty"` } // StatusPolicy summarizes the policy (no heartbeat tokens). @@ -231,6 +235,17 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle st.Target.Reachable = true } } + if st.Target.Reachable && target.Prefix == "" && policy.Configured() && c.fingerprint(ctx) != "" { + mctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + if markers, err := c.ListNamespaceMarkers(mctx, cfg); err == nil { + for _, m := range markers { + if !m.Self && !slices.Contains(st.Target.OtherServers, m.Fingerprint) { + st.Target.OtherServers = append(st.Target.OtherServers, m.Fingerprint) + } + } + } + cancel() + } listed := map[string][]ListedBackup{} listNS := func(app string) []ListedBackup { if rows, ok := listed[app]; ok { diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index 40bed98..3e2b800 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -1035,6 +1035,10 @@ func doctorBackupPolicyCheck(ctx context.Context, exec ssh.Executor, now time.Ti remedies = append(remedies, "printf '\\n' | teploy server backup-policy set --host --heartbeat-stdin") } } + if len(st.Target.OtherServers) > 0 { + warns = append(warns, fmt.Sprintf("this policy runs with no prefix in a bucket other servers also write to (%s): same-named apps share a namespace, and that server's backups cannot be told from this one's until it has upgraded", strings.Join(st.Target.OtherServers, ", "))) + remedies = append(remedies, "give each server its own prefix: teploy server backup-target set --host --bucket --prefix ") + } if st.LastRun != nil && len(st.LastRun.Warnings) > 0 { warns = append(warns, "the last backup run warned: "+strings.Join(st.LastRun.Warnings, "; ")) remedies = append(remedies, "a refused prune means retention has stopped and storage keeps growing: `teploy server backup-target owners` for a stale marker") diff --git a/internal/cli/doctor_backup_test.go b/internal/cli/doctor_backup_test.go index b4fe2b2..c6ef36e 100644 --- a/internal/cli/doctor_backup_test.go +++ b/internal/cli/doctor_backup_test.go @@ -67,3 +67,30 @@ func TestDoctorBackupPolicyCheck(t *testing.T) { t.Fatalf("%s", c.Remediation) } } + +// No prefix in a bucket another server writes to: same-named apps would collide, +// and a server that has not upgraded carries no marker to refuse on. +func TestDoctorFlagsNoPrefixInASharedBucket(t *testing.T) { + now := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) + mk := func(prefix string) *ssh.MockExecutor { + return ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "path /etc/teploy/backup-policy.json\n" + `{"v":"1","interval":"1h","recipient":"` + testRecipient + `","heartbeat_urls":["https://d/x"]}`}, + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"b","region":"r","prefix":"` + prefix + `"}`}, + ssh.MockCommand{Match: "command -v aws >/dev/null", Output: "aws\nage\n"}, + ssh.MockCommand{Match: "crontab -l", Output: "5 * * * * /usr/local/bin/teploy backup run --all # [teploy-backup-all]\n"}, + ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: ""}, + ssh.MockCommand{Match: "{ cat /etc/machine-id", Output: "0123456789abcdef0123456789abcdef -\n"}, + ssh.MockCommand{Match: "aws s3 ls", Output: "d t 20 shop/.owners/aaaaaaaaaaaaaaaa\nd t 20 shop/.owners/0123456789abcdef\n"}, + ssh.MockCommand{Match: "python3 -c", Output: `{"v":1,"id":"1","seq":1,"ts":"2026-11-04T11:30:00Z","app":"_fleet","kind":"backup.all","outcome":"succeeded","detail":{"status":"ok"}}` + "\n"}, + ssh.MockCommand{Match: "for d in /deployments/*/volumes", Output: ""}, + ) + } + c := doctorBackupPolicyCheck(context.Background(), mk(""), now) + if c.Result != doctorWarn || !strings.Contains(c.Detail, "aaaaaaaaaaaaaaaa") || !strings.Contains(c.Remediation, "--prefix") || strings.Contains(c.Detail, "0123456789abcdef") { + t.Fatalf("%+v", c) + } + // A prefix is the answer: not even looked for. + if c := doctorBackupPolicyCheck(context.Background(), mk("srv1"), now); strings.Contains(c.Detail, "no prefix") { + t.Fatalf("%+v", c) + } +} From cefca5148c6f6db814c14c03a68966719bc26d8d Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 01:17:12 -0700 Subject: [PATCH 12/17] docs(audit): record the post-merge integration results Co-Authored-By: Claude Opus 5.5 --- AUDIT_OPEN.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 0589308..9959c27 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -39,7 +39,7 @@ multipart abort, TEPLOY_BACKUP_TMPDIR workspaces invisible, postgresql:// names, holder's children, self-referential checksums above 256 MiB, safety copies prunable. Verified live (colima, MinIO via the real aws CLI, real age, the real linux binary): -`go test -tags integration -run TestFleetIntegration ./internal/cli`, 22 scenarios, on real +`go test -tags integration -run TestFleetIntegration ./internal/cli`, 23 scenarios, on real multi-database accessories; `TEPLOY_IT_INSTALL_AWS=1 go test -tags integration -run TestAWSCLIInstallScriptLive ./internal/backup` installs AWS's bundle for real and refuses a wrong pin. Only unit-tested: the deploy hook, healthchecks.io's /fail form, the janitors' age thresholds @@ -47,8 +47,9 @@ and multipart abort, the space preflights' refusal, a bucket refusing deletes, a upload checksums, a musl host, the clock checks and an empty machine-id, setup's stage itself (the script it runs is the live one), the /etc/teploy root path (the fixture is non-root; the non-root paths, both with sudo and without, ran live). -Regressions: `TEPLOY_IT_ENGINES=postgres,mysql,redis go test -tags integration ./internal/backup`. -The Engine matrix's other Slice 2 engines were not re-run. The DR fixture still needs a 022 umask. +Regressions: `go test -tags integration ./internal/backup`, the full engine matrix after merging +v0.1.51 (postgres, pgvector, timescale, mysql, mariadb, mongo, redis, valkey, dragonfly). The DR +fixture still needs a 022 umask. Open / deferred: - Legacy objects from a server that has not upgraded yet carry no marker. A first prune in From 38bb01820e439781407e463ec85cc8da6c552f68 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:03:14 -0700 Subject: [PATCH 13/17] fix(backup): act on the Slice 4 re-review Publish: the copy names --copy-props metadata-directive (R2 has no GetObjectTagging), and a store that refuses it gets the verified local file uploaded straight to its final name and verified there, since B2 documents that a copy cannot change a checksum algorithm. A final object that lost the recorded checksum (aws CLI v1's multipart copy) is replaced the same way. Pending uploads move to one /.incoming/ tree so a 1-day lifecycle rule can expire them on a store that keeps versions; doctor and status read the rule. Markers are written once; the store's clock comes from a probe under .incoming/. The policy is written and unset through the same sudo path reads use, a policy this user cannot replace is a refusal, WriteFleetPolicy reads itself back, and the run lock falls back to ~/.config when the policy's directory is not writable. Prune refuses when it would delete most of a series (--allow-mass-delete) and when the store's clock cannot be read. Future-dated objects leave selection instead of stopping it. Safety copies are capped at 5 per target. owners --clear refuses a namespace written to in the last week without --force, and is recorded. target_deadline and --deadline over 23h are refused. A workspace carries its own expiry for the janitor; a pending upload is a leftover after 24h. An unreachable previous prefix only warns. aws CLI is held to 2.23.0/1.37.0, the head-object checksum read falls back through checksumUnsupported, the stuck holder dies before its children, and the chmod 711 remedy is gone. Co-Authored-By: Claude Opus 5.5 --- contracts/MANIFEST.md | 2 +- .../backup-status-envelope/valid/fleet.json | 3 +- .../schema/backup-status-envelope.schema.json | 7 + internal/backup/accessory.go | 3 + internal/backup/backup.go | 54 +- internal/backup/engines_integration_test.go | 4 +- internal/backup/fleet.go | 31 +- internal/backup/fleet_test.go | 4 +- internal/backup/list.go | 29 +- internal/backup/policy.go | 93 +- internal/backup/retention.go | 181 ++-- internal/backup/retention_test.go | 4 +- internal/backup/review_regression_test.go | 814 ++++++++++++++++++ internal/backup/runlock_unix.go | 24 +- internal/backup/status.go | 104 ++- internal/backup/store.go | 414 ++++++--- internal/backup/store_test.go | 62 +- internal/backup/store_test_helpers_test.go | 8 +- internal/cli/accessory.go | 3 + internal/cli/backup.go | 12 +- internal/cli/backup_cron_test.go | 2 +- internal/cli/backup_destructive_test.go | 2 +- internal/cli/backup_fleet.go | 18 +- internal/cli/backup_fleet_test.go | 38 +- internal/cli/backup_target.go | 54 +- internal/cli/backup_target_test.go | 50 ++ internal/cli/contracts_golden_test.go | 2 +- internal/cli/doctor.go | 18 +- internal/cli/doctor_backup_test.go | 72 +- internal/cli/doctor_test.go | 8 +- internal/cli/fleet_integration_test.go | 189 +++- internal/cli/setup.go | 3 +- internal/events/events.go | 2 + 33 files changed, 1972 insertions(+), 342 deletions(-) create mode 100644 internal/backup/review_regression_test.go diff --git a/contracts/MANIFEST.md b/contracts/MANIFEST.md index b91de19..5f6720b 100644 --- a/contracts/MANIFEST.md +++ b/contracts/MANIFEST.md @@ -10,7 +10,7 @@ Neutron/Nucleus dependency and a public mirror. | Corpus rev | Emitting CLI | Machine Interface | Notes | |---|---|---|---| -| 18 | main (Dash auto-config Slice 4: fleet backup engine) | 2 | Additive. New artifacts backup-status-envelope (`backup status --json`: every DB-like container and app volume with coverage state, last success/failure, newest object, last verify, plus target and policy summaries; no secrets), backup-run-envelope (`backup run --all --json`) and backup-policy-envelope (`server backup-policy show/set --json`), fixtures GENERATED from the real DTOs. backup-list rows gain optional `encrypted`, `previous_prefix`, and `database` only with `--databases` or `--adopted` (a caller restoring by id alone would replace the primary); backup-create gains `encrypted` and `objects`; backup-restore gains `pre_backup_id`; backup-target gains `previous_prefixes`. Event kinds `backup.all` and `backup_policy.set` under the pseudo-app `_fleet`; adopted containers log under `_adopted`. version-handshake advertises `backup-fleet-v1` (regenerated mi2.json). No MI bump. | +| 18 | main (Dash auto-config Slice 4: fleet backup engine) | 2 | Additive. New artifacts backup-status-envelope (`backup status --json`: every DB-like container and app volume with coverage state, last success/failure, newest object, last verify, plus target and policy summaries, the aws CLI's `aws_cli_problem` and the target's `incoming_lifecycle`; no secrets), backup-run-envelope (`backup run --all --json`) and backup-policy-envelope (`server backup-policy show/set --json`), fixtures GENERATED from the real DTOs. backup-list rows gain optional `encrypted`, `previous_prefix`, and `database` only with `--databases` or `--adopted` (a caller restoring by id alone would replace the primary); backup-create gains `encrypted` and `objects`; backup-restore gains `pre_backup_id`; backup-target gains `previous_prefixes`. Event kinds `backup.all`, `backup_policy.set` and `backup_target.owners_clear` (a marker removal or refusal) under the pseudo-app `_fleet`; adopted containers log under `_adopted`. version-handshake advertises `backup-fleet-v1` (regenerated mi2.json). No MI bump. | | 17 | main (dash auto-config slice 5: domain health) | 2 | Additive. New artifact domains-check-envelope (`teploy domains check [--server] [--app] [--log-window] --json`: per (server, app, host) DNS via the fixed public resolvers, Caddy certificate failures from its logs — `failures` is null, never 0, on short or unavailable logs — and the route with upstream resolution; per-server state so an unreachable server is explicit), fixture GENERATED from the real `sweepDomains` over a mock executor. version-handshake advertises `domains-check-v1` (regenerated mi2.json). release-record's `caddy` gains optional `tls_http_only` (absent on older records; every caddy release since records true or false). No MI bump. | | 16 | main (server-authority spec P3 / plan D4, git sources) | 2 | Additive. New artifacts source-show-envelope (`config source show --json`: repository/branch/poll, public credential facts — type, deploy-key public half + fingerprint, token username, pinned host-key fingerprints — and the per-ref admission state), source-key-envelope (`config source key --json` for ssh/token and `host-key --json`) and trigger-poll-envelope (`trigger poll --json`: one result object per app — ref, head, admitted, reason vocabulary incl. force-push/retry-limit/not-due, deployed_release), all fixtures GENERATED from the real DTOs. version-handshake advertises `git-source-v1` (git: code source, per-app credentials outside /deployments, build_secrets) and `trigger-poll-v1` (the one admitter + reset/retry/install-timer) (regenerated mi2.json). No MI bump. | | 15 | main (server-authority spec D3: backup target) | 2 | Additive. One backup target per server, stored outside /deployments with age-encrypted credentials and read at run time (`server backup-target set/show/unset`; `backup-target-v1`). New artifacts backup-target-envelope, backup-list-envelope, backup-create-envelope, backup-restore-envelope — fixtures GENERATED from the real DTO encoders (`backup list --json` now reports parsed rows {id, kind, accessory, size_bytes, created_at, key} plus schedules discovered from teploy cron markers; the old bare-array-of-raw-`aws s3 ls`-lines `backup list --json` shape is gone — it was never a corpus artifact and pinned no consumer). `backup create/restore --json` envelopes; restore quiesces the app under its fenced lock (backup-json-v1). version-handshake advertises `backup-target-v1` and `backup-json-v1` (regenerated mi2.json). No MI bump. | diff --git a/contracts/fixtures/backup-status-envelope/valid/fleet.json b/contracts/fixtures/backup-status-envelope/valid/fleet.json index 4940c7b..ce7a44c 100644 --- a/contracts/fixtures/backup-status-envelope/valid/fleet.json +++ b/contracts/fixtures/backup-status-envelope/valid/fleet.json @@ -13,7 +13,8 @@ "prefix": "" } ], - "reachable": true + "reachable": true, + "incoming_lifecycle": "covered" }, "policy": { "configured": true, diff --git a/contracts/schema/backup-status-envelope.schema.json b/contracts/schema/backup-status-envelope.schema.json index 95e0038..59f94e3 100644 --- a/contracts/schema/backup-status-envelope.schema.json +++ b/contracts/schema/backup-status-envelope.schema.json @@ -77,6 +77,10 @@ "error": { "type": "string" }, + "incoming_lifecycle": { + "type": "string", + "enum": ["covered", "absent", "unknown"] + }, "other_servers": { "type": "array", "items": { @@ -202,6 +206,9 @@ "type": "boolean", "description": "New objects will be encrypted. false: Dash should warn." }, + "aws_cli_problem": { + "type": "string" + }, "aws_cli": { "type": "boolean" }, diff --git a/internal/backup/accessory.go b/internal/backup/accessory.go index bcaea75..7a28f01 100644 --- a/internal/backup/accessory.go +++ b/internal/backup/accessory.go @@ -78,6 +78,9 @@ func (c *Client) newBackupWorkspace(ctx context.Context) (string, func(), error) return "", nil, fmt.Errorf("creating backup workspace: %w", err) } workDir := strings.TrimSpace(runOut) + // Says how long this run may live, for a janitor that does not know its deadline. + // Best effort: without it the fixed leftover age applies. + c.exec.Run(ctx, fmt.Sprintf(`printf '%%s\n' "$(( $(date +%%s) + %d ))" > %s`, int64((c.backupDeadline()+time.Hour).Seconds()), ssh.ShellQuote(workDir+"/"+workspaceExpiry))) return workDir, func() { rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) defer cancel() diff --git a/internal/backup/backup.go b/internal/backup/backup.go index 546ffff..a18e86c 100644 --- a/internal/backup/backup.go +++ b/internal/backup/backup.go @@ -801,7 +801,40 @@ func (c *Client) SetSchedule(ctx context.Context, schedule, command, marker stri // AWSCLIRemedy is the exact way to give a server the aws CLI. `teploy setup` // runs AWSCLIInstallScript; Ubuntu 24.04+ has no apt package for it. -const AWSCLIRemedy = "run `teploy setup` against the server (it installs the distribution's package, else AWS's bundle with its signature checked), or on the server: apt-get install -y awscli, else AWS's installer as documented at https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html" +const AWSCLIRemedy = "run `teploy setup` against the server (it installs the distribution's package if that is new enough, else AWS's bundle with its signature checked), or on the server: AWS's installer as documented at https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html" + +// The oldest aws CLIs teploy runs on: the first to send a checksum header on every +// upload by default. A store with Object Lock (B2's included) refuses an upload with +// neither that nor Content-MD5. --checksum-algorithm came earlier (2.18, 1.35); aws +// CLI's own changelog, "default integrity protections". +const ( + awsCLIMinV2Minor = 23 + awsCLIMinV1Minor = 37 +) + +var awsCLIMinimum = fmt.Sprintf("2.%d.0 (or 1.%d.0)", awsCLIMinV2Minor, awsCLIMinV1Minor) + +var awsVersionRE = regexp.MustCompile(`aws-cli/(\d+)\.(\d+)\.(\d+)`) + +// AWSCLIVersionProblem reads `aws --version` output and says what is wrong with it, +// "" when nothing is, or when it is not a version (a wrapper): the first upload would +// answer for that. +func AWSCLIVersionProblem(versionOutput string) string { + m := awsVersionRE.FindStringSubmatch(versionOutput) + if m == nil { + return "" + } + major, _ := strconv.Atoi(m[1]) + minor, _ := strconv.Atoi(m[2]) + if (major == 2 && minor < awsCLIMinV2Minor) || (major == 1 && minor < awsCLIMinV1Minor) || major < 1 { + return fmt.Sprintf("aws CLI %s.%s.%s is older than %s, the first to send a checksum on every upload, which a bucket with Object Lock requires", m[1], m[2], m[3], awsCLIMinimum) + } + return "" +} + +// awsCLIShellOK is a shell function: is the installed aws new enough. Same numbers +// as above (a test holds them together). +const awsCLIShellOK = `aws_ok() { v=$(aws --version 2>&1) || return 1; v=${v#aws-cli/}; v=${v%% *}; a=${v%%.*}; r=${v#*.}; b=${r%%.*}; case "$a" in 1) [ "$b" -ge 37 ];; 2) [ "$b" -ge 23 ];; *) [ "$a" -gt 2 ];; esac; }` // awsCLIKeyFingerprint and awsCLIPublicKey pin AWS's bundle signer, from // docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html (key @@ -845,11 +878,16 @@ E91G7bb0hOb/cA== // against the pinned key first (TLS alone is not enough for that). python3 unpacks // it (unzip may be absent), which drops the file modes, so they are restored for // every user before installing: with u+x alone the installed aws runs for root and -// "Permission denied"s for cron. Success is `aws --version` running, not `command -// -v aws`: the glibc bundle on a musl host is present and cannot start. +// "Permission denied"s for cron. Success is `aws --version` running at the minimum +// version, not `command -v aws`: the glibc bundle on a musl host is present and cannot +// start. A distribution package too old to use is removed, because cron's PATH would +// find it before the bundle in /usr/local/bin. const AWSCLIInstallScript = `set -e export DEBIAN_FRONTEND=noninteractive -if apt-get install -y -qq awscli >/dev/null 2>&1 && aws --version >/dev/null 2>&1; then exit 0; fi +` + awsCLIShellOK + ` +if aws_ok; then exit 0; fi +if apt-get install -y -qq awscli >/dev/null 2>&1 && aws_ok; then exit 0; fi +apt-get remove -y -qq awscli >/dev/null 2>&1 || true command -v gpg >/dev/null 2>&1 || apt-get install -y -qq gnupg >/dev/null 2>&1 || { echo "gpg is needed to verify AWS's bundle" >&2; exit 1; } d=$(mktemp -d /tmp/teploy-awscli.XXXXXXXX) trap 'rm -rf "$d"' EXIT @@ -864,12 +902,16 @@ gpg --batch --status-fd 1 --verify "$d/a.sig" "$d/a.zip" 2>/dev/null | grep -q " python3 -m zipfile -e "$d/a.zip" "$d/x" chmod -R 755 "$d/x/aws" "$d/x/aws/install" --update -aws --version >/dev/null 2>&1 || { echo "AWS's bundle installed but does not run here (a musl libc?): install your distribution's aws-cli package" >&2; exit 1; }` +aws_ok || { echo "AWS's bundle installed but does not run here (a musl libc?): install your distribution's aws-cli package, at least 2.23.0" >&2; exit 1; }` func (c *Client) ensureAWSCLI(ctx context.Context) error { // That it runs, not only that it is there: a glibc bundle on a musl host is. - if _, err := c.exec.Run(ctx, "which aws >/dev/null && aws --version >/dev/null 2>&1"); err != nil { + out, err := c.exec.Run(ctx, "which aws >/dev/null && aws --version 2>&1") + if err != nil { return fmt.Errorf("aws CLI not found or not runnable on server — %s", AWSCLIRemedy) } + if problem := AWSCLIVersionProblem(out); problem != "" { + return fmt.Errorf("%s — %s", problem, AWSCLIRemedy) + } return nil } diff --git a/internal/backup/engines_integration_test.go b/internal/backup/engines_integration_test.go index 2886268..747b92e 100644 --- a/internal/backup/engines_integration_test.go +++ b/internal/backup/engines_integration_test.go @@ -43,7 +43,7 @@ if [ "$1" = s3api ]; then shift 2; b=""; k="" while [ $# -gt 0 ]; do case "$1" in --bucket) b=$2; shift 2;; --key) k=$2; shift 2;; *) shift;; esac; done f="$root/$b/$k"; [ -f "$f" ] || { echo "An error occurred (404) when calling the HeadObject operation" >&2; exit 254; } - printf '{"ContentLength":%s,"Metadata":{"sha256":"%s"}}\n' "$(wc -c < "$f" | tr -d ' ')" "$(sha256sum "$f" | cut -d' ' -f1)" + printf '{"ContentLength":%s,"LastModified":"%s","Metadata":{"sha256":"%s"}}\n' "$(wc -c < "$f" | tr -d ' ')" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$(sha256sum "$f" | cut -d' ' -f1)" exit 0 fi [ "$1" = s3 ] || { echo "fake aws: only s3 is supported" >&2; exit 2; } @@ -52,7 +52,7 @@ args="" recursive=no while [ $# -gt 0 ]; do case "$1" in - --region|--endpoint-url|--metadata) shift 2 ;; + --region|--endpoint-url|--metadata|--copy-props) shift 2 ;; --recursive) recursive=yes; shift ;; --only-show-errors) shift ;; *) args="$args $1"; shift ;; diff --git a/internal/backup/fleet.go b/internal/backup/fleet.go index 9932495..b3275d9 100644 --- a/internal/backup/fleet.go +++ b/internal/backup/fleet.go @@ -450,15 +450,15 @@ func (c *Client) RunFleetTarget(ctx context.Context, t FleetTarget, s3 S3Config, default: for _, row := range rows { if row.Accessory == t.Name && row.Restorable { - r.Status, r.Reason = TargetFailed, "this target has earlier backups and now holds nothing to back up: its databases are gone. Nothing was pruned" + r.Status, r.Reason = TargetFailed, "this target has earlier backups and now holds nothing to back up: its databases are gone. Nothing was pruned. If it is meant to be empty: teploy server backup-policy set --exclude "+excludeName(t) break } } } return r } - c.CleanStaleIncoming(ctx, t.App, s3) if t.Retention.IsZero() { + c.CleanStaleIncoming(ctx, t.App, s3) return r } deleted, perr := c.PruneBackups(ctx, t.App, prefix, s3, t.Retention) @@ -599,13 +599,34 @@ func (c *Client) ProbeTarget(ctx context.Context, s3 S3Config) error { return err } -// RunLockPath is where the overlap lock lives: beside the policy. +// excludeName is how `--exclude` names a target. +func excludeName(t FleetTarget) string { + switch { + case t.Kind == TargetVolumes: + return t.App + "/volumes" + case t.App == AdoptedApp || t.App == "": + return t.Container + } + return t.App + "/" + t.Name +} + +// RunLockPath is where the overlap lock lives: beside the policy when this user can +// write there, else in its own config. A policy read through sudo sits in a +// directory the run itself cannot touch, and a cron line failing on its lock is +// every backup failing. func RunLockPath(policyPath string) string { if policyPath != "" { - return path.Dir(policyPath) + "/backup-run.lock" + p := path.Dir(policyPath) + "/backup-run.lock" + if f, err := os.OpenFile(p, os.O_CREATE|os.O_RDWR, 0o600); err == nil { + f.Close() + return p + } } if d := os.Getenv("HOME"); d != "" { - return d + "/.config/teploy/backup-run.lock" + dir := d + "/.config/teploy" + if os.MkdirAll(dir, 0o700) == nil { + return dir + "/backup-run.lock" + } } return "/tmp/teploy-backup-run.lock" } diff --git a/internal/backup/fleet_test.go b/internal/backup/fleet_test.go index d44171a..7cade47 100644 --- a/internal/backup/fleet_test.go +++ b/internal/backup/fleet_test.go @@ -351,7 +351,7 @@ func TestFleetStatusCoverage(t *testing.T) { evA := `{"v":1,"id":"1","seq":1,"ts":"2026-02-01T11:00:00Z","app":"a","kind":"backup.run","outcome":"succeeded","detail":{"accessory":"db","kind":"accessory","policy_managed":true}}` evAd := `{"v":1,"id":"2","seq":1,"ts":"2026-01-01T11:00:00Z","app":"_adopted","kind":"backup.run","outcome":"failed","detail":{"accessory":"b_mysql","kind":"accessory","error":"boom","policy_managed":true}}` mock := ssh.NewMockExecutor("h", - ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "path /etc/teploy/backup-policy.json\n" + `{"v":"1","interval":"1h","recipient":"` + testOwnerRecipient + `","heartbeat_urls":["https://d/api/push/SECRETTOKEN"],"adopted":[{"container":"b_mysql","databases":"all"},{"container":"gone","databases":"all"}]}`}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ]", Output: "path /etc/teploy/backup-policy.json\n" + `{"v":"1","interval":"1h","recipient":"` + testOwnerRecipient + `","heartbeat_urls":["https://d/api/push/SECRETTOKEN"],"adopted":[{"container":"b_mysql","databases":"all"},{"container":"gone","databases":"all"}]}`}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"bk","region":"r","prefix":"p","credentials":"SECRETARMOR"}`}, ssh.MockCommand{Match: "command -v aws >/dev/null", Output: "aws\nage\n"}, ssh.MockCommand{Match: "crontab -l", Output: "5 * * * * /usr/local/bin/teploy backup run --all # [teploy-backup-all]\n"}, @@ -474,7 +474,7 @@ func TestRunFleetTargetStoredNothing(t *testing.T) { target := FleetTarget{Kind: TargetAccessory, App: "a", Name: "db", Container: "a-db", Fact: &ContainerFact{Image: "postgres:17", Env: map[string]string{"PG_MAJOR": "17", "POSTGRES_DB": "app"}}, Retention: RetentionPolicy{KeepLast: 1}} mock := mk("2026-01-01 00:00:00 1000 a/accessories/db/20260101-000000.pgdump\n") r := NewClient(mock, io.Discard).RunFleetTarget(context.Background(), target, S3Config{Bucket: "b", Region: "r"}, BackupOptions{}) - if r.Status != TargetFailed || !strings.Contains(r.Reason, "earlier backups") { + if r.Status != TargetFailed || !strings.Contains(r.Reason, "earlier backups") || !strings.Contains(r.Reason, "--exclude a/db") { t.Fatalf("%+v", r) } for _, c := range mock.Calls { diff --git a/internal/backup/list.go b/internal/backup/list.go index 96c5f5b..9b084b8 100644 --- a/internal/backup/list.go +++ b/internal/backup/list.go @@ -45,35 +45,38 @@ var listedIDRE = regexp.MustCompile(`^([0-9]{8}-[0-9]{6}(?:-[0-9a-f]{16})?)\.(ta // ListBackupsRecursive lists every object under s3://// in one // recursive call per location (current prefix, then any previous one) and // classifies each by its key layout. Objects whose name does not match the -// CLI's naming (strays, pending uploads) are skipped, and so are objects below -// their engine's plausibility floor: neither is a backup anyone should choose. +// CLI's naming (strays) are skipped, and so are objects below their engine's +// plausibility floor: neither is a backup anyone should choose. A previous location +// that does not answer is a warning: it must not make the current one unreadable. func (c *Client) ListBackupsRecursive(ctx context.Context, app string, s3 S3Config) ([]ListedBackup, error) { - rows, _, err := c.listBackupsAndPending(ctx, app, s3) + rows, warnings, err := c.listBackups(ctx, app, s3) + for _, w := range warnings { + fmt.Fprintf(c.out, "Warning: %s\n", w) + } return rows, err } -// listBackupsAndPending is ListBackupsRecursive plus the bucket-relative keys of -// verification-pending uploads, which status and doctor report. -func (c *Client) listBackupsAndPending(ctx context.Context, app string, s3 S3Config) ([]ListedBackup, []string, error) { +// listBackups is ListBackupsRecursive, handing the warnings back. +func (c *Client) listBackups(ctx context.Context, app string, s3 S3Config) ([]ListedBackup, []string, error) { if err := c.ensureAWSCLI(ctx); err != nil { return nil, nil, err } var rows []ListedBackup - var pending []string + var warnings []string for _, loc := range s3.locations() { keys, err := c.listTree(ctx, loc, app+"/") if err != nil { - return nil, nil, fmt.Errorf("listing backups: %w", err) + if loc.isPrevious { + warnings = append(warnings, fmt.Sprintf("previous prefix %s did not answer, so its backups are not listed: %s", loc.Prefix, oneLine(err.Error()))) + continue + } + return nil, warnings, fmt.Errorf("listing backups: %w", err) } for _, k := range keys { rel, ok := loc.relToBase(k.Key) if !ok { continue } - if strings.Contains(rel, "/"+incomingDir+"/") { - pending = append(pending, k.Key) - continue - } row, ok := classifyListedKey(app, rel) if !ok { continue @@ -95,7 +98,7 @@ func (c *Client) listBackupsAndPending(ctx context.Context, app string, s3 S3Con } return rows[i].Key < rows[j].Key }) - return rows, pending, nil + return rows, warnings, nil } // classifyListedKey maps one key relative to the base (the bucket root, or the diff --git a/internal/backup/policy.go b/internal/backup/policy.go index 2a6367d..96a1029 100644 --- a/internal/backup/policy.go +++ b/internal/backup/policy.go @@ -17,8 +17,9 @@ import ( // The server backup policy (G4): /etc/teploy/backup-policy.json, read at run // time by the one cron line `teploy backup run --all`, so a database deployed // tomorrow is covered with no command. Mode 0600: heartbeat URLs carry tokens. -// A non-root user that cannot write /etc/teploy falls back to -// ~/.config/teploy/backup-policy.json, like the backup target. +// A non-root user with passwordless sudo reads, writes and removes it through sudo, +// so there is one policy; one with neither keeps its own in +// ~/.config/teploy/backup-policy.json, and is refused if that would be shadowed. // Policy paths. const ( @@ -142,9 +143,13 @@ func (p FleetPolicy) Validate() error { return fmt.Errorf("tmp_dir must be an absolute path of letters, digits and . _ - /") } if p.TargetDeadline != "" { - if d, err := time.ParseDuration(p.TargetDeadline); err != nil || d <= 0 { + d, err := time.ParseDuration(p.TargetDeadline) + if err != nil || d <= 0 { return fmt.Errorf("target_deadline must be a positive duration such as 12h") } + if d > MaxDeadline { + return fmt.Errorf("target_deadline %s is over the %s a whole run may last, so it would be capped silently", p.TargetDeadline, MaxDeadline) + } } for _, e := range p.Excludes { if err := validateExclude(e); err != nil { @@ -265,25 +270,27 @@ func IntervalCron(interval, seed string) (string, error) { } // policyReadMarker starts readPolicyScript. -const policyReadMarker = `if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]; then` +const policyReadMarker = `if [ -d /etc/teploy ] && { [ ! -x /etc/teploy ] || { [ -e /etc/teploy/backup-policy.json ] && [ ! -r /etc/teploy/backup-policy.json ]; }; }; then` // readPolicyScript prints "absent" or "path

\n", or exits 3 when it // cannot tell. Root keeps /etc/teploy 0700, and to a non-root user a file in there // does not even exist (`[ -e ]` is false): it read as "no policy", which cleared -// the recipient and uploaded plaintext on a server configured to encrypt. So an -// untraversable /etc/teploy is asked through passwordless sudo when that works -// (deploy users usually have it, cron included); a user with its own policy in -// ~/.config has made its choice; anyone else gets a refusal, never a guess. +// the recipient and uploaded plaintext on a server configured to encrypt. So a +// policy this user cannot see or read is asked through passwordless sudo when that +// works (deploy users usually have it, cron included); a user with its own policy in +// ~/.config has made its choice; anyone else gets a refusal, never a guess. A +// file that is there and unreadable (/etc/teploy opened to 711: the file stays 0600) +// is refused even then, by the loop below. const readPolicyScript = policyReadMarker + ` if command -v sudo >/dev/null 2>&1 && s=$(sudo -n sh -c 'if [ -e /etc/teploy/backup-policy.json ]; then echo present; cat /etc/teploy/backup-policy.json; else echo absent; fi' 2>/dev/null); then case "$s" in present*) printf 'path /etc/teploy/backup-policy.json\n%s\n' "${s#present}"; exit 0;; esac elif [ ! -e "$HOME"/.config/teploy/backup-policy.json ]; then - echo "cannot tell whether this server's backups must be encrypted: /etc/teploy is not readable by $(id -un). Connect as root, give this user passwordless sudo, run on the server: chmod 711 /etc/teploy, or set this user's own policy (teploy server backup-policy set)" >&2 + echo "cannot tell whether this server's backups must be encrypted: /etc/teploy is not readable by $(id -un). Connect as root, give this user passwordless sudo, or set this user's own policy (teploy server backup-policy set)" >&2 exit 3 fi fi for p in /etc/teploy/backup-policy.json "$HOME"/.config/teploy/backup-policy.json; do -if [ -e "$p" ] && [ ! -r "$p" ]; then echo "cannot read $p: run as its owner" >&2; exit 3; fi +if [ -e "$p" ] && [ ! -r "$p" ]; then echo "cannot read $p: connect as root, or give this user passwordless sudo" >&2; exit 3; fi if [ -r "$p" ]; then printf 'path %s\n' "$p"; cat "$p"; exit 0; fi done echo absent` @@ -323,27 +330,43 @@ func ReadFleetPolicy(ctx context.Context, exec ssh.Executor) (FleetPolicy, error return p, nil } -const writeFleetPolicyScript = `d= +// policyStoreScript stores stdin as the file $d, atomically at 0600. No single +// quotes: it also runs inside `sudo sh -c '...'`. +const policyStoreScript = `umask 077 +t=$(mktemp "$(dirname "$d")/.backup-policy.XXXXXXXX") || exit 1 +cat > "$t" || { rm -f -- "$t"; exit 1; } +chmod 600 "$t" || { rm -f -- "$t"; exit 1; } +mv -f -- "$t" "$d" || { rm -f -- "$t"; exit 1; } +printf "written %s\n" "$d"` + +// writeFleetPolicyScript writes where readPolicyScript will look first, so what set +// stored is what show, the run and rotation see: /etc/teploy directly when this user +// can, through passwordless sudo when it cannot. ~/.config is only for a user with +// neither, and only when no policy in /etc would shadow it. A policy this user can +// see and cannot replace is a refusal: a copy it never reads would look stored. +const writeFleetPolicyScript = `e=/etc/teploy/backup-policy.json if [ "$(id -u)" = 0 ]; then mkdir -p /etc/teploy && chmod 700 /etc/teploy || exit 1 - d=/etc/teploy/backup-policy.json -elif [ -w /etc/teploy ] 2>/dev/null; then - d=/etc/teploy/backup-policy.json + d=$e +elif [ -w /etc/teploy ] && [ -x /etc/teploy ]; then + d=$e +elif [ -d /etc/teploy ] && command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then + exec sudo -n sh -c 'd=/etc/teploy/backup-policy.json +` + policyStoreScript + `' +elif [ -e "$e" ]; then + echo "$e exists and $(id -un) cannot replace it, so a policy stored anywhere else would never be read. Connect as root, or give this user passwordless sudo" >&2 + exit 3 else - dir=${XDG_CONFIG_HOME:-$HOME/.config}/teploy + dir=$HOME/.config/teploy mkdir -p "$dir" && chmod 700 "$dir" || exit 1 d=$dir/backup-policy.json echo fallback >&2 fi -umask 077 -t=$(mktemp "$(dirname "$d")/.backup-policy.XXXXXXXX") || exit 1 -cat > "$t" || { rm -f -- "$t"; exit 1; } -chmod 600 "$t" || { rm -f -- "$t"; exit 1; } -mv -f -- "$t" "$d" || { rm -f -- "$t"; exit 1; } -printf 'written %s\n' "$d"` +` + policyStoreScript // WriteFleetPolicy stores the policy atomically at 0600 and returns the path and -// whether the non-root fallback was used. +// whether the non-root fallback was used. It reads the policy back through the path +// every reader uses: a stored policy that is not the one in force is an error. func WriteFleetPolicy(ctx context.Context, exec ssh.Executor, p FleetPolicy) (string, bool, error) { if err := p.Validate(); err != nil { return "", false, err @@ -364,12 +387,32 @@ func WriteFleetPolicy(ctx context.Context, exec ssh.Executor, p FleetPolicy) (st if len(f) != 2 || f[0] != "written" { return "", false, fmt.Errorf("writing the backup policy: unrecognized output framing") } + back, err := ReadFleetPolicy(ctx, exec) + if err != nil { + return "", false, fmt.Errorf("the backup policy was written to %s but cannot be read back: %w", f[1], err) + } + if back.Path != f[1] { + return "", false, fmt.Errorf("the backup policy was written to %s but this server reads %s, so it is not in force", f[1], back.Path) + } return f[1], strings.Contains(string(res.Stderr), "fallback"), nil } -const unsetFleetPolicyScript = `for p in /etc/teploy/backup-policy.json "$HOME"/.config/teploy/backup-policy.json; do -if [ -e "$p" ]; then rm -f -- "$p" || exit 1; echo "removed $p"; fi -done` +// unsetFleetPolicyScript removes what readPolicyScript would read. Exit 3: a policy +// is, or may be, still in force and this user cannot remove it. +const unsetFleetPolicyScript = `e=/etc/teploy/backup-policy.json +h=$HOME/.config/teploy/backup-policy.json +if [ "$(id -u)" != 0 ] && [ -d /etc/teploy ] && { [ ! -x /etc/teploy ] || { [ -e "$e" ] && [ ! -w /etc/teploy ]; }; }; then + if command -v sudo >/dev/null 2>&1 && r=$(sudo -n sh -c 'if [ -e /etc/teploy/backup-policy.json ]; then rm -f -- /etc/teploy/backup-policy.json && echo removed; else echo absent; fi' 2>/dev/null); then + if [ "$r" = removed ]; then echo "removed $e"; fi + elif [ -e "$e" ] || [ ! -e "$h" ]; then + echo "cannot remove the backup policy: $(id -un) cannot change /etc/teploy, so a policy there stays in force. Connect as root, or give this user passwordless sudo" >&2 + exit 3 + fi +elif [ -e "$e" ]; then + rm -f -- "$e" || exit 1 + echo "removed $e" +fi +if [ -e "$h" ]; then rm -f -- "$h" || exit 1; echo "removed $h"; fi` // UnsetFleetPolicy removes the policy file(s), returning what it removed. func UnsetFleetPolicy(ctx context.Context, exec ssh.Executor) ([]string, error) { diff --git a/internal/backup/retention.go b/internal/backup/retention.go index 749b3cb..84aad72 100644 --- a/internal/backup/retention.go +++ b/internal/backup/retention.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "path" "regexp" "slices" "sort" @@ -35,8 +36,11 @@ type RetentionPolicy struct { KeepWeekly int KeepMonthly int MaxAgeDays int - // IgnoreClock lets a prune run despite a clock disagreement (--ignore-clock). + // IgnoreClock lets a prune run despite a clock disagreement or an unreadable + // store clock (--ignore-clock). IgnoreClock bool + // AllowMassDelete lets one prune delete most of a series (--allow-mass-delete). + AllowMassDelete bool } // IsZero reports whether the policy prunes nothing. @@ -196,7 +200,6 @@ type pruneTree struct { root []pruneObject // primary objects series map[string][]pruneObject // databases//: per-database objects sidecars []pruneObject // globals/, meta/, safety/, databases//meta/, teploy/ siblings - incoming []pruneObject // verification-pending uploads, at any depth // safety are ids of safety copies a destructive operation took, by the time // they were taken: protected for safetyWindow. safety map[string]time.Time @@ -206,6 +209,17 @@ type pruneTree struct { // to notice that a restore or an upgrade went wrong, and go back. const safetyWindow = 7 * 24 * time.Hour +// safetyKeep caps the safety copies protected per target, newest first: thirty +// deploys in a week must not hold thirty full copies past retention. Older ones +// fall back to the ordinary rules. +const safetyKeep = 5 + +// pruneBrakeMin and the more-than-half rule are the brake: a normal run deletes a +// backup or two, so one that would delete most of a series has a wrong clock (a +// store on the same host shares it, so the skew check cannot see it) or a mistaken +// policy, and a person should say which. +const pruneBrakeMin = 3 + // safetyDir marks an id as a safety copy: /safety/.safety. const safetyDir = "safety" @@ -249,8 +263,6 @@ func (c *Client) loadPruneTree(ctx context.Context, app, prefix string, s3 S3Con e.bad = k.Size < artifactFloor(file) obj := pruneObject{key: "s3://" + loc.Bucket + "/" + k.Key, id: idOf(file), entry: e, loc: loc} switch { - case slices.Contains(segs, incomingDir): - tree.incoming = append(tree.incoming, obj) case len(segs) == 1: tree.root = append(tree.root, obj) case len(segs) == 2 && segs[0] == safetyDir: @@ -278,12 +290,7 @@ func (c *Client) loadPruneTree(ctx context.Context, app, prefix string, s3 S3Con } segs := strings.Split(k.Key, "/") if e, ok := newBackupEntry(segs[len(segs)-1]); ok { - obj := pruneObject{key: "s3://" + loc.Bucket + "/" + k.Key, id: idOf(segs[len(segs)-1]), entry: e, loc: loc} - if slices.Contains(segs, incomingDir) { - tree.incoming = append(tree.incoming, obj) - } else { - tree.sidecars = append(tree.sidecars, obj) - } + tree.sidecars = append(tree.sidecars, pruneObject{key: "s3://" + loc.Bucket + "/" + k.Key, id: idOf(segs[len(segs)-1]), entry: e, loc: loc}) } } } @@ -307,17 +314,40 @@ func entriesOf(objs []pruneObject) []backupEntry { // decide a database's history. // - An id the primary series keeps is also kept in every database's series (a // consistent point in time), as an extra keep and never a reason to delete. -// - A safety copy a destructive operation took outlasts every rule for -// safetyWindow. +// - The newest safetyKeep safety copies a destructive operation took outlast every +// rule for safetyWindow. +// - A backup dated in the future is not arithmetic: it is kept, and never counts as +// the newest, so it cannot cost the real newest backup its place. // - Sidecars go when no surviving object has their id and they are older than // the newest backup. func planPrune(tree pruneTree, p RetentionPolicy, now time.Time) (primaries, sidecars []pruneObject) { surviving := map[string]bool{} - protected := func(id string) bool { - t, ok := tree.safety[id] - return ok && now.Sub(t) < safetyWindow + safetyIDs := make([]string, 0, len(tree.safety)) + for id, t := range tree.safety { + if now.Sub(t) < safetyWindow { + safetyIDs = append(safetyIDs, id) + } } - pick := func(objs []pruneObject, keepToo func(pruneObject) bool) []pruneObject { + sort.Slice(safetyIDs, func(i, j int) bool { + a, b := tree.safety[safetyIDs[i]], tree.safety[safetyIDs[j]] + if !a.Equal(b) { + return a.After(b) + } + return safetyIDs[i] > safetyIDs[j] + }) + if len(safetyIDs) > safetyKeep { + safetyIDs = safetyIDs[:safetyKeep] + } + protected := func(id string) bool { return slices.Contains(safetyIDs, id) } + pick := func(all []pruneObject, keepToo func(pruneObject) bool) []pruneObject { + var objs []pruneObject + for _, o := range all { + if futureDated(o, now) { + surviving[o.id] = true + continue + } + objs = append(objs, o) + } del := selectForDeletion(entriesOf(objs), p, now) byEntry := map[string]pruneObject{} for _, o := range objs { @@ -356,7 +386,7 @@ func planPrune(tree pruneTree, p RetentionPolicy, now time.Time) (primaries, sid var newest time.Time for _, objs := range append([][]pruneObject{tree.root}, seriesValues(tree.series)...) { for _, o := range objs { - if o.entry.time.After(newest) { + if o.entry.time.After(newest) && !futureDated(o, now) { newest = o.entry.time } } @@ -366,13 +396,45 @@ func planPrune(tree pruneTree, p RetentionPolicy, now time.Time) (primaries, sid sidecars = append(sidecars, o) } } - // Pending uploads a killed run left behind. Younger ones may be a live run. - for _, o := range tree.incoming { - if now.Sub(o.entry.time) > tmpLeftoverMinutes*time.Minute { - sidecars = append(sidecars, o) + return primaries, sidecars +} + +// futureDated says a backup's date is past anything clock skew explains. +func futureDated(o pruneObject, now time.Time) bool { + return o.entry.time.Sub(now) > maxClockSkew +} + +// pruneBrake refuses a plan that would delete most of a series. +func pruneBrake(tree pruneTree, primaries []pruneObject, now time.Time) error { + doomed := map[string]bool{} + for _, o := range primaries { + doomed[o.key] = true + } + check := func(what string, objs []pruneObject) error { + good, gone := 0, 0 + for _, o := range objs { + if o.entry.bad || futureDated(o, now) { + continue + } + good++ + if doomed[o.key] { + gone++ + } } + if gone >= pruneBrakeMin && gone*2 > good { + return fmt.Errorf("%w: this run would delete %d of the %d backups of %s, so nothing was pruned (a wrong clock or a mistaken policy looks exactly like this). If it is meant: teploy backup prune with the same rules and --allow-mass-delete", ErrPruneBrake, gone, good, what) + } + return nil } - return primaries, sidecars + if err := check("the primary series", tree.root); err != nil { + return err + } + for db, objs := range tree.series { + if err := check("database "+decodeDBSegment(db), objs); err != nil { + return err + } + } + return nil } func seriesValues(m map[string][]pruneObject) [][]pruneObject { @@ -383,26 +445,27 @@ func seriesValues(m map[string][]pruneObject) [][]pruneObject { return out } -// CleanStaleIncoming removes pending uploads a killed run left anywhere in a -// namespace, DR bundles included, whatever its retention. Best effort, and only in -// a namespace this server alone writes to. +// CleanStaleIncoming removes pending uploads a killed run left for a namespace, DR +// bundles included, whatever its retention. Best effort, and only in a namespace +// this server alone writes to. func (c *Client) CleanStaleIncoming(ctx context.Context, app string, s3 S3Config) { owners, err := c.namespaceOwners(ctx, s3, app) fp := c.fingerprint(ctx) if err != nil || fp == "" || slices.ContainsFunc(owners, func(o string) bool { return o != fp }) { return } - keys, err := c.listTree(ctx, s3, app+"/") + keys, err := c.listTree(ctx, s3, incomingDir+"/"+app+"/") if err != nil { return } now := time.Now().UTC() for _, k := range keys { - segs := strings.Split(k.Key, "/") - if !slices.Contains(segs, incomingDir) { + // Whatever a listing says, nothing outside the pending tree goes through here. + rel, ok := s3.relToBase(k.Key) + if !ok || !strings.HasPrefix(rel, incomingDir+"/"+app+"/") { continue } - if e, ok := newBackupEntry(segs[len(segs)-1]); ok && now.Sub(e.time) > tmpLeftoverMinutes*time.Minute { + if e, ok := newBackupEntry(path.Base(rel)); ok && now.Sub(e.time) > incomingStaleAfter { c.exec.Run(ctx, s3.AWS("s3 rm "+ssh.ShellQuote("s3://"+s3.Bucket+"/"+k.Key))) } } @@ -411,20 +474,14 @@ func (c *Client) CleanStaleIncoming(ctx context.Context, app string, s3 S3Config // ErrClockSkew marks a prune refused because dates cannot be trusted. var ErrClockSkew = errors.New("clock disagreement") +// ErrPruneBrake marks a prune refused because it would delete most of a series. +var ErrPruneBrake = errors.New("prune brake") + // checkPruneClock refuses arithmetic on dates that cannot be right: this machine's -// clock against the store's, and any backup dated in the future. -func checkPruneClock(tree pruneTree, storeNow, now time.Time) error { - if !storeNow.IsZero() { - if d := now.Sub(storeNow); d > maxClockSkew || d < -maxClockSkew { - return fmt.Errorf("%w: this machine's clock says %s and the store's says %s, so nothing was pruned (a wrong clock deletes real backups). Fix the clock, or pass --ignore-clock", ErrClockSkew, now.Format(time.RFC3339), storeNow.Format(time.RFC3339)) - } - } - for _, objs := range append([][]pruneObject{tree.root}, seriesValues(tree.series)...) { - for _, o := range objs { - if o.entry.time.Sub(now) > maxClockSkew { - return fmt.Errorf("%w: %s is dated in the future (now %s), and would count as the newest backup, so nothing was pruned. Remove it if it is wrong, or pass --ignore-clock", ErrClockSkew, o.key, now.Format(time.RFC3339)) - } - } +// clock against the store's. +func checkPruneClock(storeNow, now time.Time) error { + if d := now.Sub(storeNow); d > maxClockSkew || d < -maxClockSkew { + return fmt.Errorf("%w: this machine's clock says %s and the store's says %s, so nothing was pruned (a wrong clock deletes real backups). Fix the clock, or pass --ignore-clock", ErrClockSkew, now.Format(time.RFC3339), storeNow.Format(time.RFC3339)) } return nil } @@ -432,9 +489,10 @@ func checkPruneClock(tree pruneTree, storeNow, now time.Time) error { // PruneBackups deletes backups under s3:///// that fall // outside the retention policy, with every sidecar they keep beside them, // returning the deleted backups' names. A zero policy is a no-op. A namespace this -// server cannot prove is its alone is refused, and so is a clock that cannot be -// trusted. A refused delete (bucket locks, keys without delete rights) does not -// stop the rest: errors are returned together with what was deleted. +// server cannot prove is its alone is refused, so is a clock that cannot be read or +// trusted, and so is a run that would delete most of a series. A refused delete +// (bucket locks, keys without delete rights) does not stop the rest: errors are +// returned together with what was deleted. func (c *Client) PruneBackups(ctx context.Context, app, prefix string, s3 S3Config, p RetentionPolicy) ([]string, error) { if p.IsZero() { return nil, nil @@ -442,10 +500,19 @@ func (c *Client) PruneBackups(ctx context.Context, app, prefix string, s3 S3Conf if err := c.ensureAWSCLI(ctx); err != nil { return nil, err } - storeNow, err := c.claimNamespace(ctx, s3, app) - if err != nil { + if err := c.claimNamespace(ctx, s3, app); err != nil { return nil, err } + now := time.Now().UTC() + if !p.IgnoreClock { + storeNow, err := c.storeClock(ctx, s3) + if err != nil { + return nil, fmt.Errorf("%w: could not read the store's clock, so nothing was pruned (a wrong clock deletes real backups). Pass --ignore-clock to prune anyway: %v", ErrClockSkew, err) + } + if err := checkPruneClock(storeNow, now); err != nil { + return nil, err + } + } // A previous location is checked on its own: owned means this server's old // objects, and a marker from another server there overrides that word. var errs []error @@ -457,7 +524,7 @@ func (c *Client) PruneBackups(ctx context.Context, app, prefix string, s3 S3Conf } loc := s3 loc.Prefix, loc.PreviousPrefixes = prev.Prefix, nil - if _, cerr := c.claimNamespace(ctx, loc, app); cerr != nil { + if cerr := c.claimNamespace(ctx, loc, app); cerr != nil { errs = append(errs, cerr) continue } @@ -467,13 +534,20 @@ func (c *Client) PruneBackups(ctx context.Context, app, prefix string, s3 S3Conf if err != nil { return nil, err } - now := time.Now().UTC() - if !p.IgnoreClock { - if err := checkPruneClock(tree, storeNow, now); err != nil { + primaries, sidecars := planPrune(tree, p, now) + if !p.AllowMassDelete { + if err := pruneBrake(tree, primaries, now); err != nil { return nil, err } } - primaries, sidecars := planPrune(tree, p, now) + // Retention goes on around it, and the run says so until someone looks. + for _, objs := range append([][]pruneObject{tree.root}, seriesValues(tree.series)...) { + for _, o := range objs { + if futureDated(o, now) { + errs = append(errs, fmt.Errorf("%s is dated in the future (now %s): it is kept, and does not count as a backup. Remove it if it is wrong", o.key, now.Format(time.RFC3339))) + } + } + } var deleted []string for _, o := range primaries { @@ -489,5 +563,8 @@ func (c *Client) PruneBackups(ctx context.Context, app, prefix string, s3 S3Conf errs = append(errs, fmt.Errorf("deleting %s: %w", o.key, err)) } } + for _, loc := range cfg.locations() { + c.CleanStaleIncoming(ctx, app, loc) + } return deleted, errors.Join(errs...) } diff --git a/internal/backup/retention_test.go b/internal/backup/retention_test.go index fe8ca60..5656b85 100644 --- a/internal/backup/retention_test.go +++ b/internal/backup/retention_test.go @@ -108,10 +108,10 @@ func TestPruneBackups(t *testing.T) { var rmCalls int for _, c := range mock.Calls { - if strings.HasPrefix(c, "aws s3 rm") { + if strings.HasPrefix(c, "aws s3 rm") && !strings.Contains(c, "/.clock-") { rmCalls++ } - if strings.Contains(c, "20240103-000000") && strings.HasPrefix(c, "aws s3 rm") { + if strings.Contains(c, "20240103-000000") && strings.HasPrefix(c, "aws s3 rm") && !strings.Contains(c, "/.clock-") { t.Errorf("newest backup must not be deleted: %s", c) } } diff --git a/internal/backup/review_regression_test.go b/internal/backup/review_regression_test.go new file mode 100644 index 0000000..8086350 --- /dev/null +++ b/internal/backup/review_regression_test.go @@ -0,0 +1,814 @@ +//go:build !windows + +package backup + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/useteploy/teploy/internal/ssh" +) + +// Regressions from the Slice 4 re-review, named after what they protect. + +func publishCalls(mock *ssh.MockExecutor) (copies, reuploads int, joined string) { + for _, c := range mock.Calls { + if strings.HasPrefix(c, "aws s3 cp 's3://b/.incoming/k/1.pgdump' 's3://b/k/1.pgdump'") { + copies++ + } + if strings.HasPrefix(c, "aws s3 cp '/w/x' 's3://b/k/1.pgdump' --metadata 'sha256=abc'") { + reuploads++ + } + } + return copies, reuploads, strings.Join(mock.Calls, "\n") +} + +const putCopyMatch = "aws s3 cp 's3://b/.incoming/k/1.pgdump' 's3://b/k/1.pgdump'" + +// B2 refuses a copy that would change the checksum algorithm, on every object big +// enough to be multipart. The verified local file goes straight to its final name. +func TestPutReuploadsWhenTheStoreRefusesTheCopy(t *testing.T) { + ctx := context.Background() + for _, refusal := range []string{ + "exit status 1: copy failed: An error occurred (InvalidArgument) when calling the UploadPartCopy operation: checksum algorithm cannot be changed", + "exit status 1: An error occurred (NotImplemented) when calling the GetObjectTagging operation: not implemented", + "exit status 1: An error occurred (InternalError) when calling the CompleteMultipartUpload operation: copy failed", + } { + mock := putMock(ssh.MockCommand{Match: putCopyMatch, Err: errors.New(refusal)}) + var out strings.Builder + obj, err := NewClient(mock, &out).put(ctx, S3Config{Bucket: "b", Region: "r"}, "/w/x", "s3://b/k/1.pgdump") + if err != nil { + t.Fatalf("%s: %v", refusal, err) + } + if obj.Key != "s3://b/k/1.pgdump" { + t.Fatalf("%+v", obj) + } + copies, reuploads, joined := publishCalls(mock) + if copies != 1 || reuploads != 1 { + t.Fatalf("copies %d reuploads %d: %s", copies, reuploads, joined) + } + // The final object is verified as itself: its bytes are read back, and the + // pending one is gone. + if !strings.Contains(joined, "{ aws s3 cp 's3://b/k/1.pgdump' -") || !strings.Contains(joined, "aws s3 rm 's3://b/.incoming/k/1.pgdump'") { + t.Fatalf("calls: %s", joined) + } + if !strings.Contains(out.String(), "uploaded straight to its final name") { + t.Fatalf("out: %s", out.String()) + } + } +} + +// A large object's re-upload keeps the provider checksum. +func TestPutReuploadOfALargeObjectStaysProviderChecked(t *testing.T) { + mock := putMock( + ssh.MockCommand{Match: putCopyMatch, Err: errors.New("exit status 1: An error occurred (InvalidRequest) when calling the UploadPartCopy operation")}, + ssh.MockCommand{Match: "aws s3api head-object", Output: `{"ContentLength":4096,"Metadata":{"sha256":"abc"},"ChecksumSHA256":"AAA=-3"}`}, + ) + cl := NewClient(mock, io.Discard) + cl.readBack = 1 + if _, err := cl.put(context.Background(), S3Config{Bucket: "b", Region: "r"}, "/w/x", "s3://b/k/1.pgdump"); err != nil { + t.Fatal(err) + } + if _, _, joined := publishCalls(mock); !strings.Contains(joined, "aws s3 cp '/w/x' 's3://b/k/1.pgdump' --metadata 'sha256=abc' --checksum-algorithm SHA256") { + t.Fatalf("calls: %s", joined) + } +} + +// A dead connection is not a refusal: a re-upload would fail the same way, and the +// run should say the real thing. A re-upload that fails or does not verify leaves +// nothing at the final name. +func TestPutFailuresThatAreNotRefusals(t *testing.T) { + ctx := context.Background() + mock := putMock(ssh.MockCommand{Match: putCopyMatch, Err: errors.New("exit status 1: Could not connect to the endpoint URL")}) + _, err := NewClient(mock, io.Discard).put(ctx, S3Config{Bucket: "b", Region: "r"}, "/w/x", "s3://b/k/1.pgdump") + if err == nil || !strings.Contains(err.Error(), "publishing the verified upload") { + t.Fatalf("err = %v", err) + } + if _, reuploads, joined := publishCalls(mock); reuploads != 0 || !strings.Contains(joined, "aws s3 rm 's3://b/.incoming/k/1.pgdump'") { + t.Fatalf("calls: %s", joined) + } + refused := ssh.MockCommand{Match: putCopyMatch, Err: errors.New("exit status 1: An error occurred (InvalidRequest) when calling the UploadPartCopy operation")} + mock = putMock(refused, ssh.MockCommand{Match: "aws s3 cp '/w/x' 's3://b/k/1.pgdump'", Err: errors.New("exit status 1: connection reset")}) + if _, err = NewClient(mock, io.Discard).put(ctx, S3Config{Bucket: "b", Region: "r"}, "/w/x", "s3://b/k/1.pgdump"); err == nil { + t.Fatal("a failed re-upload must fail") + } + if !strings.Contains(strings.Join(rmKeys(mock), ","), "s3://b/k/1.pgdump") { + t.Fatalf("final not cleaned: %v", rmKeys(mock)) + } + mock = putMock(refused, ssh.MockCommand{Match: "{ aws s3 cp 's3://b/k/1.pgdump' -", Output: "zzz -\n"}) + if _, err = NewClient(mock, io.Discard).put(ctx, S3Config{Bucket: "b", Region: "r"}, "/w/x", "s3://b/k/1.pgdump"); err == nil || !strings.Contains(err.Error(), "checksum mismatch") { + t.Fatalf("err = %v", err) + } + if !strings.Contains(strings.Join(rmKeys(mock), ","), "s3://b/k/1.pgdump") { + t.Fatalf("an unverified final object was kept: %v", rmKeys(mock)) + } +} + +// R2 has no GetObjectTagging, which the copy's default props read. +func TestPublishNamesItsCopyProps(t *testing.T) { + mock := putMock() + if _, err := NewClient(mock, io.Discard).put(context.Background(), S3Config{Bucket: "b", Region: "r"}, "/w/x", "s3://b/k/1.pgdump"); err != nil { + t.Fatal(err) + } + if _, _, joined := publishCalls(mock); !strings.Contains(joined, putCopyMatch+" --copy-props metadata-directive") { + t.Fatalf("calls: %s", joined) + } +} + +// aws CLI v1 has no --copy-props, and its multipart copy drops metadata, so a final +// object with no recorded checksum is replaced: fetch checks against that record. +func TestPublishOnAWSCLIv1(t *testing.T) { + ctx := context.Background() + mock := putMock(ssh.MockCommand{Match: putCopyMatch + " --copy-props", Err: errors.New("exit status 2: Unknown options: --copy-props,metadata-directive")}) + if _, err := NewClient(mock, io.Discard).put(ctx, S3Config{Bucket: "b", Region: "r"}, "/w/x", "s3://b/k/1.pgdump"); err != nil { + t.Fatal(err) + } + if copies, reuploads, joined := publishCalls(mock); copies != 2 || reuploads != 0 { + t.Fatalf("copies %d reuploads %d: %s", copies, reuploads, joined) + } + mock = putMock( + ssh.MockCommand{Match: "aws s3api head-object --bucket 'b' --key 'k/1.pgdump'", Output: `{"ContentLength":4096}`}, + ) + if _, err := NewClient(mock, io.Discard).put(ctx, S3Config{Bucket: "b", Region: "r"}, "/w/x", "s3://b/k/1.pgdump"); err != nil { + t.Fatal(err) + } + if _, reuploads, joined := publishCalls(mock); reuploads != 1 { + t.Fatalf("a final object that lost its checksum was kept: %s", joined) + } +} + +// One tree under the base: a lifecycle rule can only name a prefix. +func TestPendingUploadsLiveInOneTree(t *testing.T) { + for _, tc := range []struct{ prefix, key, want string }{ + {"", "s3://b/a/accessories/db/1.pgdump", "s3://b/.incoming/a/accessories/db/1.pgdump"}, + {"srv1", "s3://b/srv1/a/accessories/db/databases/x/1.pgdump", "s3://b/srv1/.incoming/a/accessories/db/databases/x/1.pgdump"}, + {"srv1/eu", "s3://b/srv1/eu/a/volumes/1.tar.gz", "s3://b/srv1/eu/.incoming/a/volumes/1.tar.gz"}, + {"srv1", "s3://b/elsewhere/1.pgdump", "s3://b/.incoming/elsewhere/1.pgdump"}, + } { + if got := incomingKey(S3Config{Bucket: "b", Prefix: tc.prefix}, tc.key); got != tc.want { + t.Fatalf("%s: %s", tc.key, got) + } + } + // And nothing in it reads as a backup. + if _, ok := classifyListedKey(".incoming", ".incoming/a/volumes/20260101-000000.tar.gz"); ok { + t.Fatal("a pending upload was listed") + } +} + +// A marker on a versioned store costs a stored copy per write. +func TestPruneDoesNotRewriteItsMarker(t *testing.T) { + r := "a/volumes/" + marker := ssh.MockCommand{Match: "aws s3 ls 's3://b/a/.owners/'", Output: "d t 20 a/.owners/0123456789abcdef\n"} + body := ssh.MockCommand{Match: "aws s3 cp 's3://b/a/.owners/0123456789abcdef' -", Output: `{"recipient":"` + testServerRecipient + `","first_seen":"2026-01-01T00:00:00Z"}`} + recipient := ssh.MockCommand{Match: "for p in /etc/teploy/age-key", Output: testServerRecipient + "\n"} + mock := ssh.NewMockExecutor("h", append([]ssh.MockCommand{marker, body, recipient, {Match: "which aws", Output: "/usr/bin/aws\n"}, + {Match: "aws s3 ls", Output: "d t 1000 " + r + "20240103-000000.tar.gz\nd t 1000 " + r + "20240102-000000.tar.gz\n"}, {Match: "aws s3 rm", Output: ""}}, storeStubs...)...) + if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "volumes", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1}); err != nil { + t.Fatal(err) + } + clock := false + for _, c := range mock.Calls { + if strings.HasPrefix(c, "printf") && strings.Contains(c, ".owners/") { + t.Fatalf("the marker was rewritten: %s", c) + } + clock = clock || (strings.HasPrefix(c, "printf") && strings.Contains(c, "s3://b/.incoming/.clock-")) + } + if !clock { + t.Fatalf("the clock was not read under .incoming/: %v", mock.Calls) + } +} + +func TestLifecycleCovers(t *testing.T) { + s3 := S3Config{Bucket: "b", Prefix: "srv1"} + for _, tc := range []struct{ name, reply, want string }{ + {"exact", `{"Rules":[{"Status":"Enabled","Filter":{"Prefix":"srv1/.incoming/"},"Expiration":{"Days":1}}]}`, LifecycleCovered}, + {"legacy prefix field", `{"Rules":[{"Status":"Enabled","Prefix":"srv1/.incoming/","NoncurrentVersionExpiration":{"NoncurrentDays":1}}]}`, LifecycleCovered}, + {"disabled", `{"Rules":[{"Status":"Disabled","Filter":{"Prefix":"srv1/.incoming/"},"Expiration":{"Days":1}}]}`, LifecycleAbsent}, + {"another server's", `{"Rules":[{"Status":"Enabled","Filter":{"Prefix":"srv2/.incoming/"},"Expiration":{"Days":1}}]}`, LifecycleAbsent}, + {"expires nothing", `{"Rules":[{"Status":"Enabled","Filter":{"Prefix":"srv1/.incoming/"}}]}`, LifecycleAbsent}, + {"the whole bucket is not this rule", `{"Rules":[{"Status":"Enabled","Filter":{},"Expiration":{"Days":1}}]}`, LifecycleAbsent}, + {"no rules", `{"Rules":[]}`, LifecycleAbsent}, + {"garbage", `nope`, LifecycleUnknown}, + } { + if got := lifecycleCovers([]byte(tc.reply), s3); got != tc.want { + t.Fatalf("%s: %s", tc.name, got) + } + } + ctx := context.Background() + c := NewClient(ssh.NewMockExecutor("h", ssh.MockCommand{Match: "aws s3api get-bucket-lifecycle-configuration", Err: errors.New("exit status 254: An error occurred (NoSuchLifecycleConfiguration)")}), io.Discard) + if got := c.incomingLifecycle(ctx, s3); got != LifecycleAbsent { + t.Fatal(got) + } + c = NewClient(ssh.NewMockExecutor("h", ssh.MockCommand{Match: "aws s3api get-bucket-lifecycle-configuration", Err: errors.New("exit status 254: AccessDenied")}), io.Discard) + if got := c.incomingLifecycle(ctx, s3); got != LifecycleUnknown { + t.Fatal(got) + } +} + +// ---- the policy file: one location, whoever asks ---- + +// policySandbox runs the policy scripts for real with /etc/teploy moved to a temp +// directory and a sudo stand-in that opens it for the command's length. +type policySandbox struct { + t *testing.T + root, etc string + home, bin string +} + +func newPolicySandbox(t *testing.T, sudoWorks bool) *policySandbox { + t.Helper() + if os.Geteuid() == 0 { + t.Skip("modes do not stop root") + } + root := t.TempDir() + s := &policySandbox{t: t, root: root, etc: filepath.Join(root, "etc-teploy"), home: filepath.Join(root, "home"), bin: filepath.Join(root, "bin")} + for _, d := range []string{s.home, s.bin} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + sudo := "#!/bin/sh\nexit 1\n" + if sudoWorks { + sudo = fmt.Sprintf("#!/bin/sh\nshift\nm=$(stat -f %%Lp %[1]q 2>/dev/null || stat -c %%a %[1]q 2>/dev/null)\nchmod 700 %[1]q\n\"$@\"\nrc=$?\nchmod \"$m\" %[1]q\nexit $rc\n", s.etc) + } + if err := os.WriteFile(filepath.Join(s.bin, "sudo"), []byte(sudo), 0o755); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Chmod(s.etc, 0o700) }) + return s +} + +func (s *policySandbox) run(script, stdin string) (string, string, int) { + s.t.Helper() + script = strings.ReplaceAll(script, "/etc/teploy", s.etc) + cmd := exec.Command("sh", "-c", script) + cmd.Env = []string{"HOME=" + s.home, "PATH=" + s.bin + ":/usr/bin:/bin"} + cmd.Stdin = strings.NewReader(stdin) + var out, errb strings.Builder + cmd.Stdout, cmd.Stderr = &out, &errb + err := cmd.Run() + code := 0 + if err != nil { + var ee *exec.ExitError + if !errors.As(err, &ee) { + s.t.Fatal(err) + } + code = ee.ExitCode() + } + return out.String(), errb.String(), code +} + +// Rotation through passwordless sudo: what set stored is what show and the run read, +// and unset removes it. It used to land in ~/.config while /etc kept winning. +func TestPolicyWriteUsesTheSudoPathReadsUse(t *testing.T) { + s := newPolicySandbox(t, true) + if err := os.Mkdir(s.etc, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(s.etc, "backup-policy.json"), []byte(`{"v":"1","recipient":"OLD"}`), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chmod(s.etc, 0); err != nil { + t.Fatal(err) + } + out, stderr, code := s.run(writeFleetPolicyScript, `{"v":"1","recipient":"NEW"}`) + if code != 0 || strings.Contains(stderr, "fallback") { + t.Fatalf("write: %d %s %s", code, out, stderr) + } + if want := "written " + s.etc + "/backup-policy.json"; strings.TrimSpace(out) != want { + t.Fatalf("%q, want %q", out, want) + } + out, _, code = s.run(readPolicyScript, "") + if code != 0 || !strings.Contains(out, "NEW") || strings.Contains(out, "OLD") { + t.Fatalf("read: %d %s", code, out) + } + if _, err := os.Stat(filepath.Join(s.home, ".config/teploy/backup-policy.json")); err == nil { + t.Fatal("a second policy was written under ~/.config") + } + out, stderr, code = s.run(unsetFleetPolicyScript, "") + if code != 0 || !strings.Contains(out, "removed "+s.etc+"/backup-policy.json") { + t.Fatalf("unset: %d %s %s", code, out, stderr) + } + out, _, code = s.run(readPolicyScript, "") + if code != 0 || strings.TrimSpace(out) != "absent" { + t.Fatalf("after unset: %d %s", code, out) + } +} + +// A policy this user can see and not replace is a refusal, not a copy it never reads. +func TestPolicyWriteRefusesWhatItCannotReplace(t *testing.T) { + s := newPolicySandbox(t, false) + if err := os.Mkdir(s.etc, 0o700); err != nil { + t.Fatal(err) + } + f := filepath.Join(s.etc, "backup-policy.json") + if err := os.WriteFile(f, []byte(`{"v":"1","recipient":"OLD"}`), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chmod(s.etc, 0o555); err != nil { + t.Fatal(err) + } + if _, stderr, code := s.run(writeFleetPolicyScript, `{"v":"1"}`); code != 3 || !strings.Contains(stderr, "cannot replace") { + t.Fatalf("write: %d %s", code, stderr) + } + if _, err := os.Stat(filepath.Join(s.home, ".config/teploy/backup-policy.json")); err == nil { + t.Fatal("a policy nothing reads was written") + } + if out, stderr, code := s.run(unsetFleetPolicyScript, ""); code != 3 || strings.Contains(out, "removed") { + t.Fatalf("unset said it worked: %d %s %s", code, out, stderr) + } + if _, err := os.Stat(f); err != nil { + t.Fatal(err) + } + // Hidden, no sudo, no policy of its own: unset cannot say there is none. + if err := os.Chmod(s.etc, 0); err != nil { + t.Fatal(err) + } + if out, stderr, code := s.run(unsetFleetPolicyScript, ""); code != 3 || strings.Contains(out, "removed") { + t.Fatalf("hidden unset: %d %s %s", code, out, stderr) + } +} + +// With nothing in /etc to shadow it, a user with no way in keeps its own, and reads it. +func TestPolicyFallbackIsReadBack(t *testing.T) { + s := newPolicySandbox(t, false) + if err := os.Mkdir(s.etc, 0); err != nil { + t.Fatal(err) + } + out, stderr, code := s.run(writeFleetPolicyScript, `{"v":"1","recipient":"MINE"}`) + if code != 0 || !strings.Contains(stderr, "fallback") { + t.Fatalf("write: %d %s %s", code, out, stderr) + } + out, _, code = s.run(readPolicyScript, "") + if code != 0 || !strings.Contains(out, "MINE") || !strings.Contains(out, s.home) { + t.Fatalf("read: %d %s", code, out) + } + // XDG_CONFIG_HOME no longer sends the write somewhere reads do not look. + if strings.Contains(writeFleetPolicyScript, "XDG_CONFIG_HOME") { + t.Fatal("write and read disagree on where ~/.config is") + } + out, stderr, code = s.run(unsetFleetPolicyScript, "") + if code != 0 || !strings.Contains(out, "removed "+s.home) { + t.Fatalf("unset: %d %s %s", code, out, stderr) + } +} + +// Stored and not in force is an error, whatever the scripts think they did. +func TestWriteFleetPolicyReadsItselfBack(t *testing.T) { + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "e=/etc/teploy/backup-policy.json", Output: "written /home/u/.config/teploy/backup-policy.json\n"}, + ssh.MockCommand{Match: policyReadMarker, Output: "path /etc/teploy/backup-policy.json\n" + `{"v":"1"}`}, + ) + if _, _, err := WriteFleetPolicy(context.Background(), mock, FleetPolicy{}); err == nil || !strings.Contains(err.Error(), "not in force") { + t.Fatalf("err = %v", err) + } +} + +// Cron runs as the deploy user, and the policy it read through sudo sits in a +// directory it cannot open. +func TestRunLockPathWorksForAPolicyBehindSudo(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("modes do not stop root") + } + home := t.TempDir() + t.Setenv("HOME", home) + dir := filepath.Join(t.TempDir(), "etc") + if err := os.Mkdir(dir, 0); err != nil { + t.Fatal(err) + } + defer os.Chmod(dir, 0o700) + p := RunLockPath(filepath.Join(dir, "backup-policy.json")) + if want := filepath.Join(home, ".config/teploy/backup-run.lock"); p != want { + t.Fatalf("%s, want %s", p, want) + } + l, _, err := AcquireRunLock(p, time.Now(), time.Hour) + if err != nil { + t.Fatal(err) + } + l.Release() + // A writable directory keeps the lock beside the policy. + own := t.TempDir() + if p := RunLockPath(filepath.Join(own, "backup-policy.json")); p != filepath.Join(own, "backup-run.lock") { + t.Fatal(p) + } +} + +func TestPolicyRemedyIsNotChmod(t *testing.T) { + if strings.Contains(readPolicyScript, "711") { + t.Fatal("chmod 711 leaves a 0600 file unreadable: it is no remedy") + } + // A 711 directory and a sudoer: the file is reachable and unreadable, so it is + // read through sudo like a hidden one. + s := newPolicySandbox(t, true) + if err := os.Mkdir(s.etc, 0o700); err != nil { + t.Fatal(err) + } + f := filepath.Join(s.etc, "backup-policy.json") + if err := os.WriteFile(f, []byte(`{"v":"1","recipient":"X"}`), 0o200); err != nil { + t.Fatal(err) + } + // The stand-in runs as this user, so it opens the file too. + sudo := "#!/bin/sh\nshift\nchmod 600 " + f + "\n\"$@\"\nrc=$?\nchmod 200 " + f + "\nexit $rc\n" + if err := os.WriteFile(filepath.Join(s.bin, "sudo"), []byte(sudo), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Chmod(s.etc, 0o711); err != nil { + t.Fatal(err) + } + if out, stderr, code := s.run(readPolicyScript, ""); code != 0 || !strings.Contains(out, `"recipient":"X"`) { + t.Fatalf("%d %s %s", code, out, stderr) + } +} + +// ---- owner markers ---- + +func TestClearingARecentMarkerNeedsForce(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 11, 5, 12, 0, 0, 0, time.Local) + stamp := func(t time.Time) string { return t.Format("2006-01-02 15:04:05") } + mk := func(newest time.Time) *ssh.MockExecutor { + return newStoreMock("h", + ssh.MockCommand{Match: "aws s3 ls 's3://b/app/'", Output: stamp(newest) + " 1000 app/volumes/20260101-000000.tar.gz\n" + stamp(now.AddDate(0, -3, 0)) + " 20 app/.owners/aaaaaaaaaaaaaaaa\n"}, + ssh.MockCommand{Match: "aws s3 ls", Output: stamp(now.AddDate(0, -3, 0)) + " 20 app/.owners/aaaaaaaaaaaaaaaa\n"}, + ssh.MockCommand{Match: "aws s3 rm", Output: ""}, + ) + } + s3 := S3Config{Bucket: "b", Region: "r"} + mock := mk(now.Add(-time.Hour)) + removed, err := NewClient(mock, io.Discard).ClearNamespaceMarkers(ctx, s3, "aaaaaaaaaaaaaaaa", false, now) + if !errors.Is(err, ErrMarkerRecent) || len(removed) != 0 || len(rmKeys(mock)) != 0 { + t.Fatalf("%v %v %v", err, removed, rmKeys(mock)) + } + mock = mk(now.Add(-time.Hour)) + if removed, err = NewClient(mock, io.Discard).ClearNamespaceMarkers(ctx, s3, "aaaaaaaaaaaaaaaa", true, now); err != nil || len(removed) != 1 { + t.Fatalf("--force: %v %v", err, removed) + } + mock = mk(now.AddDate(0, 0, -30)) + if removed, err = NewClient(mock, io.Discard).ClearNamespaceMarkers(ctx, s3, "aaaaaaaaaaaaaaaa", false, now); err != nil || len(removed) != 1 { + t.Fatalf("a dead server: %v %v", err, removed) + } + // Cannot tell: not a yes. + mock = newStoreMock("h", + ssh.MockCommand{Match: "aws s3 ls 's3://b/app/'", Err: errors.New("exit status 255: Forbidden")}, + ssh.MockCommand{Match: "aws s3 ls", Output: stamp(now.AddDate(0, -3, 0)) + " 20 app/.owners/aaaaaaaaaaaaaaaa\n"}, + ssh.MockCommand{Match: "aws s3 rm", Output: ""}, + ) + if _, err = NewClient(mock, io.Discard).ClearNamespaceMarkers(ctx, s3, "aaaaaaaaaaaaaaaa", false, now); !errors.Is(err, ErrMarkerRecent) || len(rmKeys(mock)) != 0 { + t.Fatalf("%v %v", err, rmKeys(mock)) + } +} + +// ---- the prune's brakes ---- + +func pruneListing(n int, keep string) string { + var lines []string + base := time.Now().UTC().AddDate(0, 0, -1) + for i := 0; i < n; i++ { + lines = append(lines, "d t 1000 "+keep+base.AddDate(0, 0, -i).Format("20060102-150405")+".tar.gz") + } + return strings.Join(lines, "\n") +} + +// A store on the same host shares a wrong clock, so the skew check passes and 19 of +// 20 are selected. Tightening retention on purpose takes one explicit run. +func TestPruneBrake(t *testing.T) { + ctx := context.Background() + s3 := S3Config{Bucket: "b", Region: "r"} + mock := treeMock(pruneListing(20, "a/volumes/")) + _, err := NewClient(mock, io.Discard).PruneBackups(ctx, "a", "volumes", s3, RetentionPolicy{KeepLast: 1}) + if !errors.Is(err, ErrPruneBrake) || !strings.Contains(err.Error(), "19 of the 20") || len(rmKeys(mock)) != 0 { + t.Fatalf("%v, removed %v", err, rmKeys(mock)) + } + mock = treeMock(pruneListing(20, "a/volumes/")) + if del, err := NewClient(mock, io.Discard).PruneBackups(ctx, "a", "volumes", s3, RetentionPolicy{KeepLast: 1, AllowMassDelete: true}); err != nil || len(del) != 19 { + t.Fatalf("%v %v", err, del) + } + // One database's series brakes too. + mock = treeMock(pruneListing(1, "a/accessories/db/") + "\n" + pruneListing(10, "a/accessories/db/databases/x/")) + if _, err := NewClient(mock, io.Discard).PruneBackups(ctx, "a", "accessories/db", s3, RetentionPolicy{KeepLast: 2}); !errors.Is(err, ErrPruneBrake) || !strings.Contains(err.Error(), "database x") { + t.Fatalf("%v", err) + } + // Ordinary retention does not: one over the line, and small series. + for _, tc := range []struct{ n, keep int }{{8, 7}, {3, 1}, {2, 1}, {6, 3}} { + mock = treeMock(pruneListing(tc.n, "a/volumes/")) + if del, err := NewClient(mock, io.Discard).PruneBackups(ctx, "a", "volumes", s3, RetentionPolicy{KeepLast: tc.keep}); err != nil || len(del) != tc.n-tc.keep { + t.Fatalf("%+v: %v %v", tc, err, del) + } + } + // Empty-output failures going is not most of anything. + listing := pruneListing(1, "a/accessories/db/") + for i := 0; i < 6; i++ { + listing += fmt.Sprintf("\nd t 10 a/accessories/db/2024010%d-000000.pgdump", i+1) + } + mock = treeMock(strings.ReplaceAll(listing, ".tar.gz", ".pgdump")) + if _, err := NewClient(mock, io.Discard).PruneBackups(ctx, "a", "accessories/db", s3, RetentionPolicy{KeepLast: 1}); errors.Is(err, ErrPruneBrake) { + t.Fatal(err) + } +} + +// A clock that cannot be read is a clock that cannot be trusted. +func TestPruneRefusesAnUnreadableStoreClock(t *testing.T) { + ctx := context.Background() + s3 := S3Config{Bucket: "b", Region: "r"} + r := "a/volumes/" + mk := func(head ssh.MockCommand) *ssh.MockExecutor { + return ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "{ cat /etc/machine-id", Output: "0123456789abcdef0123456789abcdef -\n"}, + ssh.MockCommand{Match: "printf", Output: ""}, + ssh.MockCommand{Match: "for p in /etc/teploy/age-key", Output: ""}, + head, + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "aws s3 ls", Output: "d t 1000 " + r + "20240103-000000.tar.gz\nd t 1000 " + r + "20240102-000000.tar.gz\n"}, + ssh.MockCommand{Match: "aws s3 rm", Output: ""}, + ) + } + for _, head := range []ssh.MockCommand{ + {Match: "aws s3api head-object", Err: errors.New("exit status 254: Forbidden")}, + {Match: "aws s3api head-object", Output: `{"ContentLength":6}`}, + } { + mock := mk(head) + _, err := NewClient(mock, io.Discard).PruneBackups(ctx, "a", "volumes", s3, RetentionPolicy{KeepLast: 1}) + if !errors.Is(err, ErrClockSkew) || !strings.Contains(err.Error(), "could not read") || len(rmKeys(mock)) != 0 { + t.Fatalf("%v %v", err, rmKeys(mock)) + } + mock = mk(head) + if del, err := NewClient(mock, io.Discard).PruneBackups(ctx, "a", "volumes", s3, RetentionPolicy{KeepLast: 1, IgnoreClock: true}); err != nil || len(del) != 1 { + t.Fatalf("--ignore-clock: %v %v", err, del) + } + } +} + +// A future-dated object does not stop retention, is never deleted, and does not take +// the real newest backup's place: keep_last 1 with --ignore-clock deleted that. +func TestFutureDatedObjectsAreOutOfSelection(t *testing.T) { + ctx := context.Background() + s3 := S3Config{Bucket: "b", Region: "r"} + r := "a/volumes/" + future := time.Now().UTC().AddDate(1, 0, 0).Format("20060102-150405") + today := time.Now().UTC().Format("20060102-150405") + yesterday := time.Now().UTC().AddDate(0, 0, -1).Format("20060102-150405") + listing := "d t 1000 " + r + future + ".tar.gz\nd t 1000 " + r + today + ".tar.gz\nd t 1000 " + r + yesterday + ".tar.gz\n" + for _, ignore := range []bool{false, true} { + mock := treeMock(listing) + del, err := NewClient(mock, io.Discard).PruneBackups(ctx, "a", "volumes", s3, RetentionPolicy{KeepLast: 1, IgnoreClock: ignore}) + if err == nil || !strings.Contains(err.Error(), "dated in the future") { + t.Fatalf("not reported: %v", err) + } + got := strings.Join(rmKeys(mock), ",") + if len(del) != 1 || !strings.Contains(got, yesterday) || strings.Contains(got, today) || strings.Contains(got, future) { + t.Fatalf("ignore=%v deleted %v: %s", ignore, del, got) + } + } +} + +// ---- deadlines and leftovers ---- + +func TestDeadlinesOverTheRunsAreRefused(t *testing.T) { + if err := (FleetPolicy{TargetDeadline: "24h"}).Validate(); err == nil || !strings.Contains(err.Error(), "capped") { + t.Fatalf("%v", err) + } + if err := (FleetPolicy{TargetDeadline: "23h"}).Validate(); err != nil { + t.Fatal(err) + } + if err := ValidateDeadline(24 * time.Hour); err == nil { + t.Fatal("24h") + } + if err := ValidateDeadline(0); err != nil { + t.Fatal(err) + } +} + +// The janitor goes by the workspace's own expiry, so a 20h manual run is safe from a +// run whose own deadline is 6h, in the shell that actually decides. +func TestJanitorRespectsAWorkspacesExpiry(t *testing.T) { + base := t.TempDir() + old := time.Now().Add(-10 * time.Hour) + mk := func(name, expires string) string { + d := filepath.Join(base, name) + if err := os.Mkdir(d, 0o700); err != nil { + t.Fatal(err) + } + if expires != "" { + if err := os.WriteFile(filepath.Join(d, workspaceExpiry), []byte(expires+"\n"), 0o600); err != nil { + t.Fatal(err) + } + } + if err := os.Chtimes(d, old, old); err != nil { + t.Fatal(err) + } + return d + } + live := mk("teploy-backup.live", fmt.Sprint(time.Now().Add(10*time.Hour).Unix())) + expired := mk("teploy-backup.expired", fmt.Sprint(time.Now().Add(-time.Minute).Unix())) + unmarked := mk("teploy-backup.unmarked", "") + fresh := filepath.Join(base, "teploy-backup.fresh") + if err := os.Mkdir(fresh, 0o700); err != nil { + t.Fatal(err) + } + cmd := exec.Command("sh", "-c", cleanWorkspacesScript(ssh.ShellQuote(base)+"/teploy-backup.*")) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%v: %s", err, out) + } + exists := func(d string) bool { _, err := os.Stat(d); return err == nil } + if !exists(live) || !exists(fresh) || exists(expired) || exists(unmarked) { + t.Fatalf("live %v fresh %v expired %v unmarked %v: %s", exists(live), exists(fresh), exists(expired), exists(unmarked), out) + } +} + +func TestWorkspaceCarriesItsDeadline(t *testing.T) { + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "umask 077; mktemp -d", Output: "/var/tmp/teploy-backup.t\n"}, ssh.MockCommand{Match: "printf", Output: ""}) + c := NewClient(mock, io.Discard) + c.SetDeadline(20 * time.Hour) + if _, _, err := c.newBackupWorkspace(context.Background()); err != nil { + t.Fatal(err) + } + want := fmt.Sprintf("+ %d ))\" > '/var/tmp/teploy-backup.t/%s'", 21*3600, workspaceExpiry) + if !strings.Contains(strings.Join(mock.Calls, "\n"), want) { + t.Fatalf("%v, want %s", mock.Calls, want) + } +} + +// Thirty deploys in a week must not hold thirty full copies past retention. +func TestSafetyCopiesAreCapped(t *testing.T) { + r := "a/accessories/db/" + now := time.Now().UTC() + var lines []string + ids := make([]string, 0, safetyKeep+3) + for i := 0; i < safetyKeep+3; i++ { + id := now.Add(-time.Duration(i+1) * time.Hour).Format("20060102-150405") + ids = append(ids, id) + lines = append(lines, "d t 1000 "+r+id+".pgdump", "d t 8 "+r+"safety/"+id+".safety") + } + mock := treeMock(strings.Join(lines, "\n")) + if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1, AllowMassDelete: true}); err != nil { + t.Fatal(err) + } + got := strings.Join(rmKeys(mock), ",") + for i, id := range ids { + pruned := strings.Contains(got, id+".pgdump") + if want := i >= safetyKeep; pruned != want { + t.Fatalf("copy %d (%s): pruned %v: %s", i, id, pruned, got) + } + if want := i >= safetyKeep; strings.Contains(got, "safety/"+id) != want { + t.Fatalf("marker %d: %s", i, got) + } + } +} + +// ---- previous prefixes ---- + +func TestAnUnreachablePreviousPrefixOnlyWarns(t *testing.T) { + ctx := context.Background() + s3 := S3Config{Bucket: "b", Region: "r", Prefix: "now", PreviousPrefixes: []PreviousPrefix{{Prefix: "gone"}}} + mk := func(cur ssh.MockCommand) *ssh.MockExecutor { + return newStoreMock("h", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + cur, + ssh.MockCommand{Match: "aws s3 ls 's3://b/gone/a/'", Err: errors.New("exit status 255: AccessDenied")}, + ) + } + var out strings.Builder + rows, err := NewClient(mk(ssh.MockCommand{Match: "aws s3 ls 's3://b/now/a/'", Output: "2026-01-01 00:00:00 1000 now/a/volumes/20260102-000000.tar.gz\n"}), &out).ListBackupsRecursive(ctx, "a", s3) + if err != nil || len(rows) != 1 { + t.Fatalf("%v %v", err, rows) + } + if !strings.Contains(out.String(), "previous prefix gone") { + t.Fatalf("no warning: %s", out.String()) + } + if _, err := NewClient(mk(ssh.MockCommand{Match: "aws s3 ls 's3://b/now/a/'", Err: errors.New("exit status 255: AccessDenied")}), io.Discard).ListBackupsRecursive(ctx, "a", s3); err == nil { + t.Fatal("the current location failing must fail") + } +} + +// ---- smaller ones ---- + +func TestChecksumHeadRefusalFallsBackToSize(t *testing.T) { + ctx := context.Background() + big := func(head ssh.MockCommand) *Client { + mock := putMock(head) + c := NewClient(mock, &strings.Builder{}) + c.readBack = 1 + return c + } + match := "aws s3api head-object --bucket 'b' --key '.incoming/k/1.pgdump' --checksum-mode ENABLED" + var out strings.Builder + c := big(ssh.MockCommand{Match: match, Err: errors.New("exit status 2: Unknown options: --checksum-mode")}) + c.out = &out + if _, err := c.put(ctx, S3Config{Bucket: "b", Region: "r"}, "/w/x", "s3://b/k/1.pgdump"); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "verified by size alone") { + t.Fatalf("out: %s", out.String()) + } + // Anything else is still no proof. + c = big(ssh.MockCommand{Match: match, Err: errors.New("exit status 254: Forbidden")}) + if _, err := c.put(ctx, S3Config{Bucket: "b", Region: "r"}, "/w/x", "s3://b/k/1.pgdump"); err == nil || !strings.Contains(err.Error(), "verifying the upload") { + t.Fatalf("err = %v", err) + } +} + +func TestExcludeNameMatchesWhatExcludeTakes(t *testing.T) { + for _, tc := range []struct { + t FleetTarget + want string + }{ + {FleetTarget{Kind: TargetAccessory, App: "a", Name: "db", Container: "a-db"}, "a/db"}, + {FleetTarget{Kind: TargetVolumes, App: "a"}, "a/volumes"}, + {FleetTarget{Kind: TargetAdopted, App: AdoptedApp, Name: "m", Container: "m"}, "m"}, + } { + got := excludeName(tc.t) + if got != tc.want { + t.Fatalf("%+v: %s", tc.t, got) + } + if err := validateExclude(got); err != nil { + t.Fatal(err) + } + p := FleetPolicy{Excludes: []string{got}} + if !p.Excluded(tc.t.App, map[string]string{TargetVolumes: "volumes"}[tc.t.Kind]+tc.t.Name, tc.t.Container) { + t.Fatalf("%s excludes nothing", got) + } + } +} + +func TestAWSCLIMinimumVersion(t *testing.T) { + for out, bad := range map[string]bool{ + "aws-cli/2.31.8 Python/3.13.7 Linux/6.8 exe/x86_64.ubuntu.24": false, + "aws-cli/2.23.0 Python/3.12 Linux": false, + "aws-cli/2.22.35 Python/3.12 Linux": true, + "aws-cli/2.9.19 Python/3.11 Linux": true, + "aws-cli/1.37.0 Python/3.12 Linux botocore/1.36.0": false, + "aws-cli/1.22.34 Python/3.10 Linux botocore/1.23.34": true, + "aws-cli/3.0.0 Python/3.14 Linux": false, + "something else entirely": false, + "": false, + } { + if got := AWSCLIVersionProblem(out) != ""; got != bad { + t.Fatalf("%q: %v", out, got) + } + } + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "which aws", Output: "aws-cli/1.22.34 Python/3.10\n"}) + err := NewClient(mock, io.Discard).ensureAWSCLI(context.Background()) + if err == nil || !strings.Contains(err.Error(), "older than") || !strings.Contains(err.Error(), "teploy setup") { + t.Fatalf("err = %v", err) + } +} + +// The install script's shell check is the Go one: an old distribution package is not +// "installed". +func TestInstallScriptShellCheckAgrees(t *testing.T) { + if !strings.Contains(AWSCLIInstallScript, fmt.Sprintf(`2) [ "$b" -ge %d ]`, awsCLIMinV2Minor)) || !strings.Contains(AWSCLIInstallScript, fmt.Sprintf(`1) [ "$b" -ge %d ]`, awsCLIMinV1Minor)) { + t.Fatal("the script's numbers drifted") + } + bin := t.TempDir() + for version, ok := range map[string]bool{"2.23.0": true, "2.31.8": true, "2.22.35": false, "1.37.0": true, "1.22.34": false, "3.1.0": true} { + script := "#!/bin/sh\necho aws-cli/" + version + " Python/3.12 Linux\n" + if err := os.WriteFile(filepath.Join(bin, "aws"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + cmd := exec.Command("sh", "-c", awsCLIShellOK+"\naws_ok") + cmd.Env = []string{"PATH=" + bin + ":/usr/bin:/bin"} + if got := cmd.Run() == nil; got != ok { + t.Fatalf("%s: %v", version, got) + } + if got := AWSCLIVersionProblem("aws-cli/"+version+" Python") == ""; got != ok { + t.Fatalf("go disagrees on %s", version) + } + } +} + +// The parent goes first, on a list taken before: a parent still alive would answer a +// dead child by starting its next step. +func TestStuckHolderIsKilledParentFirst(t *testing.T) { + var order []int + kill := func(pid int, sig syscall.Signal) error { + if sig != syscall.SIGKILL { + t.Fatalf("signal %v", sig) + } + order = append(order, pid) + return nil + } + if err := killHolder(100, []int{101, 102}, kill); err != nil { + t.Fatal(err) + } + if fmt.Sprint(order) != "[100 -101 -102]" { + t.Fatalf("%v", order) + } + // A parent that cannot be stopped keeps its children: nothing else was asked for. + order = nil + if err := killHolder(100, []int{101}, func(pid int, _ syscall.Signal) error { + order = append(order, pid) + return errors.New("EPERM") + }); err == nil || fmt.Sprint(order) != "[100]" { + t.Fatalf("%v %v", err, order) + } +} diff --git a/internal/backup/runlock_unix.go b/internal/backup/runlock_unix.go index 281bff3..07f4a80 100644 --- a/internal/backup/runlock_unix.go +++ b/internal/backup/runlock_unix.go @@ -52,14 +52,8 @@ func AcquireRunLock(path string, now time.Time, staleAfter time.Duration) (*RunL f.Close() return nil, holder, ErrRunLocked } - // Alive, stuck and past every deadline it had: it will never let go. Its - // dumps and uploads run in process groups of their own, which would - // outlive it into the next run. - children := childPIDs(holder.PID) - for _, child := range children { - _ = syscall.Kill(-child, syscall.SIGKILL) - } - if killErr := syscall.Kill(holder.PID, syscall.SIGKILL); killErr != nil { + // Alive, stuck and past every deadline it had: it will never let go. + if killErr := killHolder(holder.PID, childPIDs(holder.PID), syscall.Kill); killErr != nil { f.Close() return nil, holder, fmt.Errorf("%w (pid %d has held it since %s and could not be stopped: %v)", ErrRunLocked, holder.PID, holder.StartedAt.Format(time.RFC3339), killErr) } @@ -82,6 +76,20 @@ func AcquireRunLock(path string, now time.Time, staleAfter time.Duration) (*RunL return &RunLock{f: f}, RunLockHolder{}, nil } +// killHolder stops a stuck holder and the process groups of its dumps and uploads, +// which would outlive it into the next run. The parent goes first, on a list taken +// before: alive, it would see a child die and start the next step, and a child +// killed first leaves a parent that can no longer be asked anything about it. +func killHolder(pid int, children []int, kill func(int, syscall.Signal) error) error { + if err := kill(pid, syscall.SIGKILL); err != nil { + return err + } + for _, child := range children { + _ = kill(-child, syscall.SIGKILL) + } + return nil +} + // Release drops the lock (and exiting does too). func (l *RunLock) Release() { if l == nil || l.f == nil { diff --git a/internal/backup/status.go b/internal/backup/status.go index 4e3dbd1..bc34bae 100644 --- a/internal/backup/status.go +++ b/internal/backup/status.go @@ -2,6 +2,7 @@ package backup import ( "context" + "encoding/json" "fmt" "io" "slices" @@ -27,6 +28,13 @@ const ( CoverageAdopted = "adopted" ) +// What a target's lifecycle rules say about its .incoming/ tree. +const ( + LifecycleCovered = "covered" + LifecycleAbsent = "absent" + LifecycleUnknown = "unknown" +) + // StatusObject is the newest good object of a series. type StatusObject struct { ID string `json:"id"` @@ -103,6 +111,10 @@ type StatusTarget struct { // OtherServers are fingerprints of servers that also write at this target's // base. Only looked for with no prefix, where same-named apps would collide. OtherServers []string `json:"other_servers,omitempty"` + // IncomingLifecycle says whether a lifecycle rule expires /.incoming/, which + // on a store that keeps versions is what stops pending uploads billing forever: + // covered, absent, or unknown (the store would not say). + IncomingLifecycle string `json:"incoming_lifecycle,omitempty"` } // StatusPolicy summarizes the policy (no heartbeat tokens). @@ -144,7 +156,9 @@ type FleetStatus struct { // Encrypted says new objects will be encrypted (a recovery key is set). Encrypted bool `json:"encrypted"` AWSCLI bool `json:"aws_cli"` - AgeCLI bool `json:"age_cli"` + // AWSCLIProblem is set when the aws CLI runs but is too old to back up with. + AWSCLIProblem string `json:"aws_cli_problem,omitempty"` + AgeCLI bool `json:"age_cli"` // RunLock names a run in progress. LastRun *StatusLastRun `json:"last_run,omitempty"` Entries []StatusEntry `json:"entries"` @@ -186,9 +200,12 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle Configured: target.Bucket != "", Bucket: target.Bucket, Region: target.Region, Endpoint: target.Endpoint, Prefix: target.Prefix, HasCredentials: target.HasCredentials, PreviousPrefixes: target.PreviousPrefixes, } - if out, err := exec.Run(ctx, "command -v aws >/dev/null 2>&1 && aws --version >/dev/null 2>&1 && echo aws; command -v age >/dev/null 2>&1 && echo age; true"); err == nil { + if out, err := exec.Run(ctx, `command -v aws >/dev/null 2>&1 && v=$(aws --version 2>&1) && echo "aws $v"; command -v age >/dev/null 2>&1 && echo age; true`); err == nil { for _, t := range splitLines(out) { - st.AWSCLI = st.AWSCLI || t == "aws" + if t == "aws" || strings.HasPrefix(t, "aws ") { + st.AWSCLI = true + st.AWSCLIProblem = AWSCLIVersionProblem(t) + } st.AgeCLI = st.AgeCLI || t == "age" } } @@ -235,6 +252,24 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle st.Target.Reachable = true } } + if st.Target.Reachable { + st.Target.IncomingLifecycle = c.incomingLifecycle(ctx, cfg) + // One tree per base, reported whatever namespace it belongs to. + lctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + if keys, err := c.listTree(lctx, cfg, incomingDir+"/"); err == nil { + for _, k := range keys { + rel, ok := cfg.relToBase(k.Key) + if !ok || !strings.HasPrefix(rel, incomingDir+"/") { + continue + } + segs := strings.Split(k.Key, "/") + if e, ok := newBackupEntry(segs[len(segs)-1]); ok && now.Sub(e.time) > incomingStaleAfter { + st.LeftoverUploads = append(st.LeftoverUploads, k.Key) + } + } + } + cancel() + } if st.Target.Reachable && target.Prefix == "" && policy.Configured() && c.fingerprint(ctx) != "" { mctx, cancel := context.WithTimeout(ctx, 2*time.Minute) if markers, err := c.ListNamespaceMarkers(mctx, cfg); err == nil { @@ -255,16 +290,15 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle if st.Target.Reachable { lctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() - r, pending, err := c.listBackupsAndPending(lctx, app, cfg) + r, warnings, err := c.listBackups(lctx, app, cfg) if err != nil { st.Target.Reachable = false st.Target.Error = oneLine(err.Error()) } rows = r - for _, k := range pending { - segs := strings.Split(k, "/") - if e, ok := newBackupEntry(segs[len(segs)-1]); ok && now.Sub(e.time) > tmpLeftoverMinutes*time.Minute { - st.LeftoverUploads = append(st.LeftoverUploads, k) + for _, w := range warnings { + if !slices.Contains(st.Warnings, w) { + st.Warnings = append(st.Warnings, w) } } } @@ -448,6 +482,60 @@ func ReadFleetStatus(ctx context.Context, exec ssh.Executor, now time.Time) (Fle return st, nil } +// incomingLifecycle reads the bucket's lifecycle rules for one that expires +// /.incoming/. Read-only. +func (c *Client) incomingLifecycle(ctx context.Context, s3 S3Config) string { + lctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + res := ssh.RunDetailed(lctx, c.exec, s3.AWS("s3api get-bucket-lifecycle-configuration --bucket "+ssh.ShellQuote(s3.Bucket)+" --output json")) + if res.Failed() { + if strings.Contains(res.ExitErrorText(), "NoSuchLifecycleConfiguration") { + return LifecycleAbsent + } + return LifecycleUnknown + } + return lifecycleCovers(res.Stdout, s3) +} + +// lifecycleCovers reads a GetBucketLifecycleConfiguration reply. +func lifecycleCovers(reply []byte, s3 S3Config) string { + var cfg struct { + Rules []struct { + Status string `json:"Status"` + Prefix string `json:"Prefix"` + Filter struct { + Prefix string `json:"Prefix"` + } `json:"Filter"` + Expiration struct { + Days int `json:"Days"` + } `json:"Expiration"` + NoncurrentVersionExpiration struct { + NoncurrentDays int `json:"NoncurrentDays"` + } `json:"NoncurrentVersionExpiration"` + } `json:"Rules"` + } + if json.Unmarshal(reply, &cfg) != nil { + return LifecycleUnknown + } + want := incomingDir + "/" + if s3.Prefix != "" { + want = s3.Prefix + "/" + want + } + for _, r := range cfg.Rules { + prefix := r.Filter.Prefix + if prefix == "" { + prefix = r.Prefix + } + // A rule on the whole bucket would expire the backups too, and is someone + // else's decision: this one is about the pending tree. + if r.Status == "Enabled" && prefix != "" && strings.HasPrefix(want, prefix) && strings.Contains(prefix, incomingDir) && + (r.Expiration.Days > 0 || r.NoncurrentVersionExpiration.NoncurrentDays > 0) { + return LifecycleCovered + } + } + return LifecycleAbsent +} + // statusUncovered reads what a running database server holds beyond this // entry's coverage. Read-only (coverageWith's mongo cleanup is not reached). func (c *Client) statusUncovered(ctx context.Context, app, name string, f *ContainerFact, engine string, opts BackupOptions, st *FleetStatus) []string { diff --git a/internal/backup/store.go b/internal/backup/store.go index 4d69c61..a76ae48 100644 --- a/internal/backup/store.go +++ b/internal/backup/store.go @@ -29,12 +29,20 @@ import ( // server, and only the ciphertext is uploaded (.age suffix). A failure to // encrypt fails the backup; it never falls back to plaintext. // -// - A final key means a verified object: the upload goes to a .incoming/ key -// beside the final one, its size and checksum are checked there, and only +// - A final key means a verified object: the upload goes under one +// /.incoming/ tree, its size and checksum are checked there, and only // then is it copied server-side to its final name. A killed run therefore // can never leave a partial or unverified object where list, latest, -// restore or prune would take it for a backup. Leftover .incoming/ objects -// are invisible to those, and pruned by the next run. +// restore or prune would take it for a backup. One tree because a store that +// keeps versions (B2 only hides on delete) bills every byte ever written +// there until a lifecycle rule expires it, and a rule can only name a prefix: +// `.incoming/` with a 1-day expiry is what doctor looks for. +// +// - A store that refuses the server-side copy (B2 cannot change a checksum +// algorithm on one, R2 has no GetObjectTagging) still gets the object: the +// verified local file is uploaded straight to the final name and verified +// there. Atomic like any upload; a kill between that upload completing and +// its check is the one window a final key is unverified. // // - Commit order: a backup's sidecars and extra databases are stored BEFORE // its primary object, so a primary implies everything that belongs to it. @@ -180,10 +188,20 @@ func (c *Client) localFacts(ctx context.Context, local string) (int64, string, e return size, f[0], nil } -// incomingKey is the verification-pending name of a final key. -func incomingKey(key string) string { - dir, file := path.Split(key) - return dir + incomingDir + "/" + file +// incomingKey is the verification-pending name of a final key: the same path +// under the base's one .incoming/ tree. +func incomingKey(s3 S3Config, key string) string { + base := "s3://" + s3.Base() + "/" + if rel, ok := strings.CutPrefix(key, base); ok { + return base + incomingDir + "/" + rel + } + // Not under the base (no caller builds such a key): under the bucket's own + // tree rather than anywhere a listing would take it for a backup. + bucket, rel, err := splitS3URI(key) + if err != nil { + return key + "." + incomingDir + } + return "s3://" + bucket + "/" + incomingDir + "/" + rel } // objectHead is the stored object's size and recorded checksum. @@ -259,6 +277,13 @@ func checksumUnsupported(err error) bool { return false } +// copyRefused recognises a store that answered a server-side copy with a refusal: +// the checksum path above, or any error reply to the copy. A connection that +// failed says none of this, and a re-upload would fail the same way. +func copyRefused(err error) bool { + return checksumUnsupported(err) || strings.Contains(strings.ToLower(err.Error()), "an error occurred (") +} + // splitS3URI splits s3://bucket/key. func splitS3URI(uri string) (bucket, key string, err error) { rest, ok := strings.CutPrefix(uri, "s3://") @@ -300,88 +325,134 @@ func (c *Client) put(ctx context.Context, s3 S3Config, local, key string) (store if err != nil { return none, err } - pending := incomingKey(final) + pending := incomingKey(s3, final) q := ssh.ShellQuote - upload := fmt.Sprintf("s3 cp %s %s --metadata %s", q(src), q(pending), q("sha256="+sum)) large := size > c.readBackLimit() - providerChecked := large - var uerr error - if large { - _, uerr = c.exec.Run(ctx, s3.AWS(upload+" --checksum-algorithm SHA256")) - if uerr != nil && checksumUnsupported(uerr) { - providerChecked = false - fmt.Fprintf(c.out, "Warning: this store or aws CLI has no upload checksums, so %s is verified by size alone: %v\n", path.Base(final), oneLine(uerr.Error())) - _, uerr = c.exec.Run(ctx, s3.AWS(upload)) - } - } else { - _, uerr = c.exec.Run(ctx, s3.AWS(upload)) - } - if uerr != nil { - return none, fmt.Errorf("uploading: %w", uerr) - } - removePending := func() { - rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) - defer cancel() - c.exec.Run(rctx, s3.AWS("s3 rm "+q(pending))) + // upload reports whether the store checks a checksum on receipt. + upload := func(dst string) (bool, error) { + cmd := fmt.Sprintf("s3 cp %s %s --metadata %s", q(src), q(dst), q("sha256="+sum)) + if !large { + _, err := c.exec.Run(ctx, s3.AWS(cmd)) + return false, err + } + _, err := c.exec.Run(ctx, s3.AWS(cmd+" --checksum-algorithm SHA256")) + if err == nil || !checksumUnsupported(err) { + return err == nil, err + } + fmt.Fprintf(c.out, "Warning: this store or aws CLI has no upload checksums, so %s is verified by size alone: %v\n", path.Base(final), oneLine(err.Error())) + _, err = c.exec.Run(ctx, s3.AWS(cmd)) + return false, err } - check := func(k string) error { + check := func(k string) (objectHead, error) { h, err := c.head(ctx, s3, k) if err != nil { - return err + return h, err } if h.Size != size { - return fmt.Errorf("stored %s is %d bytes, %d were sent", k, h.Size, size) + return h, fmt.Errorf("stored %s is %d bytes, %d were sent", k, h.Size, size) } if h.SHA256 != "" && h.SHA256 != sum { - return fmt.Errorf("stored %s records checksum %s, %s was sent", k, h.SHA256, sum) + return h, fmt.Errorf("stored %s records checksum %s, %s was sent", k, h.SHA256, sum) } - return nil + return h, nil } - if err := check(pending); err != nil { - removePending() - return none, fmt.Errorf("verifying the upload: %w", err) - } - if providerChecked { - // The store checked every part on receipt; a single-part object's digest - // can be compared whole, and a store that kept none has proved nothing. - h, err := c.headArgs(ctx, s3, pending, " --checksum-mode ENABLED") + // verify proves an upload: size and recorded checksum, then the bytes read back + // or the store's own checksum. + verify := func(k string, providerChecked bool) (objectHead, error) { + h, err := check(k) if err != nil { - removePending() - return none, fmt.Errorf("verifying the upload: %w", err) - } - if h.ProviderSHA256 != "" && !strings.Contains(h.ProviderSHA256, "-") && h.ProviderSHA256 != sha256HexToBase64(sum) { - removePending() - return none, fmt.Errorf("the store's checksum of the upload does not match what was sent (checksum mismatch)") + return h, err } - if h.ProviderSHA256 == "" { - fmt.Fprintf(c.out, "Warning: the store keeps no checksum for %s; it is verified by size alone\n", path.Base(final)) + if providerChecked { + // The store checked every part on receipt; a single-part object's digest + // can be compared whole, and a store that kept none has proved nothing. + ph, err := c.headArgs(ctx, s3, k, " --checksum-mode ENABLED") + switch { + case err != nil && checksumUnsupported(err): + fmt.Fprintf(c.out, "Warning: this store or aws CLI cannot report checksums, so %s is verified by size alone: %v\n", path.Base(final), oneLine(err.Error())) + case err != nil: + return h, err + case ph.ProviderSHA256 == "": + fmt.Fprintf(c.out, "Warning: the store keeps no checksum for %s; it is verified by size alone\n", path.Base(final)) + case !strings.Contains(ph.ProviderSHA256, "-") && ph.ProviderSHA256 != sha256HexToBase64(sum): + return h, fmt.Errorf("the store's checksum of the upload does not match what was sent (checksum mismatch)") + } + } else if !large { + // A pipe, but compared by value: anything short of the exact bytes + // (a failed read included) produces a different hash. + out, err := c.exec.Run(ctx, "{ "+s3.AWS("s3 cp "+q(k)+" -")+"; } | sha256sum") + if err != nil { + return h, fmt.Errorf("reading the upload back: %w", err) + } + if f := strings.Fields(lastLine(out)); len(f) == 0 || f[0] != sum { + return h, fmt.Errorf("the uploaded object's bytes do not match what was sent (checksum mismatch)") + } } - } else if !large { - // A pipe, but compared by value: anything short of the exact bytes - // (a failed read included) produces a different hash. - out, err := c.exec.Run(ctx, "{ "+s3.AWS("s3 cp "+q(pending)+" -")+"; } | sha256sum") + return h, nil + } + remove := func(k string) { + rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) + defer cancel() + c.exec.Run(rctx, s3.AWS("s3 rm "+q(k))) + } + providerChecked, err := upload(pending) + if err != nil { + return none, fmt.Errorf("uploading: %w", err) + } + pendingHead, err := verify(pending, providerChecked) + if err != nil { + remove(pending) + return none, fmt.Errorf("verifying the upload: %w", err) + } + if err := c.publish(ctx, s3, pending, final, pendingHead.SHA256, check, func() error { + fmt.Fprintf(c.out, "A server-side copy of %s did not work on this store, so it is uploaded straight to its final name\n", path.Base(final)) + checked, err := upload(final) if err != nil { - removePending() - return none, fmt.Errorf("reading the upload back: %w", err) + remove(final) + return fmt.Errorf("uploading: %w", err) } - if f := strings.Fields(lastLine(out)); len(f) == 0 || f[0] != sum { - removePending() - return none, fmt.Errorf("the uploaded object's bytes do not match what was sent (checksum mismatch)") + if _, err := verify(final, checked); err != nil { + remove(final) + return fmt.Errorf("verifying the published object: %w", err) } + return nil + }, remove); err != nil { + remove(pending) + return none, err } - if _, err := c.exec.Run(ctx, s3.AWS("s3 cp "+q(pending)+" "+q(final))); err != nil { - removePending() - return none, fmt.Errorf("publishing the verified upload: %w", err) + remove(pending) + return storedObject{Key: final, Size: size, SHA256: sum, Encrypted: encrypted}, nil +} + +// publish copies a verified upload to its final name. The copy names +// --copy-props metadata-directive because a copy of 8 MB or more is multipart and +// its default also reads the source's tags, which R2 does not implement; metadata +// is what fetch checks the object against. aws CLI v1 has no such flag, and its +// multipart copy drops metadata: a final object that lost the checksum the pending +// one had is replaced too. reupload is the way out whenever the store refuses. +func (c *Client) publish(ctx context.Context, s3 S3Config, pending, final, recorded string, check func(string) (objectHead, error), reupload func() error, remove func(string)) error { + q := ssh.ShellQuote + copyCmd := "s3 cp " + q(pending) + " " + q(final) + _, err := c.exec.Run(ctx, s3.AWS(copyCmd+" --copy-props metadata-directive")) + if err != nil && strings.Contains(strings.ToLower(err.Error()), "unknown options") { + _, err = c.exec.Run(ctx, s3.AWS(copyCmd)) } - if err := check(final); err != nil { - rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) - defer cancel() - c.exec.Run(rctx, s3.AWS("s3 rm "+q(final))) - removePending() - return none, fmt.Errorf("verifying the published object: %w", err) + if err != nil { + if ctx.Err() != nil || !copyRefused(err) { + return fmt.Errorf("publishing the verified upload: %w", err) + } + fmt.Fprintf(c.out, "Warning: %s\n", oneLine(err.Error())) + return reupload() } - removePending() - return storedObject{Key: final, Size: size, SHA256: sum, Encrypted: encrypted}, nil + h, err := check(final) + if err != nil { + remove(final) + return fmt.Errorf("verifying the published object: %w", err) + } + if recorded != "" && h.SHA256 == "" { + return reupload() + } + return nil } func lastLine(s string) string { @@ -531,6 +602,9 @@ func seriesPrefix(s3 S3Config, app, name, db string) string { type storedKey struct { Key string // relative to the bucket root Size int64 + // Modified is the store's date of the last write, in this machine's zone as + // the aws CLI prints it: good for "this week", not for arithmetic in minutes. + Modified time.Time } // listTree lists every object under s3:/// recursively. A prefix with @@ -561,7 +635,11 @@ func (c *Client) listTree(ctx context.Context, s3 S3Config, rel string) ([]store if err != nil { continue } - out = append(out, storedKey{Key: f[3], Size: size}) + k := storedKey{Key: f[3], Size: size} + if t, err := time.ParseInLocation("2006-01-02 15:04:05", f[0]+" "+f[1], time.Local); err == nil { + k.Modified = t + } + out = append(out, k) } return out, nil } @@ -650,11 +728,41 @@ func artifactFloor(name string) int64 { // ---- janitor ---- -// tmpLeftoverAge is how old an unattended workspace or .incoming object must be -// before it is treated as a killed run's leftover: longer than the deadline of -// any one target, shorter than a disk can afford to wait. +// MaxDeadline is the longest deadline any one backup, restore or verify may be +// given: the whole fleet run's. Anything past it is refused, not silently capped. +const MaxDeadline = 23 * time.Hour + +// ValidateDeadline checks a deadline an operator named (0 is the default). +func ValidateDeadline(d time.Duration) error { + if d < 0 { + return fmt.Errorf("deadline must be positive") + } + if d > MaxDeadline { + return fmt.Errorf("deadline %s is over the %s a run may last", d, MaxDeadline) + } + return nil +} + +// tmpLeftoverMinutes is how old a temp directory with no expiry marker must be +// before it is treated as a killed run's leftover. A backup workspace carries its +// own (workspaceExpiry), so a run with a long --deadline is not cleaned from under +// itself by a concurrent run's janitor, whatever that run's deadline is. const tmpLeftoverMinutes = 420 +// incomingStaleAfter is how old a pending upload must be before it is a leftover: +// past the longest deadline any run may have, and what a store's 1-day lifecycle +// rule on .incoming/ would do anyway. +const incomingStaleAfter = MaxDeadline + time.Hour + +// workspaceExpiry names the file in a backup workspace saying, in epoch seconds, +// when the run that made it must be over. +const workspaceExpiry = ".expires" + +// leftoverTest is shell: true when $d is a leftover. +func leftoverTest() string { + return fmt.Sprintf(`if [ -f "$d/%s" ]; then [ "$(date +%%s)" -gt "$(cat "$d/%s" 2>/dev/null || echo 0)" ]; else [ -n "$(find "$d" -maxdepth 0 -mmin +%d 2>/dev/null)" ]; fi`, workspaceExpiry, workspaceExpiry, tmpLeftoverMinutes) +} + // tmpBases are the directories backup workspaces can be under: disk, /tmp from // older runs, and a configured TEPLOY_BACKUP_TMPDIR or policy tmp_dir, which is // where a killed run's plaintext dump would otherwise sit forever unnoticed. @@ -681,29 +789,34 @@ func tmpGlobs(extra string, names ...string) string { // CleanStaleWorkspaces removes run workspaces (dumps, possibly a credential file) // and any recovery key a killed `restore --identity` left, once they are older -// than the leftover age, and returns what it removed. Only called while holding -// the run lock, so a live `backup run --all` cannot be hit; the age protects a -// concurrent manual one. Never a killed restore's own files: those can be the only -// copy of something. +// than the leftover age or past their own expiry, and returns what it removed. Only +// called while holding the run lock, so a live `backup run --all` cannot be hit; the +// expiry protects a concurrent manual one. Never a killed restore's own files: those +// can be the only copy of something. func (c *Client) CleanStaleWorkspaces(ctx context.Context) []string { - out, err := c.exec.Run(ctx, fmt.Sprintf(`for d in %s; do [ -d "$d" ] || continue; if [ -n "$(find "$d" -maxdepth 0 -mmin +%d 2>/dev/null)" ]; then rm -rf -- "$d" && printf '%%s\n' "$d"; fi; done; true`, tmpGlobs("", "teploy-backup.*", "teploy-dr-identity.*"), tmpLeftoverMinutes)) + out, err := c.exec.Run(ctx, cleanWorkspacesScript(tmpGlobs("", "teploy-backup.*", "teploy-dr-identity.*"))) if err != nil { return nil } return splitLines(out) } +func cleanWorkspacesScript(globs string) string { + return fmt.Sprintf(`for d in %s; do [ -d "$d" ] || continue; if %s; then rm -rf -- "$d" && printf '%%s\n' "$d"; fi; done; true`, globs, leftoverTest()) +} + // LeftoverWorkspacesProbe lists what doctor and status report: the above plus a // kept or killed restore's and verify's scratch files and containers. func LeftoverWorkspacesProbe(tmpDir string) string { globs := tmpGlobs(tmpDir, "teploy-backup.*", "teploy-dr-identity.*", "teploy-restore.*", "teploy-volume-restore.*", "teploy-verify.*") - return fmt.Sprintf(`for d in %s; do [ -d "$d" ] || continue; if [ -n "$(find "$d" -maxdepth 0 -mmin +%d 2>/dev/null)" ]; then printf '%%s\n' "$d"; fi; done; docker ps -a --filter name=teploy-backup-verify- --format 'container {{.Names}}' 2>/dev/null; true`, globs, tmpLeftoverMinutes) + return fmt.Sprintf(`for d in %s; do [ -d "$d" ] || continue; if %s; then printf '%%s\n' "$d"; fi; done; docker ps -a --filter name=teploy-backup-verify- --format 'container {{.Names}}' 2>/dev/null; true`, globs, leftoverTest()) } // AbortStaleUploads aborts multipart uploads a killed run left under the given -// namespaces (invisible in listings, and they bill storage). Scoped to namespaces -// only this server writes to: with no prefix a bucket-wide list would catch other -// servers' uploads. Best effort; returns how many were aborted. +// namespaces, in the .incoming/ tree and at final names (invisible in listings, and +// they bill storage). Scoped to namespaces only this server writes to: with no +// prefix a bucket-wide list would catch other servers' uploads. Best effort; returns +// how many were aborted. func (c *Client) AbortStaleUploads(ctx context.Context, s3 S3Config, namespaces []string, now time.Time) int { q := ssh.ShellQuote aborted := 0 @@ -713,31 +826,33 @@ func (c *Client) AbortStaleUploads(ctx context.Context, s3 S3Config, namespaces if err != nil || fp == "" || slices.ContainsFunc(owners, func(o string) bool { return o != fp }) { continue } - prefix := ns + "/" + base := "" if s3.Prefix != "" { - prefix = s3.Prefix + "/" + prefix + base = s3.Prefix + "/" } - res := ssh.RunDetailed(ctx, c.exec, s3.AWS("s3api list-multipart-uploads --bucket "+q(s3.Bucket)+" --prefix "+q(prefix)+" --output json")) - if res.Failed() { - continue - } - var list struct { - Uploads []struct { - Key string `json:"Key"` - UploadID string `json:"UploadId"` - Initiated string `json:"Initiated"` - } `json:"Uploads"` - } - if json.Unmarshal(res.Stdout, &list) != nil { - continue - } - for _, u := range list.Uploads { - t, err := time.Parse(time.RFC3339, u.Initiated) - if err != nil || now.Sub(t) < tmpLeftoverMinutes*time.Minute { + for _, prefix := range []string{base + ns + "/", base + incomingDir + "/" + ns + "/"} { + res := ssh.RunDetailed(ctx, c.exec, s3.AWS("s3api list-multipart-uploads --bucket "+q(s3.Bucket)+" --prefix "+q(prefix)+" --output json")) + if res.Failed() { continue } - if _, err := c.exec.Run(ctx, s3.AWS("s3api abort-multipart-upload --bucket "+q(s3.Bucket)+" --key "+q(u.Key)+" --upload-id "+q(u.UploadID))); err == nil { - aborted++ + var list struct { + Uploads []struct { + Key string `json:"Key"` + UploadID string `json:"UploadId"` + Initiated string `json:"Initiated"` + } `json:"Uploads"` + } + if json.Unmarshal(res.Stdout, &list) != nil { + continue + } + for _, u := range list.Uploads { + t, err := time.Parse(time.RFC3339, u.Initiated) + if err != nil || now.Sub(t) < incomingStaleAfter { + continue + } + if _, err := c.exec.Run(ctx, s3.AWS("s3api abort-multipart-upload --bucket "+q(s3.Bucket)+" --key "+q(u.Key)+" --upload-id "+q(u.UploadID))); err == nil { + aborted++ + } } } } @@ -757,6 +872,13 @@ func (c *Client) AbortStaleUploads(ctx context.Context, s3 S3Config, namespaces // recipient tells apart two hosts that share both. It is in the body and not the // name so that an identity created later does not make a server its own stranger. // +// Two limits, documented rather than papered over with an id file a clone would copy +// too. A renamed host gets a new fingerprint, so it sees its old marker as another +// server's and refuses to prune: safe, and `owners --clear` ends it. A clone that +// keeps the machine-id and hostname and has no age identity is the same server to +// this check: clones need their own prefix (Dash always sets one) or a fresh +// machine-id. +// // Residual: an upgraded server's first prune cannot tell its own unmarked legacy // objects from those of a server that has not upgraded yet (AUDIT_OPEN). const ownersDir = ".owners" @@ -858,19 +980,19 @@ func (c *Client) markNamespaceOwner(ctx context.Context, s3 S3Config, app string // the only one writing there. var ErrSharedNamespace = errors.New("shared backup namespace") -// claimNamespace proves this server may prune the namespace, marks it (a prune -// never runs on an unmarked namespace), and returns the store's clock as read from -// the marker it just wrote: the one reference a wrong server clock cannot fool. -func (c *Client) claimNamespace(ctx context.Context, s3 S3Config, app string) (time.Time, error) { - var none time.Time +// claimNamespace proves this server may prune the namespace and marks it (a prune +// never runs on an unmarked namespace). The marker is written once, plus once more +// to record an identity created later: on a store that keeps versions every rewrite +// is another stored copy. +func (c *Client) claimNamespace(ctx context.Context, s3 S3Config, app string) error { base := "s3://" + s3.Base() owners, err := c.namespaceOwners(ctx, s3, app) if err != nil { - return none, fmt.Errorf("%w: could not list the owners of %s/%s, so nothing was pruned: %v", ErrSharedNamespace, base, app, err) + return fmt.Errorf("%w: could not list the owners of %s/%s, so nothing was pruned: %v", ErrSharedNamespace, base, app, err) } fp := c.fingerprint(ctx) if fp == "" { - return none, fmt.Errorf("%w: this server cannot identify itself (no machine-id or hostname), so nothing was pruned", ErrSharedNamespace) + return fmt.Errorf("%w: this server cannot identify itself (no machine-id or hostname), so nothing was pruned", ErrSharedNamespace) } var others []string for _, o := range owners { @@ -879,29 +1001,50 @@ func (c *Client) claimNamespace(ctx context.Context, s3 S3Config, app string) (t } } if len(others) > 0 { - return none, fmt.Errorf("%w: another server (%s) also stores %s backups under %s, so pruning here could delete its copies; nothing was pruned. Give each server its own prefix (`teploy server backup-target set --prefix `). If that marker is this server before a rebuild: `teploy server backup-target owners --clear %s`", ErrSharedNamespace, strings.Join(others, ", "), app, base, strings.Join(others, ",")) + return fmt.Errorf("%w: another server (%s) also stores %s backups under %s, so pruning here could delete its copies; nothing was pruned. Give each server its own prefix (`teploy server backup-target set --prefix `). If that marker is this server before a rebuild: `teploy server backup-target owners --clear %s`", ErrSharedNamespace, strings.Join(others, ", "), app, base, strings.Join(others, ",")) } key := namespaceOwnersPrefix(s3, app) + fp self := c.localRecipient(ctx) m := ownerMarker{Recipient: self, FirstSeen: time.Now().UTC().Format(time.RFC3339)} if slices.Contains(owners, fp) { old := c.readOwnerMarker(ctx, s3, key) - if old.FirstSeen != "" { - m.FirstSeen = old.FirstSeen - } if old.Recipient != "" && self != "" && old.Recipient != self { - return none, fmt.Errorf("%w: the marker for this machine under %s/%s names another age identity (%s), so this is a clone or a rebuilt server; nothing was pruned. If it is this server's own old marker: `teploy server backup-target owners --clear %s`", ErrSharedNamespace, base, app, old.Recipient, fp) + return fmt.Errorf("%w: the marker for this machine under %s/%s names another age identity (%s), so this is a clone or a rebuilt server; nothing was pruned. If it is this server's own old marker: `teploy server backup-target owners --clear %s`", ErrSharedNamespace, base, app, old.Recipient, fp) + } + if old.Recipient != "" || self == "" { + return nil } - if self == "" { - m.Recipient = old.Recipient + if old.FirstSeen != "" { + m.FirstSeen = old.FirstSeen } } if err := c.writeOwnerMarker(ctx, s3, key, m); err != nil { - return none, fmt.Errorf("%w: could not mark %s/%s as this server's before pruning, so nothing was pruned: %v", ErrSharedNamespace, base, app, err) + return fmt.Errorf("%w: could not mark %s/%s as this server's before pruning, so nothing was pruned: %v", ErrSharedNamespace, base, app, err) } + return nil +} + +// storeClock reads the store's clock: a tiny object written under .incoming/ (which +// a lifecycle rule expires, so it costs nothing on a store that keeps versions), +// dated by the store, and removed. It is the one reference a wrong server clock +// cannot fool. +func (c *Client) storeClock(ctx context.Context, s3 S3Config) (time.Time, error) { + key := "s3://" + s3.Base() + "/" + incomingDir + "/.clock-" + c.fingerprint(ctx) + q := ssh.ShellQuote + if _, err := c.exec.Run(ctx, "printf '%s\\n' clock | "+s3.AWS("s3 cp - "+q(key))); err != nil { + return time.Time{}, err + } + defer func() { + rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute) + defer cancel() + c.exec.Run(rctx, s3.AWS("s3 rm "+q(key))) + }() h, err := c.head(ctx, s3, key) if err != nil { - return none, nil // no clock reading; the marker is what mattered + return time.Time{}, err + } + if h.LastModified.IsZero() { + return time.Time{}, errors.New("the store's reply carries no date") } return h.LastModified, nil } @@ -935,9 +1078,20 @@ func (c *Client) ListNamespaceMarkers(ctx context.Context, s3 S3Config) ([]Names return out, nil } +// markerRecentWindow is how long after the last write to a namespace its markers +// count as live: a server that backs up at all writes within a week. +const markerRecentWindow = 7 * 24 * time.Hour + +// ErrMarkerRecent marks a clear refused because the namespace is in use. +var ErrMarkerRecent = errors.New("recently active backup namespace") + // ClearNamespaceMarkers removes a fingerprint's markers (a server that no longer -// exists, a rebuild's old self), returning the keys removed. -func (c *Client) ClearNamespaceMarkers(ctx context.Context, s3 S3Config, fingerprint string) ([]string, error) { +// exists, a rebuild's old self), returning the keys removed. The marker is all that +// stops another server's prune from deleting that server's objects, so a namespace +// written to in the last week is refused without force: its marker may be a server +// that is still running. Markers are not rewritten, so their own date says nothing +// about that; the namespace's newest write does. All of them or none. +func (c *Client) ClearNamespaceMarkers(ctx context.Context, s3 S3Config, fingerprint string, force bool, now time.Time) ([]string, error) { if !ownerMarkerRE.MatchString(fingerprint) { return nil, fmt.Errorf("invalid fingerprint %q", fingerprint) } @@ -945,6 +1099,22 @@ func (c *Client) ClearNamespaceMarkers(ctx context.Context, s3 S3Config, fingerp if err != nil { return nil, err } + if !force { + for _, m := range markers { + if m.Fingerprint != fingerprint { + continue + } + keys, err := c.listTree(ctx, s3, m.Namespace+"/") + if err != nil { + return nil, fmt.Errorf("%w: could not tell how recently %s was written to, so nothing was removed (--force to remove it anyway): %v", ErrMarkerRecent, m.Namespace, err) + } + for _, k := range keys { + if !k.Modified.IsZero() && now.Sub(k.Modified) < markerRecentWindow { + return nil, fmt.Errorf("%w: %s was written to on %s, so %s may be a live server, and without its marker a prune here deletes its backups; nothing was removed. If that server is really gone: --force", ErrMarkerRecent, m.Namespace, k.Modified.Format("2006-01-02"), fingerprint) + } + } + } + } var removed []string var errs []error for _, m := range markers { diff --git a/internal/backup/store_test.go b/internal/backup/store_test.go index f3eaf21..a206753 100644 --- a/internal/backup/store_test.go +++ b/internal/backup/store_test.go @@ -144,7 +144,7 @@ func treeMock(listing string) *ssh.MockExecutor { func rmKeys(mock *ssh.MockExecutor) []string { var out []string for _, c := range mock.Calls { - if strings.HasPrefix(c, "aws s3 rm ") { + if strings.HasPrefix(c, "aws s3 rm ") && !strings.Contains(c, "/.clock-") { f := strings.Fields(c) out = append(out, strings.Trim(f[3], "'")) } @@ -307,15 +307,15 @@ func TestPutUploadsToIncomingThenPublishes(t *testing.T) { order = append(order, "prove") case strings.HasPrefix(c, "rm -f '/w/x.pgdump'"): order = append(order, "rm-plain") - case strings.HasPrefix(c, "aws s3 cp '/w/x.pgdump.age' 's3://b/a/accessories/db/.incoming/1.pgdump.age' --metadata 'sha256=abc'"): + case strings.HasPrefix(c, "aws s3 cp '/w/x.pgdump.age' 's3://b/.incoming/a/accessories/db/1.pgdump.age' --metadata 'sha256=abc'"): order = append(order, "upload") case strings.HasPrefix(c, "aws s3 cp '/w/x.pgdump'"): t.Fatalf("plaintext uploaded: %s", c) case strings.HasPrefix(c, "{ aws s3 cp"): order = append(order, "readback") - case strings.HasPrefix(c, "aws s3 cp 's3://b/a/accessories/db/.incoming/1.pgdump.age' 's3://b/a/accessories/db/1.pgdump.age'"): + case strings.HasPrefix(c, "aws s3 cp 's3://b/.incoming/a/accessories/db/1.pgdump.age' 's3://b/a/accessories/db/1.pgdump.age'"): order = append(order, "publish") - case strings.HasPrefix(c, "aws s3 rm 's3://b/a/accessories/db/.incoming/1.pgdump.age'"): + case strings.HasPrefix(c, "aws s3 rm 's3://b/.incoming/a/accessories/db/1.pgdump.age'"): order = append(order, "clean") } } @@ -342,13 +342,13 @@ func TestPutRefusesWhatDoesNotVerify(t *testing.T) { t.Fatalf("err = %v, want %q", err, tc.want) } for _, c := range mock.Calls { - if strings.HasPrefix(c, "aws s3 cp 's3://b/k/.incoming/1.pgdump' 's3://b/k/1.pgdump'") { + if strings.HasPrefix(c, "aws s3 cp 's3://b/.incoming/k/1.pgdump' 's3://b/k/1.pgdump'") { t.Fatalf("an unverified object was published: %s", c) } } removed := false for _, c := range mock.Calls { - removed = removed || strings.HasPrefix(c, "aws s3 rm 's3://b/k/.incoming/1.pgdump'") + removed = removed || strings.HasPrefix(c, "aws s3 rm 's3://b/.incoming/k/1.pgdump'") } if !removed { t.Fatal("the unverified upload was left behind") @@ -626,7 +626,7 @@ func TestOneDatabaseFailingDoesNotStopTheOthers(t *testing.T) { lastPut = i } } - if !strings.Contains(mock.Calls[lastPut], "a/accessories/db/.incoming/") || strings.Contains(mock.Calls[lastPut], "databases/") { + if !strings.Contains(mock.Calls[lastPut], ".incoming/a/accessories/db/") || strings.Contains(mock.Calls[lastPut], "databases/") { t.Fatalf("the primary must be the last upload: %s", mock.Calls[lastPut]) } } @@ -772,7 +772,7 @@ func TestPruneSelectsEachDatabaseSeriesOnItsOwn(t *testing.T) { } } mock := treeMock(strings.Join(lines, "\n")) - if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepDaily: 3}); err != nil { + if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepDaily: 3, AllowMassDelete: true}); err != nil { t.Fatal(err) } removed := map[string]bool{} @@ -798,7 +798,7 @@ func TestPruneDatabaseSeriesIgnoresThePrimarysPeriods(t *testing.T) { lines = append(lines, "d t 1000 "+r+"databases/x/202601"+strconv.Itoa(d)+"-000000.pgdump") } mock := treeMock(strings.Join(lines, "\n")) - if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 5}); err != nil { + if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 5, AllowMassDelete: true}); err != nil { t.Fatal(err) } if got := len(rmKeys(mock)); got != 14 { @@ -807,7 +807,7 @@ func TestPruneDatabaseSeriesIgnoresThePrimarysPeriods(t *testing.T) { // The primary long gone: the series is still selected. lines[0] = "d t 1000 " + r + "20200101-000000.pgdump" mock = treeMock(strings.Join(lines, "\n")) - if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 5}); err != nil { + if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 5, AllowMassDelete: true}); err != nil { t.Fatal(err) } if got := len(rmKeys(mock)); got != 14 { @@ -899,29 +899,30 @@ func TestPruneRefusesAWrongClock(t *testing.T) { if _, err := NewClient(mock, io.Discard).PruneBackups(ctx, "a", "volumes", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1, IgnoreClock: true}); err != nil { t.Fatalf("--ignore-clock: %v", err) } - // A backup dated in the future would count as the newest: keep_last 1 would - // then delete today's real one. - mock = treeMock("d t 1000 " + r + time.Now().UTC().AddDate(1, 0, 0).Format("20060102-150405") + ".tar.gz\nd t 1000 " + r + time.Now().UTC().Format("20060102-150405") + ".tar.gz\n") - _, err = NewClient(mock, io.Discard).PruneBackups(ctx, "a", "volumes", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1}) - if !errors.Is(err, ErrClockSkew) || len(rmKeys(mock)) != 0 { - t.Fatalf("future-dated: err = %v, removed %v", err, rmKeys(mock)) - } } -func TestPrunePendingUploadsAtAnyDepth(t *testing.T) { - r := "a/accessories/db/" +// Pending uploads live in one tree, and go once no run could still own them. +func TestPruneCleansStalePendingUploads(t *testing.T) { + stale := time.Now().UTC().Add(-25 * time.Hour).Format("20060102-150405") + // Past the old fixed 7h, and well inside a 23h deadline. + live := time.Now().UTC().Add(-8 * time.Hour).Format("20060102-150405") listing := strings.Join([]string{ - "d t 1000 " + r + "20240103-000000.pgdump", - "d t 1000 " + r + "databases/x/.incoming/20240101-000000.pgdump", - "d t 1000 " + r + "globals/.incoming/20240101-000000.sql.gz", - "d t 1000 " + r + "meta/.incoming/20240101-000000.json", - "d t 1000 " + r + ".incoming/20240101-000000.pgdump", + "d t 1000 .incoming/a/accessories/db/databases/x/" + stale + ".pgdump", + "d t 1000 .incoming/a/accessories/db/" + stale + ".pgdump", + "d t 1000 .incoming/a/accessories/db/" + live + ".pgdump", }, "\n") - mock := treeMock(listing) + r := "a/accessories/db/" + mock := newStoreMock("h", + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "aws s3 ls 's3://b/.incoming/a/'", Output: listing}, + ssh.MockCommand{Match: "aws s3 ls", Output: "d t 1000 " + r + "20240103-000000.pgdump\n"}, + ssh.MockCommand{Match: "aws s3 rm", Output: ""}, + ) if _, err := NewClient(mock, io.Discard).PruneBackups(context.Background(), "a", "accessories/db", S3Config{Bucket: "b", Region: "r"}, RetentionPolicy{KeepLast: 1}); err != nil { t.Fatal(err) } - if got := rmKeys(mock); len(got) != 4 { + got := rmKeys(mock) + if len(got) != 2 || strings.Contains(strings.Join(got, ","), live) { t.Fatalf("removed %v", got) } } @@ -975,7 +976,7 @@ func TestPutLargeObjectUsesAProviderChecksum(t *testing.T) { } // A store or CLI without it falls back, and says so. var out strings.Builder - mock = big(ssh.MockCommand{Match: "aws s3 cp '/w/x' 's3://b/k/.incoming/1.pgdump' --metadata 'sha256=abc' --checksum-algorithm", Err: errors.New("exit status 2: Unknown options: --checksum-algorithm")}) + mock = big(ssh.MockCommand{Match: "aws s3 cp '/w/x' 's3://b/.incoming/k/1.pgdump' --metadata 'sha256=abc' --checksum-algorithm", Err: errors.New("exit status 2: Unknown options: --checksum-algorithm")}) cl := NewClient(mock, &out) cl.readBack = 1 if _, err := cl.put(ctx, S3Config{Bucket: "b", Region: "r"}, "/w/x", "s3://b/k/1.pgdump"); err != nil { @@ -1023,24 +1024,25 @@ func TestStaleWorkspacesLookInTheConfiguredDir(t *testing.T) { func TestAbortStaleUploadsIsScopedToOwnedNamespaces(t *testing.T) { old := time.Now().UTC().AddDate(0, 0, -2).Format(time.RFC3339) mock := newStoreMock("h", + ssh.MockCommand{Match: "aws s3api list-multipart-uploads --bucket 'b' --prefix '.incoming/mine/'", Output: `{"Uploads":[{"Key":".incoming/mine/y","UploadId":"u2","Initiated":"` + old + `"},{"Key":".incoming/mine/fresh","UploadId":"u3","Initiated":"` + time.Now().UTC().Add(-8*time.Hour).Format(time.RFC3339) + `"}]}`}, ssh.MockCommand{Match: "aws s3 ls 's3://b/mine/.owners/'", Output: "d t 20 mine/.owners/0123456789abcdef\n"}, ssh.MockCommand{Match: "aws s3 ls 's3://b/shared/.owners/'", Output: "d t 20 shared/.owners/aaaaaaaaaaaaaaaa\n"}, ssh.MockCommand{Match: "aws s3api list-multipart-uploads", Output: `{"Uploads":[{"Key":"mine/x","UploadId":"u1","Initiated":"` + old + `"}]}`}, ssh.MockCommand{Match: "aws s3api abort-multipart-upload", Output: ""}, ) - if n := NewClient(mock, io.Discard).AbortStaleUploads(context.Background(), S3Config{Bucket: "b", Region: "r"}, []string{"mine", "shared"}, time.Now().UTC()); n != 1 { + if n := NewClient(mock, io.Discard).AbortStaleUploads(context.Background(), S3Config{Bucket: "b", Region: "r"}, []string{"mine", "shared"}, time.Now().UTC()); n != 2 { t.Fatalf("aborted %d", n) } lists := 0 for _, c := range mock.Calls { if strings.HasPrefix(c, "aws s3api list-multipart-uploads") { lists++ - if !strings.Contains(c, "--prefix 'mine/'") { + if !strings.Contains(c, "--prefix 'mine/'") && !strings.Contains(c, "--prefix '.incoming/mine/'") { t.Fatalf("unscoped: %s", c) } } } - if lists != 1 { + if lists != 2 { t.Fatalf("lists = %d", lists) } } diff --git a/internal/backup/store_test_helpers_test.go b/internal/backup/store_test_helpers_test.go index 3824842..9fefa19 100644 --- a/internal/backup/store_test_helpers_test.go +++ b/internal/backup/store_test_helpers_test.go @@ -1,6 +1,10 @@ package backup -import "github.com/useteploy/teploy/internal/ssh" +import ( + "time" + + "github.com/useteploy/teploy/internal/ssh" +) // newStoreMock is ssh.NewMockExecutor with the verified-upload path answered: // the local measurement and the stored object agree, so a test about something @@ -16,7 +20,7 @@ var storeStubs = []ssh.MockCommand{ {Match: "{ ", Output: "abc -\n"}, {Match: "printf '%s\\n'", Output: ""}, {Match: "for p in /etc/teploy/age-key", Output: ""}, - {Match: "aws s3api head-object", Output: `{"ContentLength":4096,"Metadata":{"sha256":"abc"}}`}, + {Match: "aws s3api head-object", Output: `{"ContentLength":4096,"Metadata":{"sha256":"abc"},"LastModified":"` + time.Now().UTC().Format(time.RFC3339) + `"}`}, } // Valid X25519 age recipients (fixed values: public keys, no matching identity is kept). diff --git a/internal/cli/accessory.go b/internal/cli/accessory.go index 157c23e..b5f686c 100644 --- a/internal/cli/accessory.go +++ b/internal/cli/accessory.go @@ -777,6 +777,9 @@ func runAccessoryVerifyBackup(flags *Flags, appName, name, bucket, region, date, // Keep stdout clean for the JSON result; progress goes to stderr. out = os.Stderr } + if err := backup.ValidateDeadline(deadline); err != nil { + return err + } client := backup.NewClient(executor, out) client.SetDeadline(deadline) if err := applyAdoptedEngine(ctx, client, executor, appCfg.App, name); err != nil { diff --git a/internal/cli/backup.go b/internal/cli/backup.go index 0dc244e..ae978bb 100644 --- a/internal/cli/backup.go +++ b/internal/cli/backup.go @@ -268,6 +268,9 @@ func runBackupCreate(flags *Flags, appName, accessory, database string, adopted } client := backup.NewClient(executor, backupProgressOut(flags)) + if err := backup.ValidateDeadline(deadline); err != nil { + return err + } client.SetDeadline(deadline) var res backup.BackupResult @@ -760,6 +763,9 @@ func runBackupRestoreWith(flags *Flags, appName, accessory string, adopted bool, progress := backupProgressOut(flags) client := backup.NewClient(executor, progress) + if err := backup.ValidateDeadline(restoreOpts.Deadline); err != nil { + return err + } client.SetDeadline(restoreOpts.Deadline) if err := applyAdoptedEngine(ctx, client, executor, appCfg.App, accessory); err != nil { return err @@ -1114,6 +1120,7 @@ func newBackupPruneCmd(flags *Flags) *cobra.Command { maxAgeDays int gfs backup.FleetRetention ignoreClock bool + massDelete bool ) cmd := &cobra.Command{ @@ -1128,7 +1135,7 @@ func newBackupPruneCmd(flags *Flags) *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { gfs.KeepLast, gfs.MaxAgeDays = keepLast, maxAgeDays policy := gfs.Policy() - policy.IgnoreClock = ignoreClock + policy.IgnoreClock, policy.AllowMassDelete = ignoreClock, massDelete return runBackupPrune(flags, appName, adopted, &tf, accessory, policy) }, } @@ -1138,7 +1145,8 @@ func newBackupPruneCmd(flags *Flags) *cobra.Command { cmd.Flags().IntVar(&gfs.KeepDaily, "keep-daily", 0, "keep the newest backup of the last N days that have one") cmd.Flags().IntVar(&gfs.KeepWeekly, "keep-weekly", 0, "keep the newest backup of the last N ISO weeks that have one") cmd.Flags().IntVar(&gfs.KeepMonthly, "keep-monthly", 0, "keep the newest backup of the last N months that have one") - cmd.Flags().BoolVar(&ignoreClock, "ignore-clock", false, "prune even though this machine's clock disagrees with the store's, or a backup is dated in the future") + cmd.Flags().BoolVar(&ignoreClock, "ignore-clock", false, "prune even though this machine's clock disagrees with the store's, or the store's cannot be read") + cmd.Flags().BoolVar(&massDelete, "allow-mass-delete", false, "let this one run delete most of a series, which is otherwise refused (after tightening retention on purpose)") addBackupTargetFlags(cmd, &tf) cmd.Flags().StringVar(&accessory, "accessory", "", "Prune an accessory's backups instead of volumes (accessory name)") addRetentionFlags(cmd, &keepLast, &maxAgeDays) diff --git a/internal/cli/backup_cron_test.go b/internal/cli/backup_cron_test.go index 1f75654..b9c81e9 100644 --- a/internal/cli/backup_cron_test.go +++ b/internal/cli/backup_cron_test.go @@ -151,7 +151,7 @@ func TestDiscoverBackupSchedules(t *testing.T) { func TestResolveBackupS3(t *testing.T) { ctx := context.Background() stored := ssh.NewMockExecutor("srv", - ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "absent\n"}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ]", Output: "absent\n"}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"kept","region":"eu-west-1","endpoint":"http://127.0.0.1:9100","credentials":"AGE"}`}, ) cfg, err := resolveBackupS3(ctx, stored, &backupTargetFlags{}) diff --git a/internal/cli/backup_destructive_test.go b/internal/cli/backup_destructive_test.go index 013a7fb..3d0308a 100644 --- a/internal/cli/backup_destructive_test.go +++ b/internal/cli/backup_destructive_test.go @@ -139,7 +139,7 @@ func TestPreOperationBackupAllCoversEveryDatabase(t *testing.T) { t.Fatalf("%q %v", id, err) } joined := strings.Join(mock.Calls, "\n") - if !strings.Contains(joined, "databases/analytics/.incoming/") { + if !strings.Contains(joined, ".incoming/a/accessories/db/databases/analytics/") { t.Fatalf("analytics was not stored: %s", joined) } if !strings.Contains(joined, "/safety/"+id+".safety") { diff --git a/internal/cli/backup_fleet.go b/internal/cli/backup_fleet.go index a6ab586..35caadd 100644 --- a/internal/cli/backup_fleet.go +++ b/internal/cli/backup_fleet.go @@ -29,7 +29,7 @@ import ( // fleetRunDeadline bounds a whole `backup run --all`; each target has its own // backup.BackupDeadline inside it. A holder older than this plus an hour is stuck. -const fleetRunDeadline = 23 * time.Hour +const fleetRunDeadline = backup.MaxDeadline // serverToolPaths are directories cron's minimal PATH leaves out, where aws // (the AWS installer's symlink) and age may live. @@ -170,6 +170,9 @@ func runFleet(ctx context.Context, exec ssh.Executor, out io.Writer, ignoreSpace if len(targets) == 0 && !policy.AllowEmpty { return finishFleet(ctx, exec, out, res, policy, started, fmt.Errorf("the policy found nothing to back up on this server: no accessory, app volume or adopted container. If that is right, set allow_empty (teploy server backup-policy set --allow-empty)"), hc) } + if len(targets) > 0 && policy.AllowEmpty { + res.Warnings = append(res.Warnings, fmt.Sprintf("allow_empty is set and this server has %d target(s): if they all disappear, a run reports ok instead of failing (teploy server backup-policy set --allow-empty=false)", len(targets))) + } // A killed run's leftovers, now that this run owns the lock. Bounded: a wedged // janitor must not cost the backups. @@ -356,7 +359,8 @@ func newServerBackupPolicyCmd(flags *Flags) *cobra.Command { Use: "backup-policy", Short: "Manage the server's backup policy (the fleet engine's one cron line)", Long: "One policy per server, at /etc/teploy/backup-policy.json (0600; a non-root\n" + - "user who cannot write /etc/teploy gets ~/.config/teploy/backup-policy.json).\n" + + "user goes through passwordless sudo, and one with neither write access nor sudo\n" + + "keeps ~/.config/teploy/backup-policy.json unless a policy in /etc would shadow it).\n" + "A single cron line runs `teploy backup run --all`, which works out at run\n" + "time what to back up: every labelled accessory (each of its databases), app\n" + "volumes when enabled, and adopted containers. A per-app schedule stays as an\n" + @@ -826,14 +830,16 @@ func newServerBackupPolicyUnsetCmd(flags *Flags) *cobra.Command { return err } defer executor.Close() - // Cron first: a line with no policy behind it would fail every interval. - if err := backup.NewClient(executor, io.Discard).RemoveSchedule(ctx, backup.FleetCronMarker); err != nil { - return err - } + // The policy first: it can refuse (a policy this user cannot remove stays in + // force), and that must not leave it in force with no line running it. A line + // left behind by a failure below fails loudly every interval instead. removed, err := backup.UnsetFleetPolicy(ctx, executor) if err != nil { return err } + if err := backup.NewClient(executor, io.Discard).RemoveSchedule(ctx, backup.FleetCronMarker); err != nil { + return err + } if len(removed) == 0 { fmt.Println("No backup policy configured") return nil diff --git a/internal/cli/backup_fleet_test.go b/internal/cli/backup_fleet_test.go index 5a23639..605ff5b 100644 --- a/internal/cli/backup_fleet_test.go +++ b/internal/cli/backup_fleet_test.go @@ -276,7 +276,7 @@ func TestRunFleetNoTargetHeartbeatsDown(t *testing.T) { defer srv.Close() dir := t.TempDir() mock := ssh.NewMockExecutor("h", - ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","interval":"1h","heartbeat_urls":["` + srv.URL + `/api/push/SECRET"]}`}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ]", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","interval":"1h","heartbeat_urls":["` + srv.URL + `/api/push/SECRET"]}`}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "absent\n"}, ssh.MockCommand{Match: "python3 -c", Output: `{"seq":1,"ts":"2026-01-01T00:00:00Z"}`}, ) @@ -299,7 +299,7 @@ func TestRunFleetNoTargetHeartbeatsDown(t *testing.T) { } func TestRunFleetWithNoPolicyFails(t *testing.T) { - mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "absent\n"}) + mock := ssh.NewMockExecutor("h", ssh.MockCommand{Match: "if [ -d /etc/teploy ]", Output: "absent\n"}) res, err := runFleet(context.Background(), mock, io.Discard, false, nil) if err == nil || res.Status != backup.RunFailed || !strings.Contains(res.Error, "backup-policy set") { t.Fatalf("%v %+v", err, res) @@ -318,7 +318,7 @@ func TestRunFleetStandsDownOnOverlap(t *testing.T) { } defer lock.Release() mock := ssh.NewMockExecutor("h", - ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","heartbeat_urls":["` + srv.URL + `/hb"]}`}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ]", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","heartbeat_urls":["` + srv.URL + `/hb"]}`}, ) res, err := runFleet(context.Background(), mock, io.Discard, false, nil) if err != nil || res.Status != "skipped" { @@ -342,7 +342,7 @@ func TestRunFleetPartialFailure(t *testing.T) { return string(b) } mock := ssh.NewMockExecutor("h", - ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","interval":"1h","retention":{"keep_last":1},"heartbeat_urls":["` + srv.URL + `/hb"]}`}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ]", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","interval":"1h","retention":{"keep_last":1},"heartbeat_urls":["` + srv.URL + `/hb"]}`}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"b","region":"r"}`}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "for d in", Output: ""}, @@ -458,12 +458,12 @@ func TestEnsureServerPathCompletesCronPath(t *testing.T) { func TestSetBackupPolicyFromStdin(t *testing.T) { t.Setenv("TEPLOY_SERVER_BINARY", "/opt/teploy") mock := ssh.NewMockExecutor("srv", - ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "absent\n"}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ]", Output: "path /etc/teploy/backup-policy.json\n" + `{"v":"1"}`}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"b","region":"r"}`}, ssh.MockCommand{Match: "sched_daemon=no", Output: "daemon=yes\nenabled=enabled\ncrontab=yes\nteploy_entries=0\n"}, ssh.MockCommand{Match: "which age", Output: "/usr/bin/age"}, ssh.MockCommand{Match: "'/opt/teploy' backup run --help", Output: ""}, - ssh.MockCommand{Match: "d=\nif [ \"$(id -u)\" = 0 ]", Output: "written /etc/teploy/backup-policy.json\n"}, + ssh.MockCommand{Match: "e=/etc/teploy/backup-policy.json", Output: "written /etc/teploy/backup-policy.json\n"}, ssh.MockCommand{Match: "raw=$(crontab -l", Output: ""}, ssh.MockCommand{Match: "python3 -c", Output: `{"seq":1,"ts":"2026-01-01T00:00:00Z"}`}, ) @@ -502,7 +502,7 @@ func TestSetBackupPolicyFromStdin(t *testing.T) { // Refused before anything is stored. bad := backup.FleetPolicy{Adopted: []backup.AdoptedEntry{{Container: "x'; rm -rf /", Databases: backup.DatabaseSelector{All: true}}}} - refused := ssh.NewMockExecutor("srv", ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "absent\n"}) + refused := ssh.NewMockExecutor("srv", ssh.MockCommand{Match: "if [ -d /etc/teploy ]", Output: "absent\n"}) if err := setBackupPolicyOn(context.Background(), &cobra.Command{}, refused, &Flags{}, &policySetFlags{stdin: true}, &bad, nil); err == nil { t.Fatal("a hostile container name must be refused") } @@ -546,7 +546,7 @@ func TestRunFleetZeroTargets(t *testing.T) { dir := t.TempDir() mk := func(extra string) *ssh.MockExecutor { return ssh.NewMockExecutor("h", - ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","interval":"1h"` + extra + `,"heartbeat_urls":["` + srv.URL + `/hb"]}`}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ]", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","interval":"1h"` + extra + `,"heartbeat_urls":["` + srv.URL + `/hb"]}`}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"b","region":"r"}`}, ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, ssh.MockCommand{Match: "for d in", Output: ""}, @@ -567,3 +567,25 @@ func TestRunFleetZeroTargets(t *testing.T) { t.Fatalf("%v %+v", err, res) } } + +// A server set up with nothing on it keeps allow_empty after it stops being true, and +// then losing every target reports ok. +func TestRunFleetWarnsAboutAllowEmptyWithTargets(t *testing.T) { + dir := t.TempDir() + line, _ := json.Marshal(map[string]any{"name": "/a-db", "image": "postgres:17", "state": "running", + "labels": map[string]string{"teploy.role": "accessory", "teploy.app": "a", "teploy.accessory": "db"}, + "env": []string{"PG_MAJOR=17"}, "entrypoint": []string{}, "cmd": []string{}}) + mock := ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "if [ -d /etc/teploy ]", Output: "path " + dir + "/backup-policy.json\n" + `{"v":"1","interval":"1h","allow_empty":true,"excludes":["a/db"]}`}, + ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"b","region":"r"}`}, + ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, + ssh.MockCommand{Match: "for d in", Output: ""}, + ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: string(line) + "\n"}, + ssh.MockCommand{Match: "crontab -l", Output: ""}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq":1,"ts":"2026-01-01T00:00:00Z"}`}, + ) + res, _ := runFleet(context.Background(), mock, io.Discard, false, nil) + if !strings.Contains(strings.Join(res.Warnings, "\n"), "allow_empty is set") { + t.Fatalf("%+v", res) + } +} diff --git a/internal/cli/backup_target.go b/internal/cli/backup_target.go index 6c4a770..5468f2b 100644 --- a/internal/cli/backup_target.go +++ b/internal/cli/backup_target.go @@ -3,6 +3,7 @@ package cli import ( "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -10,6 +11,7 @@ import ( "regexp" "sort" "strings" + "time" "github.com/useteploy/teploy/internal/config" @@ -51,7 +53,10 @@ func newServerBackupTargetCmd(flags *Flags) *cobra.Command { // refuses a namespace holding another server's, so a rebuilt server's old self // would stop retention there forever, and growth is the only symptom. func newServerBackupTargetOwnersCmd(flags *Flags) *cobra.Command { - var clear string + var ( + clear string + force bool + ) cmd := &cobra.Command{ Use: "owners", Short: "List which servers have written under this target; --clear removes one's markers", @@ -59,7 +64,9 @@ func newServerBackupTargetOwnersCmd(flags *Flags) *cobra.Command { "refuses a namespace holding another server's. --clear removes a\n" + "server's markers, under the current and previous prefixes: for a server that\n" + "no longer exists, or this one before a rebuild. Backups are not touched. Do not\n" + - "clear a server that still writes here: give it its own --prefix.", + "clear a server that still writes here: give it its own --prefix. A namespace\n" + + "written to in the last week is refused without --force, and every clear is\n" + + "recorded in the server's event log.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) @@ -89,13 +96,9 @@ func newServerBackupTargetOwnersCmd(flags *Flags) *cobra.Command { locs = append(locs, l) } if clear != "" { - total := 0 - for _, l := range locs { - removed, err := client.ClearNamespaceMarkers(ctx, l, clear) - total += len(removed) - if err != nil { - return err - } + total, err := clearOwnerMarkers(ctx, executor, client, locs, clear, force, time.Now()) + if err != nil { + return err } fmt.Printf("Removed %d marker(s) of %s\n", total, clear) return nil @@ -121,10 +124,41 @@ func newServerBackupTargetOwnersCmd(flags *Flags) *cobra.Command { return nil }, } - cmd.Flags().StringVar(&clear, "clear", "", "remove this fingerprint's markers") + cmd.Flags().StringVar(&clear, "clear", "", "remove this fingerprint's markers (a namespace written to in the last week needs --force)") + cmd.Flags().BoolVar(&force, "force", false, "with --clear: remove them even though the namespace is in use, which lets a prune here delete that server's backups") return cmd } +// clearOwnerMarkers removes a fingerprint's markers from every location, and records +// it: the marker is all that stops a prune deleting that server's objects, so its +// removal is an event whether it went through or was refused. +func clearOwnerMarkers(ctx context.Context, exec ssh.Executor, client *backup.Client, locs []backup.S3Config, fingerprint string, force bool, now time.Time) (int, error) { + total := 0 + var err error + for _, l := range locs { + removed, cerr := client.ClearNamespaceMarkers(ctx, l, fingerprint, force, now) + total += len(removed) + if cerr != nil { + err = cerr + break + } + } + events.Record(ctx, exec, ownerClearEvent(fingerprint, force, total, err)) + return total, err +} + +func ownerClearEvent(fingerprint string, force bool, removed int, err error) events.Event { + ev := events.Event{App: backup.FleetApp, Kind: events.KindBackupOwners, Outcome: events.Succeeded, Detail: map[string]any{"fingerprint": fingerprint, "force": force, "removed": removed}} + if err != nil { + ev.Outcome = events.Failed + if errors.Is(err, backup.ErrMarkerRecent) { + ev.Outcome = events.Refused + } + ev.Detail = events.ErrDetail(ev.Detail, err) + } + return ev +} + // backupTargetDTO is the `server backup-target show --json` envelope // (backup-target-v1). It never carries secret material — only whether any // is stored. diff --git a/internal/cli/backup_target_test.go b/internal/cli/backup_target_test.go index 9353803..7bd2d74 100644 --- a/internal/cli/backup_target_test.go +++ b/internal/cli/backup_target_test.go @@ -2,10 +2,14 @@ package cli import ( "context" + "errors" + "io" "strings" "testing" + "time" "github.com/useteploy/teploy/internal/backup" + "github.com/useteploy/teploy/internal/events" "github.com/useteploy/teploy/internal/ssh" ) @@ -156,3 +160,49 @@ func TestBackupRunNeedsTarget(t *testing.T) { } } } + +// A marker is all that stops a prune deleting that server's objects: removing one, +// or being refused, is on the record. +func TestClearOwnerMarkersIsRecorded(t *testing.T) { + now := time.Date(2026, 11, 5, 12, 0, 0, 0, time.Local) + mk := func() *ssh.MockExecutor { + return ssh.NewMockExecutor("h", + ssh.MockCommand{Match: "{ cat /etc/machine-id", Output: "0123456789abcdef0123456789abcdef -\n"}, + ssh.MockCommand{Match: "aws s3 ls", Output: now.Add(-time.Hour).Format("2006-01-02 15:04:05") + " 20 app/.owners/aaaaaaaaaaaaaaaa\n"}, + ssh.MockCommand{Match: "aws s3 rm", Output: ""}, + ssh.MockCommand{Match: "python3 -c", Output: `{"seq":1,"ts":"2026-01-01T00:00:00Z"}`}, + ) + } + s3 := []backup.S3Config{{Bucket: "b", Region: "r"}} + recorded := func(mock *ssh.MockExecutor) { + t.Helper() + for _, c := range mock.Calls { + if strings.HasPrefix(c, "python3 -c") && strings.Contains(c, "append '_fleet'") { + return + } + } + t.Fatalf("no event: %v", mock.Calls) + } + mock := mk() + n, err := clearOwnerMarkers(context.Background(), mock, backup.NewClient(mock, io.Discard), s3, "aaaaaaaaaaaaaaaa", false, now) + if !errors.Is(err, backup.ErrMarkerRecent) || n != 0 { + t.Fatalf("%d %v", n, err) + } + recorded(mock) + mock = mk() + if n, err = clearOwnerMarkers(context.Background(), mock, backup.NewClient(mock, io.Discard), s3, "aaaaaaaaaaaaaaaa", true, now); err != nil || n != 1 { + t.Fatalf("%d %v", n, err) + } + recorded(mock) + // Through Sanitize, which is what reaches the log: a kind's detail is allowlisted. + ev := ownerClearEvent("f", false, 0, backup.ErrMarkerRecent) + events.Sanitize(&ev) + if ev.Outcome != events.Refused || ev.Detail["fingerprint"] != "f" || ev.Detail["error"] == nil { + t.Fatalf("%+v", ev) + } + ev = ownerClearEvent("f", true, 1, nil) + events.Sanitize(&ev) + if ev.Outcome != events.Succeeded || ev.Detail["force"] != true || ev.Detail["removed"] != 1 { + t.Fatalf("%+v", ev) + } +} diff --git a/internal/cli/contracts_golden_test.go b/internal/cli/contracts_golden_test.go index 2e34150..a6f8cb2 100644 --- a/internal/cli/contracts_golden_test.go +++ b/internal/cli/contracts_golden_test.go @@ -635,7 +635,7 @@ func TestContractsBackupFleetGolden(t *testing.T) { writeFixture(t, "backup-policy-envelope/valid/unconfigured.json", policyDTO("192.0.2.10", backup.FleetPolicy{}, "")) status := backupStatusDTO{MachineInterface: MachineInterface, Host: "192.0.2.10", ObservedAt: at, FleetStatus: backup.FleetStatus{ - Target: backup.StatusTarget{Configured: true, Bucket: "teploy-backups", Region: "us-east-1", Prefix: "prod", HasCredentials: true, Reachable: true, + Target: backup.StatusTarget{Configured: true, Bucket: "teploy-backups", Region: "us-east-1", Prefix: "prod", HasCredentials: true, Reachable: true, IncomingLifecycle: backup.LifecycleCovered, PreviousPrefixes: []backup.PreviousPrefix{{Prefix: ""}}}, Policy: backup.StatusPolicy{Configured: true, Path: policy.Path, Interval: "1h", CronInstalled: true, Retention: policy.Retention, Excludes: policy.Excludes, Adopted: policy.Adopted, Volumes: true, VolumeMaxGB: 20, HeartbeatCount: 1, HeartbeatHosts: []string{"https://dash.example.internal"}, Recipient: policy.Recipient}, diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index 3e2b800..7cb6acb 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -988,7 +988,7 @@ func doctorBackupPolicyCheck(ctx context.Context, exec ssh.Executor, now time.Ti defer cancel() st, err := backup.ReadFleetStatus(sctx, exec, now) if errors.Is(err, backup.ErrPolicyHidden) { - return doctorCheck{Name: "backup-policy", Result: doctorFail, Detail: err.Error(), Remediation: "connect as root, give this user passwordless sudo, or run on the server: chmod 711 /etc/teploy. Until then every backup through this user is refused rather than risk plaintext"} + return doctorCheck{Name: "backup-policy", Result: doctorFail, Detail: err.Error(), Remediation: "connect as root, give this user passwordless sudo, or set this user's own policy (teploy server backup-policy set). Until then every backup through this user is refused rather than risk plaintext"} } if err != nil { return doctorCheck{Name: "backup-policy", Result: doctorWarn, Detail: "could not read the backup status: " + err.Error()} @@ -1002,6 +1002,10 @@ func doctorBackupPolicyCheck(ctx context.Context, exec ssh.Executor, now time.Ti fails = append(fails, "the aws CLI is missing, so every backup here fails") remedies = append(remedies, backup.AWSCLIRemedy) } + if (cronBased || st.Target.Configured) && st.AWSCLIProblem != "" { + fails = append(fails, st.AWSCLIProblem+", so every backup here fails") + remedies = append(remedies, backup.AWSCLIRemedy) + } if st.Policy.Configured { if !st.Target.Configured { fails = append(fails, "the backup policy has no target, so every run fails") @@ -1052,6 +1056,18 @@ func doctorBackupPolicyCheck(ctx context.Context, exec ssh.Executor, now time.Ti fails = append(fails, "the backup target did not answer: "+st.Target.Error) remedies = append(remedies, "check the bucket, endpoint and credentials (teploy server backup-target set)") } + incoming := ".incoming/" + if st.Target.Prefix != "" { + incoming = st.Target.Prefix + "/.incoming/" + } + switch st.Target.IncomingLifecycle { + case backup.LifecycleAbsent: + warns = append(warns, "no lifecycle rule expires "+incoming+": on a store that keeps versions (B2 only hides on delete) every pending upload stays billed, about one extra copy of every backup") + remedies = append(remedies, "add a rule expiring "+incoming+" after 1 day, old versions included (README, backup target)") + case backup.LifecycleUnknown: + warns = append(warns, "could not read the bucket's lifecycle rules, so whether "+incoming+" expires is unknown") + remedies = append(remedies, "give the key s3:GetLifecycleConfiguration, or check by hand that a rule expires "+incoming+" after 1 day, old versions included (README, backup target)") + } if len(st.LeftoverWorkspaces) > 0 { warns = append(warns, "killed or kept backup operations left "+strings.Join(st.LeftoverWorkspaces, ", ")+" (dumps and possibly a credential file)") remedies = append(remedies, "the next teploy backup run --all removes workspaces and identities; check a kept restore's files before removing them by hand, and docker rm -fv a verify scratch") diff --git a/internal/cli/doctor_backup_test.go b/internal/cli/doctor_backup_test.go index c6ef36e..cac4640 100644 --- a/internal/cli/doctor_backup_test.go +++ b/internal/cli/doctor_backup_test.go @@ -2,6 +2,7 @@ package cli import ( "context" + "errors" "strings" "testing" "time" @@ -10,19 +11,24 @@ import ( ) func doctorPolicyMock(policy, target string, tools string, cron, events string) *ssh.MockExecutor { - return ssh.NewMockExecutor("h", - ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: policy}, - ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: target}, - ssh.MockCommand{Match: "command -v aws >/dev/null", Output: tools}, - ssh.MockCommand{Match: "crontab -l", Output: cron}, - ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: ""}, - ssh.MockCommand{Match: "which aws", Output: "/usr/bin/aws\n"}, - ssh.MockCommand{Match: "( c=$(python3", Output: ""}, - ssh.MockCommand{Match: "aws s3 ls", Output: ""}, - ssh.MockCommand{Match: "python3 -c", Output: events}, - ssh.MockCommand{Match: "for d in /deployments/*/volumes", Output: ""}, - ssh.MockCommand{Match: "for d in '/var/tmp'/teploy-backup.", Output: ""}, - ) + return ssh.NewMockExecutor("h", doctorPolicyCommands(policy, target, tools, cron, events)...) +} + +func doctorPolicyCommands(policy, target string, tools string, cron, events string) []ssh.MockCommand { + return []ssh.MockCommand{ + {Match: "if [ -d /etc/teploy ]", Output: policy}, + {Match: "for p in /etc/teploy/backup-target.json", Output: target}, + {Match: "command -v aws >/dev/null", Output: tools}, + {Match: "crontab -l", Output: cron}, + {Match: "ids=$(docker ps -aq", Output: ""}, + {Match: "which aws", Output: "/usr/bin/aws\n"}, + {Match: "( c=$(python3", Output: ""}, + {Match: "aws s3 ls", Output: ""}, + {Match: "aws s3api get-bucket-lifecycle-configuration", Output: `{"Rules":[{"Status":"Enabled","Filter":{"Prefix":".incoming/"},"Expiration":{"Days":1}}]}`}, + {Match: "python3 -c", Output: events}, + {Match: "for d in /deployments/*/volumes", Output: ""}, + {Match: "for d in '/var/tmp'/teploy-backup.", Output: ""}, + } } func TestDoctorBackupPolicyCheck(t *testing.T) { @@ -63,7 +69,7 @@ func TestDoctorBackupPolicyCheck(t *testing.T) { } // The aws remedy is exact. c := doctorBackupPolicyCheck(context.Background(), doctorPolicyMock(policy(full), target, "age\n", cron, good), now) - if !strings.Contains(c.Remediation, "awscli") || !strings.Contains(c.Remediation, "teploy setup") { + if !strings.Contains(c.Remediation, "AWS's installer") || !strings.Contains(c.Remediation, "teploy setup") { t.Fatalf("%s", c.Remediation) } } @@ -74,7 +80,7 @@ func TestDoctorFlagsNoPrefixInASharedBucket(t *testing.T) { now := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) mk := func(prefix string) *ssh.MockExecutor { return ssh.NewMockExecutor("h", - ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "path /etc/teploy/backup-policy.json\n" + `{"v":"1","interval":"1h","recipient":"` + testRecipient + `","heartbeat_urls":["https://d/x"]}`}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ]", Output: "path /etc/teploy/backup-policy.json\n" + `{"v":"1","interval":"1h","recipient":"` + testRecipient + `","heartbeat_urls":["https://d/x"]}`}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"b","region":"r","prefix":"` + prefix + `"}`}, ssh.MockCommand{Match: "command -v aws >/dev/null", Output: "aws\nage\n"}, ssh.MockCommand{Match: "crontab -l", Output: "5 * * * * /usr/local/bin/teploy backup run --all # [teploy-backup-all]\n"}, @@ -94,3 +100,39 @@ func TestDoctorFlagsNoPrefixInASharedBucket(t *testing.T) { t.Fatalf("%+v", c) } } + +// On a store that keeps versions, nothing else stops pending uploads billing forever. +func TestDoctorChecksTheIncomingLifecycle(t *testing.T) { + now := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) + policy := "path /etc/teploy/backup-policy.json\n" + `{"v":"1","interval":"1h","recipient":"` + testRecipient + `","heartbeat_urls":["https://d/x"]}` + target := "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"b","region":"r","prefix":"srv1"}` + cron := "5 * * * * /usr/local/bin/teploy backup run --all # [teploy-backup-all]\n" + good := `{"v":1,"id":"1","seq":1,"ts":"2026-11-04T11:30:00Z","app":"_fleet","kind":"backup.all","outcome":"succeeded","detail":{"status":"ok","targets_ok":2,"targets_failed":0}}` + "\n" + mk := func(lifecycle ssh.MockCommand) *ssh.MockExecutor { + return ssh.NewMockExecutor("h", append([]ssh.MockCommand{lifecycle}, doctorPolicyCommands(policy, target, "aws\nage\n", cron, good)...)...) + } + const op = "aws s3api get-bucket-lifecycle-configuration" + c := doctorBackupPolicyCheck(context.Background(), mk(ssh.MockCommand{Match: op, Err: errors.New("exit status 254: An error occurred (NoSuchLifecycleConfiguration)")}), now) + if c.Result != doctorWarn || !strings.Contains(c.Detail, "no lifecycle rule expires srv1/.incoming/") || !strings.Contains(c.Remediation, "after 1 day") { + t.Fatalf("%+v", c) + } + c = doctorBackupPolicyCheck(context.Background(), mk(ssh.MockCommand{Match: op, Err: errors.New("exit status 254: AccessDenied")}), now) + if c.Result != doctorWarn || !strings.Contains(c.Detail, "could not read the bucket's lifecycle rules") { + t.Fatalf("%+v", c) + } + c = doctorBackupPolicyCheck(context.Background(), mk(ssh.MockCommand{Match: op, Output: `{"Rules":[{"Status":"Enabled","Filter":{"Prefix":"srv1/.incoming/"},"Expiration":{"Days":1}}]}`}), now) + if c.Result != doctorOK { + t.Fatalf("%+v", c) + } +} + +// An aws CLI too old to upload to a locked bucket fails every backup. +func TestDoctorFailsAnOldAWSCLI(t *testing.T) { + now := time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) + policy := "path /etc/teploy/backup-policy.json\n" + `{"v":"1","interval":"1h","recipient":"` + testRecipient + `","heartbeat_urls":["https://d/x"]}` + target := "path /etc/teploy/backup-target.json\n" + `{"v":"1","bucket":"b","region":"r"}` + c := doctorBackupPolicyCheck(context.Background(), doctorPolicyMock(policy, target, "aws aws-cli/2.9.19 Python/3.11\nage\n", "", ""), now) + if c.Result != doctorFail || !strings.Contains(c.Detail, "older than") || !strings.Contains(c.Remediation, "teploy setup") { + t.Fatalf("%+v", c) + } +} diff --git a/internal/cli/doctor_test.go b/internal/cli/doctor_test.go index 4718bf0..8715430 100644 --- a/internal/cli/doctor_test.go +++ b/internal/cli/doctor_test.go @@ -41,7 +41,7 @@ func doctorHappyMock() *ssh.MockExecutor { ssh.MockCommand{Match: "sched_daemon=no", Output: "daemon=yes\nenabled=enabled\ncrontab=yes\nteploy_entries=0\n"}, ssh.MockCommand{Match: "for d in /var/tmp/teploy-pre-restore.*", Output: ""}, ssh.MockCommand{Match: "docker ps --filter label=teploy.role=accessory", Output: ""}, - ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "absent\n"}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ]", Output: "absent\n"}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-target.json", Output: "absent\n"}, ssh.MockCommand{Match: "command -v aws >/dev/null", Output: "aws\nage\n"}, ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: ""}, @@ -206,7 +206,7 @@ func TestDoctorHumanTable(t *testing.T) { ssh.MockCommand{Match: "for d in /var/tmp/teploy-pre-restore.*", Output: ""}, ssh.MockCommand{Match: "docker ps --filter label=teploy.role=accessory", Output: ""}, ssh.MockCommand{Match: "for p in /etc/teploy/backup-", Output: "absent\n"}, - ssh.MockCommand{Match: "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", Output: "absent\n"}, + ssh.MockCommand{Match: "if [ -d /etc/teploy ]", Output: "absent\n"}, ssh.MockCommand{Match: "command -v aws >/dev/null", Output: "aws\nage\n"}, ssh.MockCommand{Match: "ids=$(docker ps -aq", Output: ""}, ssh.MockCommand{Match: "for d in /deployments/*/volumes", Output: ""}, @@ -727,8 +727,10 @@ func assertDoctorReadOnlyCalls(t *testing.T, calls []string) { "for d in /var/tmp/teploy-pre-restore.*", "docker ps --filter label=teploy.role=accessory", "for p in /etc/teploy/backup-", - "if [ -d /etc/teploy ] && [ ! -x /etc/teploy ]", + "if [ -d /etc/teploy ]", "command -v aws >/dev/null", + "aws s3api get-bucket-lifecycle-configuration", + "aws s3 ls", "ids=$(docker ps -aq", "for d in /deployments/*/volumes", "for d in '/var/tmp'/teploy-backup", diff --git a/internal/cli/fleet_integration_test.go b/internal/cli/fleet_integration_test.go index 3c313c1..e1c5742 100644 --- a/internal/cli/fleet_integration_test.go +++ b/internal/cli/fleet_integration_test.go @@ -8,7 +8,7 @@ // // TEPLOY_FAULT_HOST=127.0.0.1: TEPLOY_FAULT_USER=tyler \ // TEPLOY_FAULT_KEY=~/.colima/_lima/_config/user \ -// go test -tags integration -p 2 -run TestFleetIntegration -v -timeout 40m ./internal/cli +// go test -tags integration -p 2 -run TestFleetIntegration -v -timeout 80m ./internal/cli // // Containers are capped and removed on exit. package cli @@ -16,6 +16,7 @@ package cli import ( "context" "encoding/json" + "errors" "fmt" "os" "os/exec" @@ -163,7 +164,7 @@ func TestFleetIntegration(t *testing.T) { if host == "" || user == "" || keyPath == "" { t.Skip("needs TEPLOY_FAULT_HOST, TEPLOY_FAULT_USER, TEPLOY_FAULT_KEY") } - ctx, cancel := context.WithTimeout(context.Background(), 35*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 70*time.Minute) defer cancel() cctx, ccancel := context.WithTimeout(ctx, 10*time.Second) defer ccancel() @@ -174,11 +175,11 @@ func TestFleetIntegration(t *testing.T) { t.Cleanup(func() { vm.Close() }) f := &fleetIT{t: t, exec: vm, ctx: ctx, host: host, user: user, key: keyPath} - containers := []string{"teploy-it-minio", "teployit-db", "teployit-cache", "teployit2-db", "teployit3-db", "wordpress-shared-mysql", "omni_postgres", "teploy-it-hook", "teploy-it-scratch"} + containers := []string{"teploy-it-minio", "teploy-it-minio2", "teployit-db", "teployit-cache", "teployit2-db", "teployit3-db", "wordpress-shared-mysql", "omni_postgres", "teploy-it-hook", "teploy-it-scratch"} cleanup := func() { c := context.Background() vm.Run(c, "docker rm -fv "+strings.Join(containers, " ")+" >/dev/null 2>&1; docker ps -aq --filter name=teploy-backup-verify- | xargs -r docker rm -fv >/dev/null 2>&1; pkill -f teploy-it-hook.py; pkill -9 -f '[s]leep 600'; true") - vm.Run(c, "rm -rf "+itDir+" /deployments/teployit /deployments/teployit2 /deployments/teployit3 /deployments/_adopted /deployments/.teploy/apps/teployit /deployments/.teploy/apps/teployit2 /deployments/.teploy/apps/teployit3 /deployments/.teploy/apps/_adopted /deployments/.teploy/apps/_fleet /var/tmp/teploy-backup.* /var/tmp/teploy-pre-restore.* /var/tmp/teploy-verify.* /var/tmp/teploy-restore.* /var/tmp/teploy-dr /tmp/teploy-pwned; crontab -r 2>/dev/null; sudo -n rm -rf /etc/teploy/backup-policy.json; sudo -n chmod 755 /etc/teploy 2>/dev/null; sudo -n userdel -rf teployit 2>/dev/null; rm -f \"$HOME\"/.config/teploy/backup-policy.json \"$HOME\"/.config/teploy/backup-target.json \"$HOME\"/.config/teploy/backup-run.lock; true") + vm.Run(c, "rm -rf "+itDir+" /deployments/teployit /deployments/teployit2 /deployments/teployit3 /deployments/_adopted /deployments/.teploy/apps/teployit /deployments/.teploy/apps/teployit2 /deployments/.teploy/apps/teployit3 /deployments/.teploy/apps/_adopted /deployments/.teploy/apps/_fleet /var/tmp/teploy-backup.* /var/tmp/teploy-pre-restore.* /var/tmp/teploy-verify.* /var/tmp/teploy-restore.* /var/tmp/teploy-dr /tmp/teploy-pwned; crontab -r 2>/dev/null; sudo -n rm -rf /etc/teploy/backup-policy.json; sudo -n chmod 755 /etc/teploy 2>/dev/null; sudo -n rmdir /etc/teploy 2>/dev/null; sudo -n userdel -rf teployit 2>/dev/null; rm -f \"$HOME\"/.config/teploy/backup-policy.json \"$HOME\"/.config/teploy/backup-target.json \"$HOME\"/.config/teploy/backup-run.lock; true") } cleanup() t.Cleanup(cleanup) @@ -588,7 +589,7 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() } // A partial object left at the verification-pending key, as a provider // with weaker semantics or a future bug might: never a backup. - partial := "teployit/accessories/db/.incoming/29991231-235959-0123456789abcdef.pgdump.age" + partial := ".incoming/teployit/accessories/db/29991231-235959-0123456789abcdef.pgdump.age" f.sh("printf partial > " + itDir + "/partial") f.awsIT("s3 cp " + itDir + "/partial s3://itfleet/" + partial + " --only-show-errors") // A truncated object at a FINAL key is also refused by the floor. @@ -708,6 +709,15 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() f.sh(seedScript) client := backup.NewClient(vm, os.Stderr) policy := backup.RetentionPolicy{KeepLast: 2, KeepDaily: 3, KeepWeekly: 4, KeepMonthly: 2} + // Most of a series going at once is refused, and nothing goes. + beforeBrake := f.keys("gfsapp/accessories/db/") + if _, err := client.PruneBackups(ctx, "gfsapp", "accessories/db", f.s3, policy); !errors.Is(err, backup.ErrPruneBrake) { + t.Fatalf("brake: %v", err) + } + if got := f.keys("gfsapp/accessories/db/"); len(got) != len(beforeBrake) { + t.Fatalf("the brake deleted something: %d of %d left", len(got), len(beforeBrake)) + } + policy.AllowMassDelete = true deleted, err := client.PruneBackups(ctx, "gfsapp", "accessories/db", f.s3, policy) if err != nil { t.Fatalf("prune: %v (deleted %v)", err, deleted) @@ -745,7 +755,7 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() seed2 += "s " + early + ".pgdump.age; s " + late + ".pgdump.age; s databases/reports/" + early + ".pgdump.age; " } f.sh(seed2) - daily := backup.RetentionPolicy{KeepDaily: 4} + daily := backup.RetentionPolicy{KeepDaily: 4, AllowMassDelete: true} if _, err := client.PruneBackups(ctx, "seriesapp", "accessories/db", f.s3, daily); err != nil { t.Fatalf("prune: %v", err) } @@ -830,11 +840,120 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() if !strings.Contains(head, "sha256") { t.Fatalf("no recorded sha256: %s", head) } - if left := f.keys("teployit/volumes/.incoming/"); len(left) != 0 { + if left := f.keys(".incoming/teployit/volumes/"); len(left) != 0 { t.Fatalf("pending upload left: %v", left) } }) + t.Run("8 MB and up: multipart copy keeps its checksum, a refused copy re-uploads, one pending tree", func(t *testing.T) { + f.t = t + bucketKey := func(uri string) (string, string) { + b, k, _ := strings.Cut(strings.TrimPrefix(uri, "s3://"), "/") + return b, k + } + f.sh("dd if=/dev/urandom of=/deployments/teployit/volumes/data/mid bs=1M count=20 2>/dev/null") + defer f.sh("rm -f /deployments/teployit/volumes/data/mid " + itShims + "/aws") + res, code, out := f.runAll("") + if code != 0 { + t.Fatalf("exit %d: %s", code, out) + } + vol := f.target(res, "teployit/volumes") + if vol.SizeBytes < 8<<20 { + t.Fatalf("not multipart: %+v", vol) + } + b, k := bucketKey(vol.Objects[0].Key) + if head := f.awsIT("s3api head-object --bucket " + b + " --key " + k); !strings.Contains(head, `"sha256"`) { + t.Fatalf("the multipart copy lost the recorded checksum: %s", head) + } + // A store that refuses the copy, as B2 is documented to: the object still + // arrives, verified, with its checksum, and restores. + realAWS := strings.TrimSpace(f.sh("command -v aws")) + shim := "#!/bin/sh\ncase \"$*\" in *'s3 cp s3://'*.incoming*s3://*) echo 'copy failed: An error occurred (InvalidRequest) when calling the UploadPartCopy operation: checksum algorithm cannot be changed' >&2; exit 1;; esac\nexec " + realAWS + " \"$@\"\n" + if err := vm.Upload(ctx, strings.NewReader(shim), itShims+"/aws", "0755"); err != nil { + t.Fatal(err) + } + res, code, out = f.runAll("PATH=" + itShims + ":$PATH") + if code != 0 || !strings.Contains(out, "uploaded straight to its final name") { + t.Fatalf("exit %d: %s", code, out) + } + vol = f.target(res, "teployit/volumes") + b, k = bucketKey(vol.Objects[0].Key) + if head := f.awsIT("s3api head-object --bucket " + b + " --key " + k); !strings.Contains(head, `"sha256"`) { + t.Fatalf("the re-upload lost the recorded checksum: %s", head) + } + f.sh("rm -f " + itShims + "/aws") + if left := f.keys(".incoming/teployit/volumes/"); len(left) != 0 { + t.Fatalf("pending uploads left: %v", left) + } + // Doctor reads the lifecycle rule the pending tree is meant for. + if st, err := backup.ReadFleetStatus(ctx, vm, time.Now().UTC()); err != nil || st.Target.IncomingLifecycle != backup.LifecycleAbsent { + t.Fatalf("no rule yet: %q %v", st.Target.IncomingLifecycle, err) + } + if c := doctorBackupPolicyCheck(ctx, vm, time.Now().UTC()); !strings.Contains(c.Detail, "no lifecycle rule expires .incoming/") { + t.Fatalf("doctor: %+v", c) + } + f.sh("printf '%s' " + ssh.ShellQuote(`{"Rules":[{"ID":"incoming","Status":"Enabled","Filter":{"Prefix":".incoming/"},"Expiration":{"Days":1}}]}`) + " > " + itDir + "/rules.json") + f.awsIT("s3api put-bucket-lifecycle-configuration --bucket itfleet --lifecycle-configuration file://" + itDir + "/rules.json") + defer f.awsIT("s3api delete-bucket-lifecycle --bucket itfleet") + if st, err := backup.ReadFleetStatus(ctx, vm, time.Now().UTC()); err != nil || st.Target.IncomingLifecycle != backup.LifecycleCovered { + t.Fatalf("with the rule: %q %v", st.Target.IncomingLifecycle, err) + } + }) + + t.Run("on a versioned store the pending bytes sit in one tree and markers are not rewritten", func(t *testing.T) { + f.t = t + // Versioning needs erasure coding: a second MinIO on four directories. + const port = "19101" + endpoint := "http://127.0.0.1:" + port + f.sh("docker rm -fv teploy-it-minio2 >/dev/null 2>&1; docker run -d --name teploy-it-minio2 --memory 512m -p 127.0.0.1:" + port + ":9000 -e MINIO_ROOT_USER=" + itAK + " -e MINIO_ROOT_PASSWORD=" + itSK + " cgr.dev/chainguard/minio:latest server /tmp/d{1...4} >/dev/null") + defer f.sh("docker rm -fv teploy-it-minio2 >/dev/null 2>&1; true") + aws := "AWS_ACCESS_KEY_ID=" + itAK + " AWS_SECRET_ACCESS_KEY=" + itSK + " aws --region us-east-1 --endpoint-url " + endpoint + " " + f.waitFor("the second minio", aws+"s3 mb s3://itv") + f.sh(aws + "s3api put-bucket-versioning --bucket itv --versioning-configuration Status=Enabled") + cfg := backup.S3Config{Bucket: "itv", Region: "us-east-1", Endpoint: endpoint, AccessKey: itAK, SecretKey: itSK, Prefix: "srv1"} + client := backup.NewClient(vm, os.Stderr) + retention := backup.RetentionPolicy{KeepLast: 1} + for i := 0; i < 3; i++ { + if _, err := client.AccessoryBackup(ctx, "teployit", "cache", "redis:7-alpine", nil, cfg); err != nil { + t.Fatal(err) + } + if _, err := client.PruneBackups(ctx, "teployit", "accessories/cache", cfg, retention); err != nil { + t.Fatal(err) + } + time.Sleep(1100 * time.Millisecond) + } + var versions struct { + Versions []struct { + Key string `json:"Key"` + } `json:"Versions"` + } + if err := json.Unmarshal([]byte(f.sh(aws+"s3api list-object-versions --bucket itv --output json")), &versions); err != nil { + t.Fatal(err) + } + counts := map[string]int{} + for _, v := range versions.Versions { + counts[v.Key]++ + if strings.Contains(strings.TrimPrefix(v.Key, "srv1/.incoming/"), ".incoming") { + t.Fatalf("a pending upload outside the one tree: %s", v.Key) + } + } + pending, marker := 0, 0 + for k, n := range counts { + switch { + case strings.HasPrefix(k, "srv1/.incoming/"): + pending += n + case strings.Contains(k, "/.owners/"): + marker += n + } + } + if pending == 0 { + t.Fatalf("no pending versions at all, so this proved nothing: %v", counts) + } + if marker != 1 { + t.Fatalf("the marker was rewritten: %d versions: %v", marker, counts) + } + }) + t.Run("never prunes across servers", func(t *testing.T) { f.t = t client := backup.NewClient(vm, os.Stderr) @@ -854,10 +973,17 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() if err != nil || !strings.Contains(out, "aaaaaaaaaaaaaaaa") { t.Fatalf("owners: %v\n%s", err, out) } - if out, err := f.cli("", "server", "backup-target", "owners", "--clear", "aaaaaaaaaaaaaaaa"); err != nil || !strings.Contains(out, "Removed 1 marker") { + // A marker written minutes ago may be a live server: refused without --force. + if out, err := f.cli("", "server", "backup-target", "owners", "--clear", "aaaaaaaaaaaaaaaa"); err == nil || !strings.Contains(out, "--force") { + t.Fatalf("clear of a recent marker: %v\n%s", err, out) + } + if out, err := f.cli("", "server", "backup-target", "owners", "--clear", "aaaaaaaaaaaaaaaa", "--force"); err != nil || !strings.Contains(out, "Removed 1 marker") { t.Fatalf("clear: %v\n%s", err, out) } - if _, err := client.PruneBackups(ctx, "gfsapp", "accessories/db", f.s3, backup.RetentionPolicy{KeepLast: 1}); err != nil { + if log := f.sh("grep backup_target.owners_clear /deployments/.teploy/apps/_fleet/events.jsonl"); !strings.Contains(log, `"outcome":"refused"`) || !strings.Contains(log, `"force":true`) { + t.Fatalf("the clears are not on record: %s", log) + } + if _, err := client.PruneBackups(ctx, "gfsapp", "accessories/db", f.s3, backup.RetentionPolicy{KeepLast: 1, AllowMassDelete: true}); err != nil { t.Fatalf("prune after clearing: %v", err) } // Per-server prefixes: a sibling prefix is invisible to this one's prune. @@ -868,7 +994,7 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() f.awsIT("s3 cp " + itDir + "/blob s3://itfleet/server-a/shared/accessories/db/20200102-000000.pgdump --only-show-errors") a := f.s3 a.Prefix = "server-a" - if _, err := client.PruneBackups(ctx, "shared", "accessories/db", a, backup.RetentionPolicy{KeepLast: 1}); err != nil { + if _, err := client.PruneBackups(ctx, "shared", "accessories/db", a, backup.RetentionPolicy{KeepLast: 1, AllowMassDelete: true}); err != nil { t.Fatal(err) } if got := f.keys("server-a/shared/accessories/"); len(got) != 1 { @@ -939,7 +1065,7 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() } // Prune never touches the unowned old prefix. before := f.keys("teployit/accessories/db/") - if _, err := client.PruneBackups(ctx, "teployit", "accessories/db", cfg, backup.RetentionPolicy{KeepLast: 1}); err != nil { + if _, err := client.PruneBackups(ctx, "teployit", "accessories/db", cfg, backup.RetentionPolicy{KeepLast: 1, AllowMassDelete: true}); err != nil { t.Fatal(err) } if got := f.keys("teployit/accessories/db/"); len(got) != len(before) { @@ -965,7 +1091,7 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() t.Fatalf("%q owned=%v: %+v", pp.Prefix, pp.Owned, moved.PreviousPrefixes) } } - if _, err := client.PruneBackups(ctx, "teployit", "accessories/db", TargetS3(moved), backup.RetentionPolicy{KeepLast: 1}); err != nil { + if _, err := client.PruneBackups(ctx, "teployit", "accessories/db", TargetS3(moved), backup.RetentionPolicy{KeepLast: 1, AllowMassDelete: true}); err != nil { t.Fatal(err) } if got := len(f.keys("teployit/accessories/db/")); got != rootBefore { @@ -991,7 +1117,7 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() // Owned: it ages out under retention, and the newest of all survives. owned := cfg owned.PreviousPrefixes = []backup.PreviousPrefix{{Prefix: "", Owned: true}} - if _, err := client.PruneBackups(ctx, "teployit", "accessories/db", owned, backup.RetentionPolicy{KeepLast: 1}); err != nil { + if _, err := client.PruneBackups(ctx, "teployit", "accessories/db", owned, backup.RetentionPolicy{KeepLast: 1, AllowMassDelete: true}); err != nil { t.Fatal(err) } if got := f.keys("teployit/accessories/db/"); len(got) >= len(before) { @@ -1434,7 +1560,7 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() f.t = t policy := `{"v":"1","interval":"1h","recipient":"` + f.pub + `"}` f.sh("sudo -n mkdir -p /etc/teploy && printf '%s' " + ssh.ShellQuote(policy) + " | sudo -n tee /etc/teploy/backup-policy.json >/dev/null && sudo -n chmod 600 /etc/teploy/backup-policy.json && sudo -n chmod 700 /etc/teploy") - defer f.sh("sudo -n rm -f /etc/teploy/backup-policy.json; sudo -n chmod 755 /etc/teploy; sudo -n userdel -rf teployit 2>/dev/null; true") + defer f.sh("sudo -n rm -f /etc/teploy/backup-policy.json; sudo -n chmod 755 /etc/teploy; sudo -n rmdir /etc/teploy 2>/dev/null; sudo -n userdel -rf teployit 2>/dev/null; true") // This user has passwordless sudo, as deploy users do: the policy is read. got, err := backup.ReadFleetPolicy(ctx, vm) if err != nil || got.Recipient != f.pub { @@ -1455,6 +1581,41 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() if !res.Failed() || !strings.Contains(string(res.Stdout)+string(res.Stderr), "cannot tell whether") { t.Fatalf("--all: exit %d\n%s%s", res.ExitCode, res.Stdout, res.Stderr) } + // Back to the user with sudo: rotation lands where reads look, no second copy. + rotated := strings.TrimSpace(strings.Split(strings.SplitN(func() string { + c := exec.Command(f.local, "backup", "keygen") + b, err := c.Output() + if err != nil { + t.Fatal(err) + } + return string(b) + }(), "# public key: ", 2)[1], "\n")[0]) + path, fallback, err := backup.WriteFleetPolicy(ctx, vm, backup.FleetPolicy{Interval: "1h", Recipient: rotated}) + if err != nil || path != backup.PolicyFileEtc || fallback { + t.Fatalf("write through sudo: %s %v %v", path, fallback, err) + } + if got, err = backup.ReadFleetPolicy(ctx, vm); err != nil || got.Recipient != rotated { + t.Fatalf("show after set: %+v %v", got, err) + } + if !strings.Contains(f.sh("sudo -n cat "+backup.PolicyFileEtc), rotated) { + t.Fatal("the file in /etc was not rotated") + } + if left := ssh.RunDetailed(ctx, vm, `test -e "$HOME"/.config/teploy/backup-policy.json`); !left.Failed() { + t.Fatal("a second policy was written under ~/.config") + } + // The run this user's cron would make: its lock cannot live in /etc/teploy. + _, _, out := f.runAll("") + if strings.Contains(out, "backup-run.lock") || strings.Contains(out, "permission denied") { + t.Fatalf("the run failed on its lock: %s", out) + } + // Unset through the same path, and it says so. + removed, err := backup.UnsetFleetPolicy(ctx, vm) + if err != nil || strings.Join(removed, ",") != backup.PolicyFileEtc { + t.Fatalf("unset: %v %v", removed, err) + } + if got, err = backup.ReadFleetPolicy(ctx, vm); err != nil || got.Configured() { + t.Fatalf("after unset: %+v %v", got, err) + } }) t.Run("zero targets is a failure unless the policy allows it", func(t *testing.T) { diff --git a/internal/cli/setup.go b/internal/cli/setup.go index d563fef..8c3ce0d 100644 --- a/internal/cli/setup.go +++ b/internal/cli/setup.go @@ -700,7 +700,8 @@ func setupStages() []setupStage { affects: "aws CLI (apt, else AWS's own bundle) and age (apt, if missing) — the backup transport and payload encryption; a failure here warns, since nothing but backups needs them", run: func(ctx context.Context, exec ssh.Executor, w io.Writer, env *setupEnv) error { fmt.Fprintln(w, "Checking aws CLI and age...") - if _, err := exec.Run(ctx, "command -v aws >/dev/null && aws --version >/dev/null 2>&1"); err == nil { + out, err := exec.Run(ctx, "command -v aws >/dev/null && aws --version 2>&1") + if err == nil && backup.AWSCLIVersionProblem(out) == "" { fmt.Fprintln(w, " aws CLI already installed") } else { fmt.Fprintln(w, " Installing the aws CLI...") diff --git a/internal/events/events.go b/internal/events/events.go index 1c0d1d4..a534529 100644 --- a/internal/events/events.go +++ b/internal/events/events.go @@ -114,6 +114,7 @@ const ( KindBackupRestore = "backup.restore" KindBackupVerify = "backup.verify" KindBackupTarget = "backup_target.set" + KindBackupOwners = "backup_target.owners_clear" KindBackupAll = "backup.all" KindBackupPolicy = "backup_policy.set" ) @@ -150,6 +151,7 @@ var detailAllow = map[string][]string{ KindBackupVerify: {"accessory", "database", "date", "ok", "engine", "kind", "metric", "key", "size_bytes", "duration_ms", "encrypted", "error"}, KindBackupRestore: {"id", "kind", "accessory", "action", "database", "released", "pre_backup_id", "error"}, KindBackupTarget: {"id", "kind", "accessory", "error"}, + KindBackupOwners: {"fingerprint", "force", "removed", "error"}, KindBackupAll: {"status", "targets_ok", "targets_failed", "omissions", "encrypted", "duration_ms", "heartbeats_ok", "heartbeats_failed", "warnings", "error"}, KindBackupPolicy: {"interval", "encrypted", "volumes", "adopted", "error"}, } From 437b67a42615da3ce708fff58c3409a40372f6e5 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:03:14 -0700 Subject: [PATCH 14/17] docs: record the Slice 4 re-review, the lifecycle rule and the fingerprint limits Co-Authored-By: Claude Opus 5.5 --- AUDIT_OPEN.md | 26 ++++++++++++++++ CHANGELOG.md | 19 ++++++++++++ README.md | 85 +++++++++++++++++++++++++++++++++++++++------------ 3 files changed, 111 insertions(+), 19 deletions(-) diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 9959c27..daac72f 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -51,7 +51,33 @@ Regressions: `go test -tags integration ./internal/backup`, the full engine matr v0.1.51 (postgres, pgvector, timescale, mysql, mariadb, mongo, redis, valkey, dragonfly). The DR fixture still needs a 022 umask. +Re-review (2026-11-05, 16 items), all on the branch. Blocker and highs: a publish copy of 8 MB +or more becomes UploadPartCopy, which B2 documents cannot change a checksum algorithm, and R2 +cannot do with its default tag read (`--copy-props metadata-directive`, then a re-upload of the +verified local file on any refusal); pending uploads at every depth billed forever on a store +that keeps versions (one `.incoming/` tree, doctor reads the lifecycle rule, the marker is not +rewritten); a sudoer read /etc and wrote ~/.config, so rotation did nothing (one path, a read-back +in `WriteFleetPolicy`, and the lock falls back). Medium and low: `owners --clear` on a live marker, +a prune trusting a same-host clock and skipping an unreadable one, a janitor fixed at 7h, uncapped +safety copies, an unreachable previous prefix staling everything, the wrong chmod 711 remedy, +future-dated objects, the checksum head with no fallback, `--exclude` unnamed, no minimum aws +version, the kill order on a stuck holder. Decisions: the clone and renamed-host limits of the +fingerprint are documented, not papered over (an id file is copied by a clone too); marker recency +is the namespace's newest write, since a marker that is not rewritten says nothing about liveness; +the minimum aws is 2.23.0/1.37.0 from aws-cli's own changelog (first default checksum on upload). + +Proven only against MinIO, not real B2 or R2: every publish path. The refusal is simulated by a +shim, with an assumed error text, and a re-upload follows anything shaped like a store's error +reply rather than one exact string for that reason. Also unverified on B2: whether its S3 API +answers `GetBucketLifecycleConfiguration` (if not, doctor warns "unknown", which is true and +permanent there), whether `--copy-props metadata-directive` carries metadata through its multipart +copy, and Object Lock with aws 2.23+. The real-B2 smoke test is the coordinator's. + Open / deferred: +- The run lock falls back per user when the policy's directory is not writable, so a root run + and a sudoer's run do not exclude each other. Only one of them has the cron line. +- `backup status` can still read a future-dated object as the newest good one, so one stray + object can show a target covered. Prune no longer trusts it; status has not been changed. - Legacy objects from a server that has not upgraded yet carry no marker. A first prune in a shared root by an upgraded server cannot tell them from its own: only markers from the moment each server upgrades protect it. Per-server prefixes close this (decided: Dash always diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b3b6dd..6bc7435 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,6 +62,25 @@ All notable changes to teploy are documented here. Format follows [Keep a Change newest good backup is never pruned even with only `--max-age-days` set. - `backup list` shows per-database rows only with `--databases`: a caller restoring a row by id and accessory alone would replace the primary with another database's dump. +- Pending uploads live in one `/.incoming/` tree, so a 1-day lifecycle rule can + expire them on a store that keeps versions; doctor reads the rule and warns without one. + Owner markers are written once, not on every prune. +- A publish copy names `--copy-props metadata-directive`. A store that refuses the copy + gets the verified file uploaded straight to its final name. +- The policy is written and removed through passwordless sudo when that is how it is + read, and `set`/`unset` refuse when a policy would stay in force. The run lock falls + back to `~/.config/teploy` when the policy's directory is not writable. +- `owners --clear` refuses a namespace written to in the last week without `--force`, + and records every clear or refusal (`backup_target.owners_clear`). +- A prune refuses when it would delete most of a series (`--allow-mass-delete`) and when + the store's clock cannot be read. A future-dated object no longer stops retention: it + is left out of selection, never deleted, and warned about. +- `target_deadline` and `--deadline` over 23h are refused rather than capped. A workspace + records its own deadline for the janitor; a pending upload is a leftover after 24h. +- Safety copies are capped at the newest 5 per target. An unreachable previous prefix is + a warning, not a failed listing and a stale status. +- The aws CLI must be at least 2.23.0 (or 1.37.0), the first to send a checksum on every + upload, which Object Lock requires. `teploy setup` replaces an older one. ### Fixed diff --git a/README.md b/README.md index 95cab31..905173c 100644 --- a/README.md +++ b/README.md @@ -725,32 +725,66 @@ and `backup.all` (under the pseudo-app `_fleet`) and `--json` both report `ok`, hits, a DB-like container nothing adopts, including engines teploy has no method for such as ClickHouse or Qdrant, a schema the tools cannot address). A clean status with omissions is not a complete backup. Zero targets is a failure unless the policy says -`--allow-empty`: a lost accessory and an empty docker daemon look the same. A target -that has earlier backups and now holds nothing to back up (its databases dropped) is -a failure too. `--tmp-dir` builds on another disk and `--target-deadline` replaces the -6h per-target deadline; manual commands take `--deadline`. A run that finds another -running stands down; the lock is an `flock`, so a killed run cannot block the next, -and a holder alive past the 23h run deadline is killed. Each target has a 6h deadline. +`--allow-empty`: a lost accessory and an empty docker daemon look the same (and a run +warns while `allow_empty` is set on a server that has targets, since losing them all +would then report ok). A target that has earlier backups and now holds nothing to back +up (its databases dropped) is a failure too, whose message names the `--exclude` that +ends it. `--tmp-dir` builds on another disk and `--target-deadline` replaces the 6h +per-target deadline; manual commands take `--deadline`. Neither may exceed the 23h a run +may last: that is refused, not capped. A run that finds another running stands down; the +lock is an `flock`, so a killed run cannot block the next, and a holder alive past the +23h run deadline is killed. The lock sits beside the policy, or in `~/.config/teploy` +when this user cannot write there. A killed run's leftovers are invisible to later runs and visible to doctor: a final -key only ever holds a verified object (the upload goes to a `.incoming/` key, its -size and SHA-256 are checked, up to 256 MiB its bytes are read back, above that the -store's own `--checksum-algorithm SHA256` is used, and only then is it copied into place), sidecars are stored before the primary object that commits -a backup, and objects below their engine's plausibility floor are never a good copy. +key only ever holds a verified object (the upload goes to one `/.incoming/` +tree, its size and SHA-256 are checked, up to 256 MiB its bytes are read back, above that +the store's own `--checksum-algorithm SHA256` is used, and only then is it copied into +place), sidecars are stored before the primary object that commits a backup, and objects +below their engine's plausibility floor are never a good copy. A store that refuses the +server-side copy gets the verified file uploaded straight to its final name instead, and +verified there. A workspace records its own deadline, so a janitor never cleans a live +long run; a pending upload is a leftover after 24h. + +**Give `.incoming/` a 1-day lifecycle rule.** Every backup passes through it, and on a +store that keeps versions (B2 only hides on delete; a versioned bucket) what was deleted +there stays billed: about one extra copy of every backup, forever. `teploy doctor` reads +the rule through `GetBucketLifecycleConfiguration` and warns when there is none, or when +the store will not say. Scope the rule to `.incoming/` under the server's prefix: a rule +on the whole bucket deletes the backups. + +```sh +aws s3api put-bucket-lifecycle-configuration --bucket B --lifecycle-configuration '{"Rules":[{ + "ID":"teploy-incoming","Status":"Enabled","Filter":{"Prefix":"PREFIX/.incoming/"}, + "Expiration":{"Days":1},"NoncurrentVersionExpiration":{"NoncurrentDays":1}, + "AbortIncompleteMultipartUpload":{"DaysAfterInitiation":1}}]}' +``` + +On B2 the same thing in its own terms is a lifecycle rule with `fileNamePrefix` +`PREFIX/.incoming/`, `daysFromUploadingToHiding` 1 and `daysFromHidingToDeleting` 1. + +The aws CLI must be at least 2.23.0 (or 1.37.0), the first to send a checksum on every +upload, which a bucket with Object Lock requires. An older one is refused with the remedy, +and doctor fails on it; `teploy setup` replaces it. **Retention** is a union of rules over the good backups, restic `forget` style: `--keep-last N`; `--keep-daily/-weekly/-monthly N` keep the newest backup of the last N UTC days, ISO weeks and months that have one (an empty period does not use one up); `--max-age-days D` keeps everything younger than D days (restic's `keep-within`), and anything older goes unless another rule keeps it. The newest good backup is never -deleted, and a safety copy a destructive operation took outlasts every rule for 7 days. +deleted, and the newest 5 safety copies a destructive operation took, per target, outlast +every rule for 7 days. The primary and each database's series are selected on their own, so a database that fails in the last run of each day still gets its `keep_daily`; ids the primary keeps are also kept for the databases. A pruned backup takes its sidecars with it. The same flags work on `teploy backup prune`. A run that stored nothing prunes nothing. A prune refuses (a warning in the run's output, event and doctor: the backup is stored, retention -has stopped) when this machine's clock is over an hour from the store's or a backup is -dated in the future (`backup prune --ignore-clock`). +has stopped) when this machine's clock is over an hour from the store's, when the store's +clock cannot be read (`backup prune --ignore-clock` for both), and when one run would +delete most of a series, at least 3 backups and over half of it: a store on the same host +shares a wrong clock, and a mistaken policy looks the same. Tighten retention on purpose +with one `backup prune --allow-mass-delete`. A backup dated in the future is left out of +selection: never deleted, never counted as the newest, and warned about on every run. **Heartbeats**: after every run the server GETs each URL, adding `?status=down` on anything but `ok` (`/fail` for healthchecks.io). A failing receiver is reported and @@ -799,10 +833,13 @@ verify-backup`, and `teploy dr restore --identity recovery.txt --bucket B - E --prefix P`. A non-root SSH user usually cannot see into `/etc/teploy` (0700 once root has used it), -where a policy would be invisible rather than absent. Teploy asks through passwordless -sudo when it can; a user with its own `~/.config/teploy/backup-policy.json` follows that; -anyone else is refused rather than risk uploading plaintext on a server configured to -encrypt (`chmod 711 /etc/teploy` lets the refusal go away, and leaves the files 0600). +where a policy would be invisible rather than absent. Teploy reads, writes and removes the +policy there through passwordless sudo when it can, so there is one policy and rotating a +key rotates it. A user with neither sudo nor write access keeps its own +`~/.config/teploy/backup-policy.json` when nothing in `/etc` would shadow it; if it can see +one there, `set` and `unset` refuse rather than store a copy nothing reads. A user that +cannot tell is refused rather than risk uploading plaintext on a server configured to +encrypt: connect as root, give it passwordless sudo, or set its own policy. #### Shared buckets @@ -815,8 +852,18 @@ policy's retention. As a second guard every server leaves a marker at `//.owners/` before its first prune, and a namespace holding another server's marker, or one whose owners cannot be listed, is never pruned. `teploy server backup-target owners` lists the markers; `--clear ` removes those of -a server that is gone, or this one before a rebuild. A policy with no prefix in a bucket another server -writes to is flagged by `teploy doctor`. +a server that is gone, or this one before a rebuild. Without its marker a prune here deletes +that server's backups, so a namespace written to in the last week needs `--force`, and every +clear or refusal is an event in the `_fleet` log. A policy with no prefix in a bucket another +server writes to is flagged by `teploy doctor`. + +The fingerprint is a hash of `/etc/machine-id` and the hostname, plus the server's age public +key in the marker. Two things follow, on purpose rather than papered over with an id file a +clone would copy too. Renaming a host gives it a new fingerprint: it sees its old marker as +another server's and refuses to prune, which is safe, and `owners --clear --force` ends +it. A clone that keeps the machine-id and hostname and has no age identity is the same server +to this check, so clones need their own `--prefix` (Dash always sets one) or a fresh +`/etc/machine-id`. #### Before anything destructive From a76d6e46a52d1e4b76cb7d6651788bdc88b29b2b Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:49:03 -0700 Subject: [PATCH 15/17] test(backup): make the versioned-store and sudo scenarios run for real MinIO needs MINIO_CI_CD to erasure-code directories on one disk, and a home policy left by an earlier scenario is not the second copy the sudo scenario looks for. Co-Authored-By: Claude Opus 5.5 --- internal/cli/fleet_integration_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/cli/fleet_integration_test.go b/internal/cli/fleet_integration_test.go index e1c5742..fbf7f08 100644 --- a/internal/cli/fleet_integration_test.go +++ b/internal/cli/fleet_integration_test.go @@ -905,7 +905,7 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() // Versioning needs erasure coding: a second MinIO on four directories. const port = "19101" endpoint := "http://127.0.0.1:" + port - f.sh("docker rm -fv teploy-it-minio2 >/dev/null 2>&1; docker run -d --name teploy-it-minio2 --memory 512m -p 127.0.0.1:" + port + ":9000 -e MINIO_ROOT_USER=" + itAK + " -e MINIO_ROOT_PASSWORD=" + itSK + " cgr.dev/chainguard/minio:latest server /tmp/d{1...4} >/dev/null") + f.sh("docker rm -fv teploy-it-minio2 >/dev/null 2>&1; docker run -d --name teploy-it-minio2 --memory 512m -p 127.0.0.1:" + port + ":9000 -e MINIO_CI_CD=1 -e MINIO_ROOT_USER=" + itAK + " -e MINIO_ROOT_PASSWORD=" + itSK + " cgr.dev/chainguard/minio:latest server /tmp/d{1...4} >/dev/null") defer f.sh("docker rm -fv teploy-it-minio2 >/dev/null 2>&1; true") aws := "AWS_ACCESS_KEY_ID=" + itAK + " AWS_SECRET_ACCESS_KEY=" + itSK + " aws --region us-east-1 --endpoint-url " + endpoint + " " f.waitFor("the second minio", aws+"s3 mb s3://itv") @@ -1582,6 +1582,8 @@ http.server.HTTPServer(("127.0.0.1",` + itHookPort + `),H).serve_forever() t.Fatalf("--all: exit %d\n%s%s", res.ExitCode, res.Stdout, res.Stderr) } // Back to the user with sudo: rotation lands where reads look, no second copy. + // (Earlier scenarios ran with no /etc/teploy, so this user may hold its own.) + f.sh(`rm -f "$HOME"/.config/teploy/backup-policy.json`) rotated := strings.TrimSpace(strings.Split(strings.SplitN(func() string { c := exec.Command(f.local, "backup", "keygen") b, err := c.Output() From 2556dc8c30659bd8bb84d76bc636878a5bb34876 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:33:37 -0700 Subject: [PATCH 16/17] docs(audit): record the re-review's integration results Co-Authored-By: Claude Opus 5.5 --- AUDIT_OPEN.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index daac72f..345ed8a 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -39,14 +39,16 @@ multipart abort, TEPLOY_BACKUP_TMPDIR workspaces invisible, postgresql:// names, holder's children, self-referential checksums above 256 MiB, safety copies prunable. Verified live (colima, MinIO via the real aws CLI, real age, the real linux binary): -`go test -tags integration -run TestFleetIntegration ./internal/cli`, 23 scenarios, on real +`go test -tags integration -run TestFleetIntegration ./internal/cli`, 25 scenarios, on real multi-database accessories; `TEPLOY_IT_INSTALL_AWS=1 go test -tags integration -run TestAWSCLIInstallScriptLive ./internal/backup` installs AWS's bundle for real and refuses a wrong -pin. Only unit-tested: the deploy hook, healthchecks.io's /fail form, the janitors' age thresholds -and multipart abort, the space preflights' refusal, a bucket refusing deletes, a store without -upload checksums, a musl host, the clock checks and an empty machine-id, setup's stage itself -(the script it runs is the live one), the /etc/teploy root path (the fixture is non-root; the -non-root paths, both with sudo and without, ran live). +pin. Only unit-tested: the deploy hook, healthchecks.io's /fail form, the janitors' thresholds +(the expiry check ran in a real shell, not on a live run) and multipart abort, the space preflights' +refusal, a bucket refusing deletes, a store without upload checksums, a musl host, the clock checks +and an empty machine-id, an old aws CLI on a server and an old distribution package being replaced, +a previous prefix that does not answer, the stuck-holder kill order (the kill itself ran live), +setup's stage itself (the script it runs is the live one), the /etc/teploy root path (the fixture +is non-root; the non-root paths, both with sudo and without, ran live). Regressions: `go test -tags integration ./internal/backup`, the full engine matrix after merging v0.1.51 (postgres, pgvector, timescale, mysql, mariadb, mongo, redis, valkey, dragonfly). The DR fixture still needs a 022 umask. From 7b244dfd25077ed209d4169be556d8a08e219181 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:35:13 -0700 Subject: [PATCH 17/17] fix(backup): a future-dated object no longer shows a target covered Co-Authored-By: Claude Opus 5.5 --- AUDIT_OPEN.md | 2 -- internal/backup/fleet_test.go | 18 ++++++++++++++++++ internal/backup/status.go | 7 ++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/AUDIT_OPEN.md b/AUDIT_OPEN.md index 345ed8a..15dffac 100644 --- a/AUDIT_OPEN.md +++ b/AUDIT_OPEN.md @@ -78,8 +78,6 @@ copy, and Object Lock with aws 2.23+. The real-B2 smoke test is the coordinator' Open / deferred: - The run lock falls back per user when the policy's directory is not writable, so a root run and a sudoer's run do not exclude each other. Only one of them has the cron line. -- `backup status` can still read a future-dated object as the newest good one, so one stray - object can show a target covered. Prune no longer trusts it; status has not been changed. - Legacy objects from a server that has not upgraded yet carry no marker. A first prune in a shared root by an upgraded server cannot tell them from its own: only markers from the moment each server upgrades protect it. Per-server prefixes close this (decided: Dash always diff --git a/internal/backup/fleet_test.go b/internal/backup/fleet_test.go index 7cade47..fb617f8 100644 --- a/internal/backup/fleet_test.go +++ b/internal/backup/fleet_test.go @@ -513,6 +513,24 @@ func TestUnsupportedDatabasesAreReported(t *testing.T) { } // Covered needs a restorable object now, from what covers the entry. +func TestStatusIgnoresFutureDatedObjects(t *testing.T) { + var e StatusEntry + future := time.Now().Add(365 * 24 * time.Hour).UTC().Format(time.RFC3339) + past := time.Now().Add(-time.Hour).UTC().Format(time.RFC3339) + fillEntryObjects(&e, []ListedBackup{ + {Kind: KindAccessory, Accessory: "db", Restorable: true, ID: "stray", CreatedAt: future}, + {Kind: KindAccessory, Accessory: "db", Restorable: true, ID: "real", CreatedAt: past}, + }, "db") + if e.Newest == nil || e.Newest.ID != "real" { + t.Fatalf("newest = %+v, want the real backup", e.Newest) + } + var only StatusEntry + fillEntryObjects(&only, []ListedBackup{{Kind: KindAccessory, Accessory: "db", Restorable: true, ID: "stray", CreatedAt: future}}, "db") + if only.Newest != nil { + t.Fatalf("a future-dated object alone counted as coverage: %+v", only.Newest) + } +} + func TestStatusCoverageNeedsAnObject(t *testing.T) { labels := map[string]string{"teploy.role": "accessory", "teploy.app": "a", "teploy.accessory": "db"} now := mustTime(t, "20260201-120000") diff --git a/internal/backup/status.go b/internal/backup/status.go index bc34bae..9fadfd2 100644 --- a/internal/backup/status.go +++ b/internal/backup/status.go @@ -598,9 +598,11 @@ func fillEntryEvents(e *StatusEntry, evs []events.Event, name string) { } // fillEntryObjects sets the newest object of each series from a listing (newest -// first). Below-floor objects are not Restorable and never count. +// first). Below-floor objects are not Restorable and never count, and neither do +// future-dated ones: as in prune, one stray object must not show a target covered. func fillEntryObjects(e *StatusEntry, rows []ListedBackup, name string) { seen := map[string]bool{} + now := time.Now() for _, r := range rows { switch { case name == "" && r.Kind != KindVolumes: @@ -610,6 +612,9 @@ func fillEntryObjects(e *StatusEntry, rows []ListedBackup, name string) { case !r.Restorable: continue } + if t, err := time.Parse(time.RFC3339, r.CreatedAt); err == nil && t.Sub(now) > maxClockSkew { + continue + } if seen[r.Database] { continue }