From dee408ec4a65c1c1df0a2a0cff6b71904954a9c4 Mon Sep 17 00:00:00 2001
From: Ameen Rashad <7609895+vazra@users.noreply.github.com>
Date: Sun, 11 Oct 2026 02:35:57 -0700
Subject: [PATCH 1/5] fix: keep app sidecars on DB loss so per-app config is
 recovered

PruneOrphanSidecars ran on boot before the reconciler re-created app rows,
so on an empty DB every app's simpledeploy.yml was deleted. A sidecar is now
an orphan only when its folder has no docker-compose.yml, no app row and no
tombstone, and nothing is pruned while the DB has no apps.

ReconcileDBFromFS skips apps without a row (the reconciler imports them).
A DR import of config.yml marks the first-boot seed done so a failed import
can no longer overwrite config.yml/secrets.yml from the partial DB.
---
 internal/configsync/configsync.go    |  49 +++-
 internal/configsync/firstboot.go     |   8 +-
 internal/configsync/recovery_test.go | 319 +++++++++++++++++++++++++++
 internal/gitsync/gitsync_test.go     |   5 +-
 4 files changed, 373 insertions(+), 8 deletions(-)
 create mode 100644 internal/configsync/recovery_test.go

diff --git a/internal/configsync/configsync.go b/internal/configsync/configsync.go
index ca6043ee..25b18fee 100644
--- a/internal/configsync/configsync.go
+++ b/internal/configsync/configsync.go
@@ -26,6 +26,7 @@ import (
 
 const (
 	appSidecarName        = "simpledeploy.yml"
+	composeFileName       = "docker-compose.yml"
 	globalSidecar         = "config.yml"
 	redactedGlobalSidecar = "_global.yml"
 	debounceDelay         = 500 * time.Millisecond
@@ -260,7 +261,12 @@ func (s *Syncer) DeleteAppSidecar(slug string) error {
 	return nil
 }
 
-// PruneOrphanSidecars removes per-app sidecar files for slugs not present in the DB.
+// PruneOrphanSidecars removes per-app sidecar files from folders that are no
+// longer apps: no docker-compose.yml, no app row (active or archived) and no
+// archive tombstone. A folder with a compose file is never an orphan, DB row
+// or not: after a DB loss the reconciler re-creates the row from it and
+// imports the sidecar. Nothing is pruned while the DB has no apps (fresh or
+// wiped DB), or when a check fails; a stale file is safer than lost config.
 // Safe to call on every startup. Returns the list of slugs whose sidecars were removed.
 // Skips hidden dirs and names starting with '_' or '.'.
 func (s *Syncer) PruneOrphanSidecars() ([]string, error) {
@@ -272,6 +278,18 @@ func (s *Syncer) PruneOrphanSidecars() ([]string, error) {
 		return nil, fmt.Errorf("PruneOrphanSidecars: readdir %s: %w", s.appsDir, err)
 	}
 
+	apps, err := s.store.ListAppsWithOptions(store.ListAppsOptions{IncludeArchived: true})
+	if err != nil {
+		return nil, fmt.Errorf("PruneOrphanSidecars: list apps: %w", err)
+	}
+	if len(apps) == 0 {
+		return nil, nil
+	}
+	known := make(map[string]bool, len(apps))
+	for _, a := range apps {
+		known[a.Slug] = true
+	}
+
 	var removed []string
 	for _, e := range entries {
 		if !e.IsDir() {
@@ -285,12 +303,13 @@ func (s *Syncer) PruneOrphanSidecars() ([]string, error) {
 		if _, err := os.Stat(sidecar); os.IsNotExist(err) {
 			continue
 		}
-		_, err := s.store.GetAppBySlug(name)
-		if err == nil {
-			// App exists in DB; keep sidecar.
+		// Keep it when the DB knows the app, the folder still has a compose
+		// file, or a tombstone will restore the archived row later in boot.
+		if known[name] ||
+			!pathAbsent(filepath.Join(s.appsDir, name, composeFileName)) ||
+			!pathAbsent(filepath.Join(s.ArchiveDir(), name+".yml")) {
 			continue
 		}
-		// Not in DB; remove sidecar.
 		if err := os.Remove(sidecar); err != nil && !os.IsNotExist(err) {
 			return removed, fmt.Errorf("PruneOrphanSidecars: remove %s: %w", sidecar, err)
 		}
@@ -303,6 +322,13 @@ func (s *Syncer) PruneOrphanSidecars() ([]string, error) {
 	return removed, nil
 }
 
+// pathAbsent reports whether nothing (not even a dangling symlink) exists at
+// path. Any error other than not-exist counts as present.
+func pathAbsent(path string) bool {
+	_, err := os.Lstat(path)
+	return os.IsNotExist(err)
+}
+
 // WriteAppSidecar reads the app and its config from the store and writes the
 // sidecar pair atomically. Secrets file (mode 0600) is written FIRST; if that
 // fails, the non-secret sidecar is not written. Non-secret sidecar is mode 0644.
@@ -873,6 +899,8 @@ func (s *Syncer) ImportGlobal(data *GlobalSidecar) error {
 // ImportGlobalIfEmpty imports the global sidecar only when the DB has zero users.
 // Returns (imported bool, err error). If no sidecar exists -> (false, nil).
 // If users table is non-empty -> (false, nil) without modifying DB.
+// When it imports, it also records the first-boot seed as done (see
+// RunFirstBootSeedIfNeeded), even if the import then fails.
 func (s *Syncer) ImportGlobalIfEmpty() (bool, error) {
 	users, err := s.store.ListUsers()
 	if err != nil {
@@ -882,6 +910,17 @@ func (s *Syncer) ImportGlobalIfEmpty() (bool, error) {
 		return false, nil
 	}
 
+	if pathAbsent(filepath.Join(s.dataDir, globalSidecar)) {
+		return false, nil
+	}
+	// The DB is rebuilt from the files on disk, so they already are the
+	// source of truth. Mark the first-boot seed done before importing: a
+	// failed or partial import must never let RunFirstBootSeedIfNeeded
+	// overwrite config.yml and secrets.yml with the incomplete DB.
+	if err := s.store.SetMeta(fsSeededKey, time.Now().UTC().Format(time.RFC3339)); err != nil {
+		return false, fmt.Errorf("ImportGlobalIfEmpty: mark seeded: %w", err)
+	}
+
 	data, err := s.ReadGlobal()
 	if err != nil {
 		return false, fmt.Errorf("ImportGlobalIfEmpty: read global: %w", err)
diff --git a/internal/configsync/firstboot.go b/internal/configsync/firstboot.go
index db2726e2..2cd3db52 100644
--- a/internal/configsync/firstboot.go
+++ b/internal/configsync/firstboot.go
@@ -99,7 +99,9 @@ func ensureGitignore(dir string, lines []string) error {
 
 // ReconcileDBFromFS scans the apps directory and applies each per-app sidecar
 // to the DB, then applies the global sidecar. Missing files are skipped (not
-// treated as deletions) so this is safe to run on every boot.
+// treated as deletions) so this is safe to run on every boot. Apps with no DB
+// row (e.g. after a DB loss) are left alone: the reconciler re-creates the
+// row from the app folder and then imports the sidecar.
 func (s *Syncer) ReconcileDBFromFS(ctx context.Context) error {
 	entries, err := os.ReadDir(s.appsDir)
 	if err != nil {
@@ -123,6 +125,10 @@ func (s *Syncer) ReconcileDBFromFS(ctx context.Context) error {
 			if loaded == nil || loaded.Sidecar == nil {
 				continue
 			}
+			if _, err := s.store.GetAppBySlug(name); err != nil {
+				log.Printf("[fs-auth] %s: no app row yet; its sidecar is imported once the reconciler adds the app", name)
+				continue
+			}
 			if err := s.ApplyAppSidecar(name, loaded); err != nil {
 				log.Printf("[fs-auth] apply %s: %v", name, err)
 			}
diff --git a/internal/configsync/recovery_test.go b/internal/configsync/recovery_test.go
new file mode 100644
index 00000000..5534299b
--- /dev/null
+++ b/internal/configsync/recovery_test.go
@@ -0,0 +1,319 @@
+package configsync
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/vazra/simpledeploy/internal/store"
+)
+
+// drInstall is a live install's on-disk state: apps_dir with app folders and
+// sidecars, data_dir with config.yml + secrets.yml and the SQLite DB.
+type drInstall struct {
+	appsDir, dataDir string
+}
+
+func (d drInstall) dbPath() string { return filepath.Join(d.dataDir, "simpledeploy.db") }
+
+func (d drInstall) open(t *testing.T) *store.Store {
+	t.Helper()
+	st, err := store.Open(d.dbPath())
+	if err != nil {
+		t.Fatalf("open store: %v", err)
+	}
+	t.Cleanup(func() { st.Close() })
+	return st
+}
+
+// wipeDB deletes the DB files, as in the disaster-recovery procedure.
+func (d drInstall) wipeDB(t *testing.T) {
+	t.Helper()
+	for _, f := range []string{"simpledeploy.db", "simpledeploy.db-wal", "simpledeploy.db-shm"} {
+		if err := os.Remove(filepath.Join(d.dataDir, f)); err != nil && !os.IsNotExist(err) {
+			t.Fatal(err)
+		}
+	}
+}
+
+// seedDRInstall builds an install with app "web" (compose file, alert rule,
+// backup config with an encrypted target, access grant for "editor"), runs
+// the first-boot seed so every sidecar is on disk, then closes the DB.
+func seedDRInstall(t *testing.T) drInstall {
+	t.Helper()
+	d := drInstall{appsDir: t.TempDir(), dataDir: t.TempDir()}
+	st, err := store.Open(d.dbPath())
+	if err != nil {
+		t.Fatalf("open store: %v", err)
+	}
+	defer st.Close()
+
+	writeCompose(t, d.appsDir, "web")
+	app := &store.App{Name: "web", Slug: "web", ComposePath: filepath.Join(d.appsDir, "web", composeFileName), Status: "running"}
+	if err := st.UpsertApp(app, nil); err != nil {
+		t.Fatal(err)
+	}
+	wh := &store.Webhook{Name: "ops", Type: "slack", URL: "https://hooks.example.com/ops"}
+	if err := st.CreateWebhook(wh); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := st.CreateUser("admin", "$2a$10$admin", "super_admin", "", ""); err != nil {
+		t.Fatal(err)
+	}
+	editor, err := st.CreateUser("editor", "$2a$10$editor", "manage", "", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := st.GrantAppAccess(editor.ID, app.ID); err != nil {
+		t.Fatal(err)
+	}
+	if err := st.CreateAlertRule(&store.AlertRule{AppID: &app.ID, Metric: "cpu_pct", Operator: ">", Threshold: 90, DurationSec: 300, WebhookID: wh.ID, Enabled: true}); err != nil {
+		t.Fatal(err)
+	}
+	if err := st.CreateBackupConfig(&store.BackupConfig{AppID: app.ID, Strategy: "volume", Target: "s3", ScheduleCron: "0 3 * * *", TargetConfigJSON: "enc:target", RetentionMode: "count", RetentionCount: 7}); err != nil {
+		t.Fatal(err)
+	}
+	if err := RunFirstBootSeedIfNeeded(context.Background(), st, New(st, d.appsDir, d.dataDir), nil); err != nil {
+		t.Fatalf("seed: %v", err)
+	}
+	return d
+}
+
+func writeCompose(t *testing.T, appsDir, slug string) {
+	t.Helper()
+	dir := filepath.Join(appsDir, slug)
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, composeFileName), []byte("services:\n  web:\n    image: nginx\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func writeSidecarFile(t *testing.T, appsDir, slug string) string {
+	t.Helper()
+	dir := filepath.Join(appsDir, slug)
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	p := filepath.Join(dir, appSidecarName)
+	if err := os.WriteFile(p, []byte("version: 1\napp:\n  slug: "+slug+"\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	return p
+}
+
+func assertWebRestored(t *testing.T, st *store.Store) {
+	t.Helper()
+	app, err := st.GetAppBySlug("web")
+	if err != nil {
+		t.Fatalf("app row: %v", err)
+	}
+	rules, err := st.ListAlertRules(&app.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(rules) != 1 || rules[0].Metric != "cpu_pct" || rules[0].Threshold != 90 || rules[0].DurationSec != 300 {
+		t.Errorf("alert rules not restored: %+v", rules)
+	}
+	backups, err := st.ListBackupConfigs(&app.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(backups) != 1 || backups[0].Target != "s3" || backups[0].TargetConfigJSON != "enc:target" {
+		t.Errorf("backup configs not restored: %+v", backups)
+	}
+	access, err := st.ListAccessForApp(app.ID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !slices.Equal(access, []string{"editor"}) {
+		t.Errorf("access grants not restored: %v", access)
+	}
+}
+
+// TestDRBootRecoversAppSidecars runs the serve boot steps on a wiped DB:
+// the app sidecars must survive until the reconciler re-creates the app
+// row, and the per-app config must then be imported from them.
+func TestDRBootRecoversAppSidecars(t *testing.T) {
+	d := seedDRInstall(t)
+	d.wipeDB(t)
+
+	st := d.open(t)
+	syncer := New(st, d.appsDir, d.dataDir)
+	t.Cleanup(func() { _ = syncer.Close() })
+
+	if imported, err := syncer.ImportGlobalIfEmpty(); err != nil || !imported {
+		t.Fatalf("ImportGlobalIfEmpty = %v, %v", imported, err)
+	}
+	pruned, err := syncer.PruneOrphanSidecars()
+	if err != nil {
+		t.Fatalf("PruneOrphanSidecars: %v", err)
+	}
+	if len(pruned) != 0 {
+		t.Errorf("pruned %v; an app folder with a compose file is never an orphan", pruned)
+	}
+	if err := RunFirstBootSeedIfNeeded(context.Background(), st, syncer, nil); err != nil {
+		t.Fatalf("seed: %v", err)
+	}
+	if err := syncer.ReconcileDBFromFS(context.Background()); err != nil {
+		t.Fatalf("ReconcileDBFromFS: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(d.appsDir, "web", appSidecarName)); err != nil {
+		t.Fatalf("app sidecar gone before the reconciler ran: %v", err)
+	}
+
+	// What the reconciler does for an app folder with no DB row.
+	if err := st.UpsertApp(&store.App{Name: "web", Slug: "web", ComposePath: filepath.Join(d.appsDir, "web", composeFileName), Status: "running"}, nil); err != nil {
+		t.Fatal(err)
+	}
+	if imported, err := syncer.ImportAppSidecarIfMissing("web"); err != nil || !imported {
+		t.Fatalf("ImportAppSidecarIfMissing = %v, %v", imported, err)
+	}
+	assertWebRestored(t, st)
+}
+
+func TestPruneOrphanSidecarsKeepsAppFolders(t *testing.T) {
+	st := openTestStore(t)
+	appsDir := t.TempDir()
+	dataDir := t.TempDir()
+	syncer := New(st, appsDir, dataDir)
+
+	// One live app so the DB is not empty.
+	writeCompose(t, appsDir, "alive")
+	if err := st.UpsertApp(&store.App{Name: "alive", Slug: "alive", ComposePath: filepath.Join(appsDir, "alive", composeFileName), Status: "running"}, nil); err != nil {
+		t.Fatal(err)
+	}
+	alive := writeSidecarFile(t, appsDir, "alive")
+
+	// App folder with a compose file but no DB row (e.g. lost with the DB).
+	writeCompose(t, appsDir, "lost")
+	lost := writeSidecarFile(t, appsDir, "lost")
+
+	// Compose file present as a dangling symlink: still an app folder.
+	linked := writeSidecarFile(t, appsDir, "linked")
+	if err := os.Symlink(filepath.Join(appsDir, "missing.yml"), filepath.Join(appsDir, "linked", composeFileName)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Archived app (compose removed, tombstone written), row lost with the DB.
+	archived := writeSidecarFile(t, appsDir, "archived")
+	if err := syncer.writeTombstoneFile("archived", &Tombstone{Version: Version, ArchivedAt: time.Now().UTC(), App: AppMeta{Slug: "archived"}}); err != nil {
+		t.Fatal(err)
+	}
+
+	// Archived app whose row is still in the DB.
+	archivedRow := writeSidecarFile(t, appsDir, "archived-row")
+	if err := st.UpsertApp(&store.App{Name: "archived-row", Slug: "archived-row", Status: "stopped"}, nil); err != nil {
+		t.Fatal(err)
+	}
+	if err := st.MarkAppArchived("archived-row", time.Now()); err != nil {
+		t.Fatal(err)
+	}
+
+	// Real orphan: sidecar only.
+	ghost := writeSidecarFile(t, appsDir, "ghost")
+
+	pruned, err := syncer.PruneOrphanSidecars()
+	if err != nil {
+		t.Fatalf("PruneOrphanSidecars: %v", err)
+	}
+	if !slices.Equal(pruned, []string{"ghost"}) {
+		t.Errorf("pruned = %v, want [ghost]", pruned)
+	}
+	for _, p := range []string{alive, lost, linked, archived, archivedRow} {
+		if _, err := os.Stat(p); err != nil {
+			t.Errorf("%s removed: %v", p, err)
+		}
+	}
+	if _, err := os.Stat(ghost); !os.IsNotExist(err) {
+		t.Errorf("orphan %s kept", ghost)
+	}
+}
+
+// TestPruneOrphanSidecarsEmptyDB: a fresh or wiped DB cannot tell an orphan
+// from an app it lost, so nothing is pruned.
+func TestPruneOrphanSidecarsEmptyDB(t *testing.T) {
+	st := openTestStore(t)
+	appsDir := t.TempDir()
+	syncer := New(st, appsDir, t.TempDir())
+	ghost := writeSidecarFile(t, appsDir, "ghost")
+
+	pruned, err := syncer.PruneOrphanSidecars()
+	if err != nil || len(pruned) != 0 {
+		t.Fatalf("PruneOrphanSidecars = %v, %v; want nothing pruned", pruned, err)
+	}
+	if _, err := os.Stat(ghost); err != nil {
+		t.Errorf("sidecar removed on an empty DB: %v", err)
+	}
+}
+
+// TestReconcileDBFromFSSkipsAppsWithoutRow: boot-time reload leaves sidecars
+// of apps the DB does not know yet for the reconciler to import, and does
+// not create rows itself.
+func TestReconcileDBFromFSSkipsAppsWithoutRow(t *testing.T) {
+	d := seedDRInstall(t)
+	d.wipeDB(t)
+	st := d.open(t)
+	syncer := New(st, d.appsDir, d.dataDir)
+	if _, err := syncer.ImportGlobalIfEmpty(); err != nil {
+		t.Fatal(err)
+	}
+	if err := syncer.ReconcileDBFromFS(context.Background()); err != nil {
+		t.Fatalf("ReconcileDBFromFS: %v", err)
+	}
+	if _, err := st.GetAppBySlug("web"); err == nil {
+		t.Error("ReconcileDBFromFS created an app row; the reconciler owns that")
+	}
+	if _, err := os.Stat(filepath.Join(d.appsDir, "web", appSidecarName)); err != nil {
+		t.Errorf("sidecar removed: %v", err)
+	}
+}
+
+// TestDRImportFailureKeepsGlobalSidecar: when importing config.yml into an
+// empty DB fails part way, the first-boot seed must not overwrite config.yml
+// and secrets.yml with the incomplete DB.
+func TestDRImportFailureKeepsGlobalSidecar(t *testing.T) {
+	st := openTestStore(t)
+	dataDir := t.TempDir()
+	syncer := New(st, t.TempDir(), dataDir)
+
+	// The API key's user is missing, so ImportGlobal fails after importing
+	// users and before registries.
+	sidecar := GlobalSidecar{
+		Version:    Version,
+		Users:      []UserEntry{{Username: "admin", Role: "super_admin"}},
+		APIKeys:    []APIKeyEntry{{Username: "gone", Name: "ci"}},
+		Registries: []RegistryEntry{{ID: "reg-1", Name: "ghcr", URL: "ghcr.io"}},
+	}
+	if err := atomicWriteYAMLMode(filepath.Join(dataDir, globalSidecar), 0644, sidecar); err != nil {
+		t.Fatal(err)
+	}
+	if err := syncer.WriteGlobalSecrets(&GlobalSecrets{
+		Version: Version,
+		Users:   []UserSecretsEntry{{Username: "admin", PasswordHash: "$2a$10$admin"}},
+	}); err != nil {
+		t.Fatal(err)
+	}
+
+	if _, err := syncer.ImportGlobalIfEmpty(); err == nil {
+		t.Fatal("expected ImportGlobalIfEmpty to fail on the dangling API key")
+	}
+	if err := RunFirstBootSeedIfNeeded(context.Background(), st, syncer, nil); err != nil {
+		t.Fatalf("seed: %v", err)
+	}
+
+	raw, err := os.ReadFile(filepath.Join(dataDir, globalSidecar))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, want := range []string{"name: ghcr", "username: gone"} {
+		if !strings.Contains(string(raw), want) {
+			t.Errorf("config.yml lost %q after a failed import:\n%s", want, raw)
+		}
+	}
+}
diff --git a/internal/gitsync/gitsync_test.go b/internal/gitsync/gitsync_test.go
index f8a7daea..a4764ee0 100644
--- a/internal/gitsync/gitsync_test.go
+++ b/internal/gitsync/gitsync_test.go
@@ -917,9 +917,10 @@ func TestPruneOrphanSidecarsCommitsToRemote(t *testing.T) {
 	appsDir := makeAppsDir(t)
 	bareDir := makeBareRemote(t)
 
-	// Three app dirs on disk.
+	// Three dirs on disk; only "alive" is still an app folder (an orphan has
+	// no compose file, a folder with one is never pruned).
+	writeFile(t, filepath.Join(appsDir, "alive", "docker-compose.yml"), "services:\n  web:\n    image: nginx\n")
 	for _, slug := range []string{"orphan1", "orphan2", "alive"} {
-		writeFile(t, filepath.Join(appsDir, slug, "docker-compose.yml"), "services:\n  web:\n    image: nginx\n")
 		writeFile(t, filepath.Join(appsDir, slug, "simpledeploy.yml"), "version: 1\napp:\n  slug: "+slug+"\n")
 	}
 

From 8225936734782e3e1cb02304d9ceed8d3e7b4590 Mon Sep 17 00:00:00 2001
From: Ameen Rashad <7609895+vazra@users.noreply.github.com>
Date: Sun, 11 Oct 2026 02:37:05 -0700
Subject: [PATCH 2/5] test: reconciler recovers app config after DB wipe with
 boot-time prune

---
 internal/reconciler/recovery_test.go | 135 +++++++++++++++++++++++++++
 1 file changed, 135 insertions(+)

diff --git a/internal/reconciler/recovery_test.go b/internal/reconciler/recovery_test.go
index a60d800d..40fa128e 100644
--- a/internal/reconciler/recovery_test.go
+++ b/internal/reconciler/recovery_test.go
@@ -193,3 +193,138 @@ func TestReconcilerArchivesOnComposeDelete(t *testing.T) {
 		t.Fatalf("expected tombstone at %s: %v", tombPath, err)
 	}
 }
+
+// TestReconcileDRAfterBootPrune runs a live install, wipes its DB, then the
+// serve boot order (global import, orphan prune, seed, FS reload, initial
+// Reconcile). The app's alert rules, backup configs and access grants must
+// come back from simpledeploy.yml, no sidecar may be deleted, and archived
+// apps must stay archived.
+func TestReconcileDRAfterBootPrune(t *testing.T) {
+	ctx := context.Background()
+	appsDir := t.TempDir()
+	dataDir := t.TempDir()
+	dbPath := filepath.Join(dataDir, "simpledeploy.db")
+	for _, slug := range []string{"web", "gone", "nocompose"} {
+		writeComposeFile(t, appsDir, slug)
+	}
+
+	// Live install.
+	st1, err := store.Open(dbPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cs1 := configsync.New(st1, appsDir, dataDir)
+	r1 := New(st1, &mockDeployer{}, nil, appsDir, nil, cs1)
+	if err := r1.Reconcile(ctx); err != nil {
+		t.Fatal(err)
+	}
+	web, err := st1.GetAppBySlug("web")
+	if err != nil {
+		t.Fatal(err)
+	}
+	wh := &store.Webhook{Name: "ops", Type: "slack", URL: "https://hooks.example.com/ops"}
+	if err := st1.CreateWebhook(wh); err != nil {
+		t.Fatal(err)
+	}
+	editor, err := st1.CreateUser("editor", "$2a$10$editor", "manage", "", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := st1.GrantAppAccess(editor.ID, web.ID); err != nil {
+		t.Fatal(err)
+	}
+	if err := st1.CreateAlertRule(&store.AlertRule{AppID: &web.ID, Metric: "cpu_pct", Operator: ">", Threshold: 90, DurationSec: 300, WebhookID: wh.ID, Enabled: true}); err != nil {
+		t.Fatal(err)
+	}
+	if err := st1.CreateBackupConfig(&store.BackupConfig{AppID: web.ID, Strategy: "volume", Target: "s3", ScheduleCron: "0 3 * * *", TargetConfigJSON: "enc:target", RetentionMode: "count", RetentionCount: 7}); err != nil {
+		t.Fatal(err)
+	}
+	if err := configsync.RunFirstBootSeedIfNeeded(ctx, st1, cs1, nil); err != nil {
+		t.Fatal(err)
+	}
+	// Archive two apps: one folder removed, one with only its compose removed.
+	if err := os.RemoveAll(filepath.Join(appsDir, "gone")); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Remove(filepath.Join(appsDir, "nocompose", "docker-compose.yml")); err != nil {
+		t.Fatal(err)
+	}
+	if err := r1.Reconcile(ctx); err != nil {
+		t.Fatal(err)
+	}
+	_ = cs1.Close()
+	st1.Close()
+
+	// Disaster: the DB is lost; data_dir and apps_dir survive.
+	for _, f := range []string{"simpledeploy.db", "simpledeploy.db-wal", "simpledeploy.db-shm"} {
+		if err := os.Remove(filepath.Join(dataDir, f)); err != nil && !os.IsNotExist(err) {
+			t.Fatal(err)
+		}
+	}
+
+	st, err := store.Open(dbPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { st.Close() })
+	cs := configsync.New(st, appsDir, dataDir)
+	if imported, err := cs.ImportGlobalIfEmpty(); err != nil || !imported {
+		t.Fatalf("ImportGlobalIfEmpty = %v, %v", imported, err)
+	}
+	if pruned, err := cs.PruneOrphanSidecars(); err != nil || len(pruned) != 0 {
+		t.Fatalf("PruneOrphanSidecars = %v, %v; want nothing pruned", pruned, err)
+	}
+	st.SetMutationHook(func(scope store.MutationScope, slug string) {
+		if scope == store.ScopeApp && slug != "" {
+			cs.ScheduleAppWrite(slug)
+		}
+	})
+	if err := configsync.RunFirstBootSeedIfNeeded(ctx, st, cs, nil); err != nil {
+		t.Fatal(err)
+	}
+	if err := cs.ReconcileDBFromFS(ctx); err != nil {
+		t.Fatal(err)
+	}
+	mock := &mockDeployer{}
+	if err := New(st, mock, nil, appsDir, nil, cs).Reconcile(ctx); err != nil {
+		t.Fatal(err)
+	}
+	if err := cs.Close(); err != nil { // flush debounced sidecar writes
+		t.Fatal(err)
+	}
+
+	web, err = st.GetAppBySlug("web")
+	if err != nil {
+		t.Fatalf("web not re-created: %v", err)
+	}
+	if web.ArchivedAt.Valid {
+		t.Error("web archived after recovery")
+	}
+	rules, _ := st.ListAlertRules(&web.ID)
+	if len(rules) != 1 || rules[0].Metric != "cpu_pct" || rules[0].Threshold != 90 {
+		t.Errorf("alert rules not restored: %+v", rules)
+	}
+	backups, _ := st.ListBackupConfigs(&web.ID)
+	if len(backups) != 1 || backups[0].Target != "s3" || backups[0].TargetConfigJSON != "enc:target" {
+		t.Errorf("backup configs not restored: %+v", backups)
+	}
+	if access, _ := st.ListAccessForApp(web.ID); len(access) != 1 || access[0] != "editor" {
+		t.Errorf("access grants not restored: %v", access)
+	}
+	sc, err := cs.ReadAppSidecar("web")
+	if err != nil || sc == nil || len(sc.AlertRules) != 1 || len(sc.BackupConfigs) != 1 || len(sc.Access) != 1 {
+		t.Errorf("web sidecar after recovery = %+v, %v", sc, err)
+	}
+	if _, err := os.Stat(filepath.Join(appsDir, "nocompose", "simpledeploy.yml")); err != nil {
+		t.Errorf("archived app's sidecar removed: %v", err)
+	}
+	for _, slug := range []string{"gone", "nocompose"} {
+		a, err := st.GetAppBySlug(slug)
+		if err != nil || !a.ArchivedAt.Valid {
+			t.Errorf("%s not restored as archived: %+v, %v", slug, a, err)
+		}
+		if mock.hasCall("Deploy:" + slug) {
+			t.Errorf("archived app %s redeployed", slug)
+		}
+	}
+}

From 0948f89afd05918e5b76026c464780a7b56d252f Mon Sep 17 00:00:00 2001
From: Ameen Rashad <7609895+vazra@users.noreply.github.com>
Date: Sun, 11 Oct 2026 02:39:21 -0700
Subject: [PATCH 3/5] test(cli): boot sequence restores sidecar config after DB
 wipe

Move serve's boot-time configsync steps into initConfigSync and
reloadConfigFromFS (same order) so a test can run them.
---
 cmd/simpledeploy/configboot.go      |  96 +++++++++++++++++
 cmd/simpledeploy/configboot_test.go | 158 ++++++++++++++++++++++++++++
 cmd/simpledeploy/main.go            |  69 +-----------
 3 files changed, 257 insertions(+), 66 deletions(-)
 create mode 100644 cmd/simpledeploy/configboot.go
 create mode 100644 cmd/simpledeploy/configboot_test.go

diff --git a/cmd/simpledeploy/configboot.go b/cmd/simpledeploy/configboot.go
new file mode 100644
index 00000000..83d37104
--- /dev/null
+++ b/cmd/simpledeploy/configboot.go
@@ -0,0 +1,96 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/vazra/simpledeploy/internal/config"
+	"github.com/vazra/simpledeploy/internal/configsync"
+	"github.com/vazra/simpledeploy/internal/store"
+)
+
+// initConfigSync runs the boot-time sidecar steps that come before git sync
+// and the reconciler: disaster-recovery import of config.yml into an empty
+// DB, orphan sidecar pruning, the mutation hook and the one-time backfill.
+// App sidecars are imported later, by the reconciler, once it re-creates
+// each app row from its folder.
+func initConfigSync(db *store.Store, syncer *configsync.Syncer, dataDir string) {
+	// Rehydrate global config if DB is empty (first boot after DR).
+	if imported, err := syncer.ImportGlobalIfEmpty(); err != nil {
+		log.Printf("[configsync] global import failed: %v", err)
+	} else if imported {
+		log.Printf("[configsync] imported global sidecar into empty DB")
+	}
+
+	// Prune sidecar files left in folders that are no longer apps (e.g. from
+	// prior delete cycles before this feature existed). App folders with a
+	// compose file keep theirs even without a DB row.
+	if pruned, err := syncer.PruneOrphanSidecars(); err != nil {
+		log.Printf("[configsync] prune orphan sidecars: %v", err)
+	} else if len(pruned) > 0 {
+		log.Printf("[configsync] pruned orphan sidecars for: %v", pruned)
+	}
+
+	// Install mutation hook so future mutations update sidecars.
+	// FS-authoritative state writes are eventually consistent: this hook fires
+	// after a successful DB commit and schedules a debounced FS write (500ms).
+	// If the FS write fails, the reconciler watcher (internal/reconciler) re-
+	// applies on the next file edit. Contract: DB success => FS reflects soon.
+	// See docs/operations/state-on-disk.md.
+	db.SetMutationHook(func(scope store.MutationScope, slug string) {
+		switch scope {
+		case store.ScopeApp:
+			if slug != "" {
+				syncer.ScheduleAppWrite(slug)
+			}
+		case store.ScopeGlobal:
+			syncer.ScheduleGlobalWrite()
+		}
+	})
+
+	// First-boot sidecar backfill: write sidecars for all existing DB state
+	// so that upgrades from pre-configsync installs get sidecars on first boot.
+	backfillMarker := filepath.Join(dataDir, ".configsync_backfill_v1")
+	if _, err := os.Stat(backfillMarker); os.IsNotExist(err) {
+		allApps, listErr := db.ListApps()
+		if listErr != nil {
+			log.Printf("[configsync] backfill: list apps failed: %v (skipping)", listErr)
+		} else {
+			allOK := true
+			if backfillErr := syncer.WriteGlobal(); backfillErr != nil {
+				log.Printf("[configsync] backfill: write global failed: %v", backfillErr)
+				allOK = false
+			}
+			for _, a := range allApps {
+				if werr := syncer.WriteAppSidecar(a.Slug); werr != nil {
+					log.Printf("[configsync] backfill: write sidecar for %s: %v", a.Slug, werr)
+					allOK = false
+				}
+			}
+			if allOK {
+				log.Printf("[configsync] first-boot sidecar backfill: wrote %d apps + global", len(allApps))
+				if werr := os.WriteFile(backfillMarker, nil, 0600); werr != nil {
+					log.Printf("[configsync] backfill: write marker failed: %v", werr)
+				}
+			}
+		}
+	}
+}
+
+// reloadConfigFromFS runs the one-time FS-authoritative seed, then applies
+// the sidecars on disk to the DB. Runs after git sync is wired (it decides
+// whether access lists are applied) and before the reconciler starts.
+func reloadConfigFromFS(ctx context.Context, db *store.Store, syncer *configsync.Syncer, cfg *config.Config) error {
+	// FS-authoritative seed (only on first boot post-upgrade).
+	if err := configsync.RunFirstBootSeedIfNeeded(ctx, db, syncer, cfg); err != nil {
+		return fmt.Errorf("fs-auth seed: %w", err)
+	}
+	// Reconcile DB cache from FS files (idempotent on every boot).
+	if err := syncer.ReconcileDBFromFS(ctx); err != nil {
+		return fmt.Errorf("fs-auth reload: %w", err)
+	}
+	return nil
+}
diff --git a/cmd/simpledeploy/configboot_test.go b/cmd/simpledeploy/configboot_test.go
new file mode 100644
index 00000000..c27bfee8
--- /dev/null
+++ b/cmd/simpledeploy/configboot_test.go
@@ -0,0 +1,158 @@
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/vazra/simpledeploy/internal/compose"
+	"github.com/vazra/simpledeploy/internal/config"
+	"github.com/vazra/simpledeploy/internal/configsync"
+	"github.com/vazra/simpledeploy/internal/deployer"
+	"github.com/vazra/simpledeploy/internal/reconciler"
+	"github.com/vazra/simpledeploy/internal/store"
+)
+
+// stubDeployer reports success for every action without running docker.
+type stubDeployer struct{}
+
+func (stubDeployer) Deploy(context.Context, *compose.AppConfig, ...deployer.RegistryAuth) deployer.DeployResult {
+	return deployer.DeployResult{}
+}
+func (stubDeployer) RollbackDeploy(context.Context, *compose.AppConfig, int, *int64, ...deployer.RegistryAuth) deployer.DeployResult {
+	return deployer.DeployResult{}
+}
+func (stubDeployer) Teardown(context.Context, string) error { return nil }
+func (stubDeployer) Restart(context.Context, *compose.AppConfig) deployer.DeployResult {
+	return deployer.DeployResult{}
+}
+func (stubDeployer) Stop(context.Context, string) error  { return nil }
+func (stubDeployer) Start(context.Context, string) error { return nil }
+func (stubDeployer) Pull(context.Context, *compose.AppConfig, []deployer.RegistryAuth) deployer.DeployResult {
+	return deployer.DeployResult{}
+}
+func (stubDeployer) Scale(context.Context, *compose.AppConfig, map[string]int) error { return nil }
+func (stubDeployer) Status(context.Context, string) ([]deployer.ServiceStatus, error) {
+	return nil, nil
+}
+func (stubDeployer) Cancel(context.Context, *compose.AppConfig) error { return nil }
+
+// bootServe runs the config steps of `serve` in boot order (no git sync)
+// up to and including the reconciler's initial pass. The returned func is
+// the graceful-shutdown flush.
+func bootServe(t *testing.T, cfg *config.Config) (*store.Store, func()) {
+	t.Helper()
+	ctx := context.Background()
+	db, err := store.Open(filepath.Join(cfg.DataDir, "simpledeploy.db"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	syncer := configsync.New(db, cfg.AppsDir, cfg.DataDir)
+	initConfigSync(db, syncer, cfg.DataDir)
+	if err := reloadConfigFromFS(ctx, db, syncer, cfg); err != nil {
+		t.Fatalf("reloadConfigFromFS: %v", err)
+	}
+	if err := reconciler.New(db, stubDeployer{}, nil, cfg.AppsDir, cfg, syncer).Reconcile(ctx); err != nil {
+		t.Fatalf("Reconcile: %v", err)
+	}
+	return db, func() {
+		if err := syncer.Close(); err != nil {
+			t.Errorf("syncer close: %v", err)
+		}
+		db.Close()
+	}
+}
+
+// TestServeBootRecoversConfigAfterDBWipe mirrors the disaster-recovery E2E:
+// configure an app, stop, delete the DB, start again. Global config comes
+// back from config.yml and the app's alert rules, backup configs and access
+// grants from its simpledeploy.yml, which must not be pruned.
+func TestServeBootRecoversConfigAfterDBWipe(t *testing.T) {
+	root := t.TempDir()
+	cfg := &config.Config{
+		DataDir:      filepath.Join(root, "data"),
+		AppsDir:      filepath.Join(root, "apps"),
+		MasterSecret: "test-secret-0123456789abcdef0123456789",
+	}
+	appDir := filepath.Join(cfg.AppsDir, "dr-nginx")
+	for _, d := range []string{cfg.DataDir, appDir} {
+		if err := os.MkdirAll(d, 0o755); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := os.WriteFile(filepath.Join(appDir, "docker-compose.yml"), []byte("services:\n  web:\n    image: nginx:alpine\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	// First run: the reconciler adopts the app, then it is configured.
+	db, shutdown := bootServe(t, cfg)
+	app, err := db.GetAppBySlug("dr-nginx")
+	if err != nil {
+		t.Fatalf("app not adopted: %v", err)
+	}
+	wh := &store.Webhook{Name: "dr-hook", Type: "slack", URL: "http://127.0.0.1:9/dr-hook"}
+	if err := db.CreateWebhook(wh); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.CreateUser("dr-admin", "$2a$10$admin", "super_admin", "", ""); err != nil {
+		t.Fatal(err)
+	}
+	editor, err := db.CreateUser("dr-editor", "$2a$10$editor", "manage", "DR Editor", "editor@dr.local")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.GrantAppAccess(editor.ID, app.ID); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.CreateAlertRule(&store.AlertRule{AppID: &app.ID, Metric: "cpu_pct", Operator: ">", Threshold: 90, DurationSec: 300, WebhookID: wh.ID, Enabled: true}); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.CreateBackupConfig(&store.BackupConfig{AppID: app.ID, Strategy: "volume", Target: "local", ScheduleCron: "0 3 * * *", TargetConfigJSON: "enc:target", RetentionMode: "count", RetentionCount: 7}); err != nil {
+		t.Fatal(err)
+	}
+	shutdown()
+
+	sidecar := filepath.Join(appDir, "simpledeploy.yml")
+	if _, err := os.Stat(sidecar); err != nil {
+		t.Fatalf("app sidecar not written on shutdown: %v", err)
+	}
+	for _, f := range []string{"simpledeploy.db", "simpledeploy.db-wal", "simpledeploy.db-shm"} {
+		if err := os.Remove(filepath.Join(cfg.DataDir, f)); err != nil && !os.IsNotExist(err) {
+			t.Fatal(err)
+		}
+	}
+
+	// Restart on the empty DB.
+	db, shutdown = bootServe(t, cfg)
+	defer shutdown()
+
+	if _, err := os.Stat(sidecar); err != nil {
+		t.Fatalf("app sidecar deleted on restart: %v", err)
+	}
+	u, err := db.GetUserByUsername("dr-editor")
+	if err != nil || u.Role != "manage" || u.DisplayName != "DR Editor" {
+		t.Fatalf("editor not restored: %+v, %v", u, err)
+	}
+	webhooks, _ := db.ListWebhooks()
+	if len(webhooks) != 1 || webhooks[0].Name != "dr-hook" || webhooks[0].URL != "http://127.0.0.1:9/dr-hook" {
+		t.Fatalf("webhook not restored: %+v", webhooks)
+	}
+	hook := webhooks[0]
+	app, err = db.GetAppBySlug("dr-nginx")
+	if err != nil {
+		t.Fatalf("app not re-adopted: %v", err)
+	}
+	rules, _ := db.ListAlertRules(&app.ID)
+	if len(rules) != 1 || rules[0].Metric != "cpu_pct" || rules[0].Operator != ">" || rules[0].Threshold != 90 ||
+		rules[0].DurationSec != 300 || rules[0].WebhookID != hook.ID || !rules[0].Enabled {
+		t.Errorf("alert rule not restored: %+v", rules)
+	}
+	backups, _ := db.ListBackupConfigs(&app.ID)
+	if len(backups) != 1 || backups[0].ScheduleCron != "0 3 * * *" || backups[0].TargetConfigJSON != "enc:target" {
+		t.Errorf("backup config not restored: %+v", backups)
+	}
+	if access, _ := db.ListAccessForApp(app.ID); len(access) != 1 || access[0] != "dr-editor" {
+		t.Errorf("access grant not restored: %v", access)
+	}
+}
diff --git a/cmd/simpledeploy/main.go b/cmd/simpledeploy/main.go
index 57fb9113..01613416 100644
--- a/cmd/simpledeploy/main.go
+++ b/cmd/simpledeploy/main.go
@@ -476,65 +476,7 @@ func runServe(cmd *cobra.Command, args []string) error {
 	syncer := configsync.New(db, cfg.AppsDir, cfg.DataDir)
 	defer syncer.Close()
 
-	// Rehydrate global config if DB is empty (first boot after DR).
-	if imported, err := syncer.ImportGlobalIfEmpty(); err != nil {
-		log.Printf("[configsync] global import failed: %v", err)
-	} else if imported {
-		log.Printf("[configsync] imported global sidecar into empty DB")
-	}
-
-	// Prune sidecar files for apps that no longer exist in the DB (e.g. from
-	// prior delete cycles before this feature existed).
-	if pruned, err := syncer.PruneOrphanSidecars(); err != nil {
-		log.Printf("[configsync] prune orphan sidecars: %v", err)
-	} else if len(pruned) > 0 {
-		log.Printf("[configsync] pruned orphan sidecars for: %v", pruned)
-	}
-
-	// Install mutation hook so future mutations update sidecars.
-	// FS-authoritative state writes are eventually consistent: this hook fires
-	// after a successful DB commit and schedules a debounced FS write (500ms).
-	// If the FS write fails, the reconciler watcher (internal/reconciler) re-
-	// applies on the next file edit. Contract: DB success => FS reflects soon.
-	// See docs/operations/state-on-disk.md.
-	db.SetMutationHook(func(scope store.MutationScope, slug string) {
-		switch scope {
-		case store.ScopeApp:
-			if slug != "" {
-				syncer.ScheduleAppWrite(slug)
-			}
-		case store.ScopeGlobal:
-			syncer.ScheduleGlobalWrite()
-		}
-	})
-
-	// First-boot sidecar backfill: write sidecars for all existing DB state
-	// so that upgrades from pre-configsync installs get sidecars on first boot.
-	backfillMarker := filepath.Join(cfg.DataDir, ".configsync_backfill_v1")
-	if _, err := os.Stat(backfillMarker); os.IsNotExist(err) {
-		allApps, listErr := db.ListApps()
-		if listErr != nil {
-			log.Printf("[configsync] backfill: list apps failed: %v (skipping)", listErr)
-		} else {
-			allOK := true
-			if backfillErr := syncer.WriteGlobal(); backfillErr != nil {
-				log.Printf("[configsync] backfill: write global failed: %v", backfillErr)
-				allOK = false
-			}
-			for _, a := range allApps {
-				if werr := syncer.WriteAppSidecar(a.Slug); werr != nil {
-					log.Printf("[configsync] backfill: write sidecar for %s: %v", a.Slug, werr)
-					allOK = false
-				}
-			}
-			if allOK {
-				log.Printf("[configsync] first-boot sidecar backfill: wrote %d apps + global", len(allApps))
-				if werr := os.WriteFile(backfillMarker, nil, 0600); werr != nil {
-					log.Printf("[configsync] backfill: write marker failed: %v", werr)
-				}
-			}
-		}
-	}
+	initConfigSync(db, syncer, cfg.DataDir)
 
 	// GitSync wiring (optional; non-fatal on error).
 	// Build syncer using DB-wins resolver. Start deferred to after ctx creation.
@@ -615,13 +557,8 @@ func runServe(cmd *cobra.Command, args []string) error {
 	ctx, cancel := context.WithCancel(cmd.Context())
 	defer cancel()
 
-	// FS-authoritative seed (only on first boot post-upgrade).
-	if err := configsync.RunFirstBootSeedIfNeeded(ctx, db, syncer, cfg); err != nil {
-		return fmt.Errorf("fs-auth seed: %w", err)
-	}
-	// Reconcile DB cache from FS files (idempotent on every boot).
-	if err := syncer.ReconcileDBFromFS(ctx); err != nil {
-		return fmt.Errorf("fs-auth reload: %w", err)
+	if err := reloadConfigFromFS(ctx, db, syncer, cfg); err != nil {
+		return err
 	}
 
 	// Start gitsync now that ctx exists.

From 6fd292c7308cf2e24f232660d6bf9414887f8f4c Mon Sep 17 00:00:00 2001
From: Ameen Rashad <7609895+vazra@users.noreply.github.com>
Date: Sun, 11 Oct 2026 02:45:55 -0700
Subject: [PATCH 4/5] test: cover malformed config.yml in DR seed guard

---
 internal/configsync/recovery_test.go | 72 +++++++++++++---------------
 1 file changed, 34 insertions(+), 38 deletions(-)

diff --git a/internal/configsync/recovery_test.go b/internal/configsync/recovery_test.go
index 5534299b..47887efb 100644
--- a/internal/configsync/recovery_test.go
+++ b/internal/configsync/recovery_test.go
@@ -5,7 +5,6 @@ import (
 	"os"
 	"path/filepath"
 	"slices"
-	"strings"
 	"testing"
 	"time"
 
@@ -275,45 +274,42 @@ func TestReconcileDBFromFSSkipsAppsWithoutRow(t *testing.T) {
 }
 
 // TestDRImportFailureKeepsGlobalSidecar: when importing config.yml into an
-// empty DB fails part way, the first-boot seed must not overwrite config.yml
-// and secrets.yml with the incomplete DB.
+// empty DB fails, the first-boot seed must not overwrite config.yml with the
+// empty or incomplete DB.
 func TestDRImportFailureKeepsGlobalSidecar(t *testing.T) {
-	st := openTestStore(t)
-	dataDir := t.TempDir()
-	syncer := New(st, t.TempDir(), dataDir)
+	cases := []struct {
+		name   string
+		config string
+	}{
+		{"malformed yaml", "version: 1\nusers:\n  - username: admin\n    role: [super_admin\nregistries:\n  - name: ghcr\n"},
+		// The API key's user is missing: users import, then the key fails
+		// before registries.
+		{"dangling api key", "version: 1\nusers:\n  - username: admin\n    role: super_admin\napi_keys:\n  - username: gone\n    name: ci\nregistries:\n  - id: reg-1\n    name: ghcr\n    url: ghcr.io\n"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			st := openTestStore(t)
+			dataDir := t.TempDir()
+			syncer := New(st, t.TempDir(), dataDir)
+			path := filepath.Join(dataDir, globalSidecar)
+			if err := os.WriteFile(path, []byte(tc.config), 0o644); err != nil {
+				t.Fatal(err)
+			}
 
-	// The API key's user is missing, so ImportGlobal fails after importing
-	// users and before registries.
-	sidecar := GlobalSidecar{
-		Version:    Version,
-		Users:      []UserEntry{{Username: "admin", Role: "super_admin"}},
-		APIKeys:    []APIKeyEntry{{Username: "gone", Name: "ci"}},
-		Registries: []RegistryEntry{{ID: "reg-1", Name: "ghcr", URL: "ghcr.io"}},
-	}
-	if err := atomicWriteYAMLMode(filepath.Join(dataDir, globalSidecar), 0644, sidecar); err != nil {
-		t.Fatal(err)
-	}
-	if err := syncer.WriteGlobalSecrets(&GlobalSecrets{
-		Version: Version,
-		Users:   []UserSecretsEntry{{Username: "admin", PasswordHash: "$2a$10$admin"}},
-	}); err != nil {
-		t.Fatal(err)
-	}
+			if _, err := syncer.ImportGlobalIfEmpty(); err == nil {
+				t.Fatal("expected ImportGlobalIfEmpty to fail")
+			}
+			if err := RunFirstBootSeedIfNeeded(context.Background(), st, syncer, nil); err != nil {
+				t.Fatalf("seed: %v", err)
+			}
 
-	if _, err := syncer.ImportGlobalIfEmpty(); err == nil {
-		t.Fatal("expected ImportGlobalIfEmpty to fail on the dangling API key")
-	}
-	if err := RunFirstBootSeedIfNeeded(context.Background(), st, syncer, nil); err != nil {
-		t.Fatalf("seed: %v", err)
-	}
-
-	raw, err := os.ReadFile(filepath.Join(dataDir, globalSidecar))
-	if err != nil {
-		t.Fatal(err)
-	}
-	for _, want := range []string{"name: ghcr", "username: gone"} {
-		if !strings.Contains(string(raw), want) {
-			t.Errorf("config.yml lost %q after a failed import:\n%s", want, raw)
-		}
+			raw, err := os.ReadFile(path)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if string(raw) != tc.config {
+				t.Errorf("config.yml rewritten after a failed import:\n%s", raw)
+			}
+		})
 	}
 }

From 93e5706a86cf935c9ad5becbdfea5fd869717640 Mon Sep 17 00:00:00 2001
From: Ameen Rashad <7609895+vazra@users.noreply.github.com>
Date: Sun, 11 Oct 2026 02:45:55 -0700
Subject: [PATCH 5/5] docs: fix sidecar recovery docs (file layout, modes,
 secrets.yml, prune rule)

config-sidecars.md claimed config.yml holds bcrypt hashes at 0600 and that
master_secret lives in data_dir/.secret; hashes are in secrets.yml (0600),
config.yml/simpledeploy.yml are 0644, master_secret is in config.yaml.
Document secrets sidecars, the orphan prune rule, the reconciler-driven
per-app import, and that files win over the DB.
---
 docs/architecture/reconciler.md    |  2 +-
 docs/operations/config-sidecars.md | 56 ++++++++++++++++++------------
 docs/operations/git-sync.md        |  2 +-
 docs/operations/state-on-disk.md   |  2 +-
 4 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/docs/architecture/reconciler.md b/docs/architecture/reconciler.md
index 526a5fbc..555d4508 100644
--- a/docs/architecture/reconciler.md
+++ b/docs/architecture/reconciler.md
@@ -17,7 +17,7 @@ There is no internal periodic tick. If you want a heartbeat sweep, restart the p
 
 1. `scanAppsDir()` walks `apps_dir`, parses and validates every `docker-compose.yml`, and returns a `scanResult`: `desired` (apps that pass) and `refused` (apps on disk that fail compose security rules, have a symlinked compose file, or reference files outside the app folder). Hidden directories (leading `.`) and directories without a compose file are skipped silently.
 2. `store.ListApps()` returns currently-known apps from SQLite.
-3. For each desired app: if not in the store, deploy. If in the store but the file's SHA-256 differs from `apps.compose_hash`, redeploy.
+3. For each desired app: if not in the store, deploy, then import its `simpledeploy.yml` when the app has no alert rules, backup configs or access grants yet (`ImportAppSidecarIfMissing`; this is how per-app config comes back after a DB loss). If in the store but the file's SHA-256 differs from `apps.compose_hash`, redeploy.
 4. For each store app neither desired nor refused, whose compose file is gone: archive it (teardown, tombstone, row marked archived; see [Archived apps](/operations/archived-apps/)).
 5. Recompute the route table from desired apps plus route-only refused apps (below), tag every route with its app's store ID (`Route.AppID`, `math.MaxInt64` for apps not in the store yet), and call `proxy.SetRoutes()`. The proxy uses `AppID` to give a domain claimed by several apps to the one created first (see [Proxy](/architecture/proxy/)).
 
diff --git a/docs/operations/config-sidecars.md b/docs/operations/config-sidecars.md
index 8422179e..68b78f7c 100644
--- a/docs/operations/config-sidecars.md
+++ b/docs/operations/config-sidecars.md
@@ -15,32 +15,36 @@ Historical data (metrics, deploy events, alert history, backup run records) is n
 {data_dir}/
   simpledeploy.db          # SQLite database
   simpledeploy.db-wal      # WAL journal (normal)
-  .secret                  # master_secret (keep this safe)
   .configsync_backfill_v1  # sentinel written after first-boot backfill
-  config.yml               # global sidecar (users, api keys, registries, webhooks, db backup config)
+  config.yml               # global sidecar (users, api keys, registries, webhooks, db backup config), no secrets
+  secrets.yml              # password hashes, api key hashes, encrypted registry creds, webhook URLs
+  archive/<slug>.yml       # tombstones of archived apps (see Archived apps)
 
 {apps_dir}/
+  _global.yml              # redacted copy of config.yml for git sync
   my-app/
-    compose.yaml           # your compose file
-    simpledeploy.yml       # per-app sidecar (alert rules, backup configs, access grants)
+    docker-compose.yml        # your compose file
+    simpledeploy.yml          # per-app sidecar (alert rules, backup configs, access grants)
+    simpledeploy.secrets.yml  # encrypted backup target configs
   another-app/
-    compose.yaml
+    docker-compose.yml
     simpledeploy.yml
+    simpledeploy.secrets.yml
 ```
 
-Both sidecar files use YAML `version: 1` and are written with file mode `0600`.
+All sidecar files use YAML `version: 1`. `config.yml` and `simpledeploy.yml` hold no secrets and are written with file mode `0644`; `secrets.yml` and `simpledeploy.secrets.yml` are written with mode `0600`. `master_secret` is not stored here: it lives in your server config file (`/etc/simpledeploy/config.yaml` by default).
 
 ## What is and isn't mirrored
 
 | Data | Mirrored | Notes |
 |------|----------|-------|
-| Users (display name, email, bcrypt hash) | Yes | `config.yml` |
-| API keys (hashed) | Yes | `config.yml` |
-| Registries (encrypted credentials) | Yes | `config.yml` |
-| Webhooks | Yes | `config.yml` |
-| DB backup config | Yes | `config.yml` |
+| Users (role, display name, email) | Yes | `config.yml`; bcrypt hash in `secrets.yml` |
+| API keys (name, expiry) | Yes | `config.yml`; key hash in `secrets.yml` |
+| Registries (name, URL) | Yes | `config.yml`; encrypted credentials in `secrets.yml` |
+| Webhooks (name, type) | Yes | `config.yml`; URL, headers, template in `secrets.yml` |
+| DB backup config | Yes | `config.yml`; encrypted target config in `secrets.yml` |
 | Alert rules (per app) | Yes | `simpledeploy.yml` |
-| Backup configs (per app, encrypted) | Yes | `simpledeploy.yml` |
+| Backup configs (per app) | Yes | `simpledeploy.yml`; encrypted target config in `simpledeploy.secrets.yml` |
 | User access grants (per app) | Yes | `simpledeploy.yml` |
 | Metrics / request stats | **No** | High-volume, by design |
 | Deploy events | **No** | Historical, by design |
@@ -54,19 +58,27 @@ After every mutation (API call that changes a mirrored setting), the server sche
 
 ## How recovery works
 
-**Global sidecar import**: on startup, if the `users` table is empty and `{data_dir}/config.yml` exists, the global sidecar is imported automatically. A log line confirms:
+**Global sidecar import**: on startup, if the `users` table is empty and `{data_dir}/config.yml` exists, the global sidecar (with `secrets.yml`) is imported automatically. A log line confirms:
 
 ```
 [configsync] imported global sidecar into empty DB
 ```
 
-**Per-app sidecar import**: during each reconcile pass, any app whose compose file is present on disk but has no DB-side config (no alert rules, no backup configs, no access grants) has its `simpledeploy.yml` imported. Log line:
+If `config.yml` cannot be imported (for example malformed YAML after a hand edit), the server does not replace it with the empty DB's contents: startup stops with an error naming the file, so you can fix it and start again.
+
+**Per-app sidecar import**: the reconciler re-adds every app folder that has a `docker-compose.yml` but no DB row, then imports that app's `simpledeploy.yml` (and `simpledeploy.secrets.yml`) if the app has no alert rules, backup configs or access grants yet. Log line:
 
 ```
-[configsync] imported sidecar for my-app
+[configsync] imported app sidecar for my-app
 ```
 
-Neither import overwrites a healthy DB. If the DB already has data for a given app or user, it is left untouched.
+Until then, the boot-time reload skips the app with `[fs-auth] my-app: no app row yet; its sidecar is imported once the reconciler adds the app`.
+
+**Archived apps** come back from their tombstones in `{data_dir}/archive/`, still archived.
+
+These two imports never overwrite existing rows: they run only for an empty `users` table, or for an app with no alert rules, backup configs or access grants yet.
+
+**Orphan cleanup never removes live config.** On startup, a `simpledeploy.yml` is deleted only from a folder that is no longer an app: no `docker-compose.yml`, no DB row (active or archived) and no tombstone. Nothing is deleted while the DB has no apps at all, so a wiped DB never costs you app sidecars. Removals are logged as `[configsync] pruned orphan sidecars for: [...]`.
 
 With [git sync](/operations/git-sync/) enabled, access grants are managed in the dashboard only: the `access` list in a pulled or hand-edited `simpledeploy.yml` is never applied, and the server rewrites a pulled file's `access` list back to the dashboard's grants. Disaster-recovery imports into an empty DB from sidecars already on disk still restore grants. Sidecar and data-dir files must be regular files; symlinks are refused.
 
@@ -76,7 +88,7 @@ With [git sync](/operations/git-sync/) enabled, access grants are managed in the
 This procedure recovers configuration only. Historical data (metrics, deploy events, alert history) cannot be recovered from sidecars. For a full recovery including app volumes, follow [Disaster recovery](/operations/disaster-recovery/) alongside this procedure.
 </Aside>
 
-1. **Preserve `master_secret` and `apps_dir`.** The `master_secret` lives in `{data_dir}/.secret` (or is set via `master_secret:` in your config YAML). Without it, encrypted blobs (registry credentials, S3 backup target configs) are unreadable even though the sidecar files still exist. Users and API keys are still restored because only their bcrypt hashes are stored.
+1. **Preserve `master_secret`, `{data_dir}` and `apps_dir`.** `master_secret` is set via `master_secret:` in your server config file (`/etc/simpledeploy/config.yaml` by default). Without it, encrypted blobs (registry credentials, S3 backup target configs) are unreadable and API keys stop working, even though the sidecar files still exist. Users can still log in because their passwords are stored as bcrypt hashes.
 
 2. **Stop SimpleDeploy.**
    ```bash
@@ -95,7 +107,7 @@ This procedure recovers configuration only. Historical data (metrics, deploy eve
    sudo systemctl start simpledeploy
    journalctl -u simpledeploy -f
    ```
-   Watch for the import log lines. Within seconds of startup you should see the global sidecar import, then per-app imports as the reconciler runs.
+   Watch for the import log lines. Within seconds of startup you should see the global sidecar import, then per-app imports as the reconciler re-adds each app (it runs `docker compose up` for each one; unchanged containers keep running).
 
 5. **Log in with your existing admin credentials.** They are restored from the sidecar.
 
@@ -121,10 +133,10 @@ simpledeploy config import --force --wipe
 
 ## Cautions
 
-**Losing `master_secret` with a wiped DB**: registry credentials and S3 backup target configs are unrecoverable. Users, API keys, alert rules, and webhooks are still restored. Keep `master_secret` in a password manager separate from the host.
+**Losing `master_secret` with a wiped DB**: registry credentials and S3 backup target configs are unrecoverable, and API keys must be re-created (their hashes are keyed with `master_secret`). Users, alert rules, webhooks and access grants are still restored. Keep `master_secret` in a password manager separate from the host.
 
-**Sidecar files contain sensitive data**: bcrypt password hashes, encrypted credential blobs. File mode is `0600` by default. Do not commit them to a public repo. [Git sync](/operations/git-sync/) commits the app folders and the redacted `_global.yml`, never `{data_dir}/config.yml`.
+**Secrets files contain sensitive data**: `secrets.yml` and `simpledeploy.secrets.yml` hold bcrypt password hashes, API key hashes, encrypted credential blobs and webhook URLs, with file mode `0600`. Do not commit them to a public repo. [Git sync](/operations/git-sync/) commits the app folders (its `.gitignore` excludes `*.secrets.yml`) and the redacted `_global.yml`, never anything in `{data_dir}`.
 
-**Hand-editing sidecars**: supported. Edits take effect on the next startup or reconcile pass. Malformed YAML fails the import with a log line and leaves the DB unchanged.
+**Hand-editing sidecars**: supported. The file watcher applies edits within about a second; otherwise they apply on the next start. A malformed `simpledeploy.yml` is skipped with a log line and leaves the DB unchanged; a malformed `config.yml` stops the server from starting until it is fixed.
 
-**DB wins over sidecar**: if the same app or user already exists in the DB, the sidecar is ignored for that record. Sidecars are authoritative only when the DB is empty for that entity.
+**Files win over the DB**: on every start, and whenever a sidecar changes, the server applies the files to the DB (see [State on disk](/operations/state-on-disk/)). For an app the DB does not have yet, its sidecar is applied once the reconciler re-adds the app.
diff --git a/docs/operations/git-sync.md b/docs/operations/git-sync.md
index 9d459aef..8d22c466 100644
--- a/docs/operations/git-sync.md
+++ b/docs/operations/git-sync.md
@@ -9,7 +9,7 @@ When enabled, SimpleDeploy treats your `apps_dir` as a git working tree and comm
 
 Git sync complements [config sidecars](./config-sidecars), which write app config to local YAML files. Think of sidecars as the local source of truth and git sync as the transport layer that makes that truth portable. Sidecars are what get committed; git sync is what moves them to a remote.
 
-Git sync is **not** a replacement for database backup. It does not capture metrics, deploy history, audit logs, or any file outside `apps_dir`. Secrets stay local: `config.yml` (which holds password hashes and encrypted registry credentials) and the SQLite database are never committed.
+Git sync is **not** a replacement for database backup. It does not capture metrics, deploy history, audit logs, or any file outside `apps_dir`. Secrets stay local: `{data_dir}/config.yml`, `{data_dir}/secrets.yml` (password hashes, API key hashes, encrypted registry credentials, webhook URLs), `*.secrets.yml` in app folders, and the SQLite database are never committed.
 
 ## What gets committed
 
diff --git a/docs/operations/state-on-disk.md b/docs/operations/state-on-disk.md
index 9d04c543..a8acc051 100644
--- a/docs/operations/state-on-disk.md
+++ b/docs/operations/state-on-disk.md
@@ -96,7 +96,7 @@ Pruning is logged: whenever `ApplyGlobalSidecar` (boot or watcher) deletes a use
 
 1. Stop the server.
 2. Move `{data_dir}/simpledeploy.db` aside (rename, don't delete until you've verified recovery).
-3. Restart. The first-boot seed marker (`system_meta`) is in the DB, so the server runs the FS-to-DB reconcile via `ReconcileDBFromFS`. All FS-authoritative state rebuilds.
+3. Restart. The server imports `config.yml` + `secrets.yml` into the empty DB and records the first-boot seed as done, so the seed never rewrites those files from the empty DB. The reconciler then re-adds every app folder that has a `docker-compose.yml` and imports its `simpledeploy.yml` + `simpledeploy.secrets.yml`. App sidecars are never pruned in this state: orphan cleanup only touches folders without a compose file, DB row or tombstone, and skips an empty DB. See [Config sidecars](/operations/config-sidecars/#how-recovery-works).
 
 Caveat: runtime state (metrics, audit log, deploy history, archived-app rows that have no tombstone) is lost.