perf: reduce Activity memory and repeated team and store work

CLAUDE.md

@@ -38,6 +38,17 @@ and obsolete assumptions in the entry; see [changelog rules](docs/rules/changelo
 
 ## Build and test
 
+Apply [Matt Pocock's engineering skills](https://github.com/mattpocock/skills)
+when writing code: diagnose unexplained failures before changing code, use TDD
+for fixes and features at agreed observable interfaces, and review the final
+diff against both repository standards and the request. Pass these practices
+to coding delegates. Claude can load them with
+`claude plugin install mattpocock-skills@claude-plugins-official`.
+
+Keep module interfaces small and concentrate related behavior behind them.
+Prefer reusing an existing module over adding a pass-through abstraction.
+Measure performance before and after; preserve behavior during cleanup.
+
 Use `make build` for the runnable binary at `bin/codeaf`. Rebuild after merging.
 Do not overwrite a running executable in place; use the installation/update path.
 

PERF.md

@@ -17,6 +17,27 @@ is diagnostic; regression gates count deterministic work, bytes, allocations,
 request fanout, retained data and cache reuse. Do not gate on wall-clock thresholds.
 Use scripted providers and explicit synchronization rather than real sleeps.
 
+Activity layout lines share immutable task records instead of copying both task
+and conversation payloads into every line. Conversation overlays keep their own
+copy, so live titles do not mutate the held reading. Paint scratch still expires
+before the next input. `TestActivityScalePaintAllocationBudget` exercises the
+actual body with 5,120 tasks and enforces a 4 MiB allocation budget per paint;
+`BenchmarkActivityScaleFrame` and `TestActivityScaleTerminalLatency` measure the
+same workload at the body and real-terminal interfaces.
+
+Team disbanding reads reporting homes once through the existing batch reader
+before changing memberships. `BenchmarkDisbandScale` covers 10,000 and 40,000
+memberships; the batch trades temporary indexed storage for repeated scans.
+`TestTeamsDisbandScaleTerminalLatency` measures confirmation through the visible
+saved result, including persistence and redraw work.
+
+Store reopening probes distinct nonempty session IDs through the existing
+message index before checking for missing session rows. It still scans message
+IDs, and still discovers older writers' missing rows on a later open; no cached
+migration flag skips that check. `BenchmarkOpenSettledSessions` measures the
+whole public open/close operation. `TestSettledStoreTerminalStartup` separates
+fixture construction from the first interactive frame of the running terminal.
+
 ## What the gates check
 
 `make test-quick` checks compilation, vet, formatting, packed manuals, change

docs/changes/unreleased/1925-efficient-history-team-store.md

@@ -0,0 +1,15 @@
+---
+kind: fixed
+title: Reduce repeated work in large histories and team changes
+pr: 1925
+surface: [chat, engine, docs]
+invalidates:
+  - "Activity layouts previously copied full task and conversation records into each line. Lines now share immutable task records, while live conversation overlays keep independent copies; the 5,120-task paint has a 4 MiB allocation regression budget."
+  - "Disbanding a team previously scanned all memberships again for every reporting-home lookup. It now reuses the existing batched home reading before changing coordination, preserving history and unrelated memberships."
+  - "Opening a settled store previously checked for a missing session once per message. It now checks distinct nonempty session IDs through the existing covering index; later legacy writers' missing rows are still discovered."
+---
+
+No dependency, schema, persistent cache, product setting, or public interface is
+added. Team mutation trades transient indexed storage for fewer repeated scans;
+store reopening still scans message IDs. Contributor guidance records the Matt
+Pocock diagnosis, TDD, module-design, and independent review workflow.

internal/e2e/activity_scale_e2e_test.go

@@ -145,4 +145,5 @@ func TestActivityScaleTerminalLatency(t *testing.T) {
 	}
 	sort.Float64s(samples)
 	t.Logf("chats=40 tasks=5120 setup=%s first_screen=%s hover_ms p50=%.3f p95=%.3f samples=%v model_calls=0", setup, opening, samples[len(samples)/2], samples[len(samples)-1], samples)
+	r.quit()
 }

internal/e2e/skills_efficiency_e2e_test.go

@@ -0,0 +1,192 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/Agent-Field/codeaf/internal/config"
+	"github.com/Agent-Field/codeaf/internal/store"
+	teamstore "github.com/Agent-Field/codeaf/internal/teams"
+)
+
+// Navigation and startup measurements exclude provider work entirely. The real
+// binary still uses its ordinary configured model roles and rejects chat calls.
+func efficiencyWire(t *testing.T) (string, *atomic.Int64) {
+	t.Helper()
+	calls := &atomic.Int64{}
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		if strings.HasSuffix(r.URL.Path, "/chat/completions") {
+			calls.Add(1)
+			http.Error(w, "no model work belongs in this measurement", 500)
+			return
+		}
+		fmt.Fprint(w, `{"data":[]}`)
+	}))
+	t.Cleanup(s.Close)
+	return s.URL, calls
+}
+
+func efficiencyWait(t *testing.T, r *rig, start time.Time, text string) time.Duration {
+	t.Helper()
+	for time.Since(start) < 30*time.Second {
+		if strings.Contains(r.capture(), text) {
+			return time.Since(start)
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	t.Fatalf("screen did not display %q:\n%s", text, r.capture())
+	return 0
+}
+
+func TestTeamsDisbandScaleTerminalLatency(t *testing.T) {
+	for _, size := range []struct{ teams, members int }{{100, 100}, {200, 200}} {
+		t.Run(fmt.Sprintf("%dteams_%dmembers", size.teams, size.members), func(t *testing.T) {
+			if _, err := exec.LookPath("tmux"); err != nil {
+				t.Skip("tmux required")
+			}
+			began := time.Now()
+			home := newHome(t, map[string]any{"memory.enabled": false, "model_pool": "off"})
+			ws := newWorkspace(t, "disband-scale", false)
+			seedTeamsScale(t, size.teams, size.members, home, ws)
+			setup := time.Since(began)
+			url, calls := efficiencyWire(t)
+			r := startWithEnv(t, []string{config.APIKeyEnv + "=mock-efficiency-key", "CODEAF_BASE_URL=" + url}, "disband_scale", home, ws, 160, 55, "chat", "--model", "deepseek/deepseek-v4.1-flash", "--one-model", "--no-host")
+			r.skipSetup(t)
+			r.lit("/teams")
+			r.keys("Enter")
+			screen := r.waitFor(30*time.Second, "Scale team 000")
+			selected := false
+			for y, line := range strings.Split(screen, "\n") {
+				if x := strings.Index(line, "Scale team 000"); y > 2 && x >= 0 && x < 30 {
+					r.mouseClick(x+2, y+1)
+					selected = true
+					break
+				}
+			}
+			if !selected {
+				t.Fatal("no team sidebar target")
+			}
+			r.waitFor(30*time.Second, "Disband")
+			r.keys("c")
+			r.waitFor(30*time.Second, "Disband these teams:", "Cancel")
+			r.keys("Down")
+			start := time.Now()
+			if out, err := exec.Command("tmux", "send-keys", "-t", r.name, "Enter").CombinedOutput(); err != nil {
+				t.Fatalf("confirm: %v %s", err, out)
+			}
+			elapsed := efficiencyWait(t, r, start, "Scale team 000 is disbanded; conversations and current work continue")
+			f, err := teamstore.Load(home)
+			if err != nil {
+				t.Fatal(err)
+			}
+			closed, _ := f.Team("000000000001")
+			other, _ := f.Team("000000000002")
+			if !closed.Closed() || other.Closed() || len(closed.Members) != size.members || len(other.Members) != size.members {
+				t.Fatal("disband lost history or changed an unrelated team")
+			}
+			if calls.Load() != 0 {
+				t.Fatalf("disband made %d model calls", calls.Load())
+			}
+			t.Logf("teams=%d memberships=%d setup=%s confirm_to_saved_notice=%s model_calls=0", size.teams, size.teams*size.members, setup, elapsed)
+			r.quit()
+		})
+	}
+}
+
+func TestSettledStoreTerminalStartup(t *testing.T) {
+	if _, err := exec.LookPath("tmux"); err != nil {
+		t.Skip("tmux required")
+	}
+	began := time.Now()
+	home := newHome(t, map[string]any{"memory.enabled": false, "model_pool": "off"})
+	ws := newWorkspace(t, "settled-store", false)
+	path := filepath.Join(home, "graph.db")
+	s, err := store.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := s.Close(); err != nil {
+		t.Fatal(err)
+	}
+	db, err := sql.Open("sqlite", path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = db.Close() })
+	_, err = db.Exec(`WITH RECURSIVE fixture(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM fixture WHERE i < 1000000)
+INSERT INTO events(ts,node_id,kind,payload)
+SELECT '2026-01-01T00:00:00Z','','message_posted',json_object('session_id','mock-'||(i%1000),'role','user','body','Mock message') FROM fixture`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := db.Exec(`INSERT INTO messages(seq,ts,session_id,role,body)
+SELECT seq,ts,json_extract(payload,'$.session_id'),'user','Mock message' FROM events WHERE kind='message_posted'`); err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Close(); err != nil {
+		t.Fatal(err)
+	}
+	s, err = store.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rooms, err := s.Sessions()
+	if err != nil || len(rooms) != 1000 {
+		t.Fatalf("fixture has %d rooms: %v", len(rooms), err)
+	}
+	if err := s.Close(); err != nil {
+		t.Fatal(err)
+	}
+	setup := time.Since(began)
+	url, calls := efficiencyWire(t)
+	var samples []float64
+	for i := 0; i < 3; i++ {
+		name := fmt.Sprintf("settled_start_%d", i)
+		target := fmt.Sprintf("p%d-%s", os.Getpid(), name)
+		ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
+		defer cancel()
+		ready := make(chan time.Time, 1)
+		start := time.Now()
+		// Observe the real first interactive surface without the rig's coarser
+		// startup polling. Fixture creation and later setup dismissal are separate.
+		go func() {
+			for ctx.Err() == nil {
+				out, _ := exec.CommandContext(ctx, "tmux", "capture-pane", "-p", "-t", target).Output()
+				screen := string(out)
+				if strings.Contains(screen, "alt+e effort") || strings.Contains(screen, "esc skips setup") || strings.Contains(screen, "↑↓ moves") || strings.Contains(screen, "needs you") {
+					ready <- time.Now()
+					return
+				}
+				time.Sleep(5 * time.Millisecond)
+			}
+		}()
+		r := startWithEnv(t, []string{config.APIKeyEnv + "=mock-efficiency-key", "CODEAF_BASE_URL=" + url}, name, home, ws, 160, 55, "chat", "--model", "deepseek/deepseek-v4.1-flash", "--one-model", "--no-host")
+		select {
+		case shown := <-ready:
+			samples = append(samples, float64(shown.Sub(start))/float64(time.Millisecond))
+		case <-ctx.Done():
+			t.Fatalf("no observed interactive surface:\n%s", r.capture())
+		}
+		cancel()
+		r.kill()
+	}
+	if calls.Load() != 0 {
+		t.Fatalf("startup made %d model calls", calls.Load())
+	}
+	sort.Float64s(samples)
+	t.Logf("messages=1000000 sessions=1000 setup=%s launch_to_first_surface_ms median=%.3f samples=%v model_calls=0", setup, samples[len(samples)/2], samples)
+}

internal/e2e/teams_scale_e2e_test.go

@@ -36,30 +36,7 @@ func runTeamsScaleTerminal(t *testing.T, count, members int) {
 	began := time.Now()
 	home := newHome(t, map[string]any{"model_pool": "off", "memory.enabled": false})
 	ws := newWorkspace(t, "teams-scale", false)
-	var teams []teamstore.Team
-	for i := 0; i < count; i++ {
-		tm := teamstore.Team{ID: fmt.Sprintf("%012x", i+1), Name: fmt.Sprintf("Scale team %03d", i)}
-		for j := 0; j < members; j++ {
-			sid := fmt.Sprintf("%016x", 0x6000000000000000+i*members+j)
-			dir := filepath.Join(home, "v3", "projects", strings.ReplaceAll(ws, string(filepath.Separator), "-"), sid)
-			if err := os.MkdirAll(dir, 0o700); err != nil {
-				t.Fatal(err)
-			}
-			at := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
-			writeJSON(t, filepath.Join(dir, "meta.json"), map[string]any{"id": sid, "title": fmt.Sprintf("Mock member %03d %03d", i, j), "workspace": ws, "created": at, "lastUserAt": at})
-			file := filepath.Join(dir, "transcript.jsonl")
-			journal := fmt.Sprintf("{\"type\":\"session\",\"version\":1,\"id\":%q,\"cwd\":%q}\n{\"type\":\"message\",\"role\":\"user\",\"content\":\"Review this mock change\",\"timestamp\":\"2026-01-01T00:00:00Z\"}\n{\"type\":\"message\",\"role\":\"assistant\",\"content\":\"Mock change reviewed\",\"timestamp\":\"2026-01-01T00:00:01Z\"}\n", sid, ws)
-			if err := os.WriteFile(file, []byte(journal), 0o600); err != nil {
-				t.Fatal(err)
-			}
-			tm.Members = append(tm.Members, teamstore.Member{Key: file, File: file, Where: ws, Handle: fmt.Sprintf("mock-%03d-%03d", i, j), Word: "Mock member", Home: j != 0, HandleBy: teamstore.HandleByTyped})
-			if j == 0 {
-				tm.Manager = file
-			}
-		}
-		teams = append(teams, tm)
-	}
-	writeJSON(t, teamstore.Path(home), teamstore.File{Version: teamstore.Version, Teams: teams})
+	seedTeamsScale(t, count, members, home, ws)
 	seed := time.Since(began)
 	var chats atomic.Int64
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -129,3 +106,31 @@ func runTeamsScaleTerminal(t *testing.T, count, members int) {
 	}
 	t.Logf("teams=%d memberships=%d setup=%s first_screen=%s hover_ms p50=%.3f p95=%.3f max=%.3f samples=%v model_calls=0", count, count*members, seed, opening, samples[len(samples)/2], samples[len(samples)-1], samples[len(samples)-1], samples)
 }
+
+func seedTeamsScale(t *testing.T, count, members int, home, ws string) {
+	t.Helper()
+	var teams []teamstore.Team
+	for i := 0; i < count; i++ {
+		tm := teamstore.Team{ID: fmt.Sprintf("%012x", i+1), Name: fmt.Sprintf("Scale team %03d", i)}
+		for j := 0; j < members; j++ {
+			sid := fmt.Sprintf("%016x", 0x6000000000000000+i*members+j)
+			dir := filepath.Join(home, "v3", "projects", strings.ReplaceAll(ws, string(filepath.Separator), "-"), sid)
+			if err := os.MkdirAll(dir, 0o700); err != nil {
+				t.Fatal(err)
+			}
+			at := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
+			writeJSON(t, filepath.Join(dir, "meta.json"), map[string]any{"id": sid, "title": fmt.Sprintf("Mock member %03d %03d", i, j), "workspace": ws, "created": at, "lastUserAt": at})
+			file := filepath.Join(dir, "transcript.jsonl")
+			journal := fmt.Sprintf("{\"type\":\"session\",\"version\":1,\"id\":%q,\"cwd\":%q}\n{\"type\":\"message\",\"role\":\"user\",\"content\":\"Review this mock change\",\"timestamp\":\"2026-01-01T00:00:00Z\"}\n{\"type\":\"message\",\"role\":\"assistant\",\"content\":\"Mock change reviewed\",\"timestamp\":\"2026-01-01T00:00:01Z\"}\n", sid, ws)
+			if err := os.WriteFile(file, []byte(journal), 0o600); err != nil {
+				t.Fatal(err)
+			}
+			tm.Members = append(tm.Members, teamstore.Member{Key: file, File: file, Where: ws, Handle: fmt.Sprintf("mock-%03d-%03d", i, j), Word: "Mock member", Home: j != 0, HandleBy: teamstore.HandleByTyped})
+			if j == 0 {
+				tm.Manager = file
+			}
+		}
+		teams = append(teams, tm)
+	}
+	writeJSON(t, teamstore.Path(home), teamstore.File{Version: teamstore.Version, Teams: teams})
+}

internal/store/sessions.go

@@ -503,19 +503,18 @@ func (s *Store) SessionLastNonUserMessageSeq(sessionID string) (int64, error) {
 // construction: the anti-join leaves it a no-op once every session has its row,
 // and it runs at open rather than on any read path.
 //
-// Idempotent is not free, though. The insert groups every message in the store
-// to decide it has nothing to do, and it decides that on every open but the
-// first — so the ordinary case pays a full scan of the thread to write nothing
-// at all. The probe below asks the same question with the same anti-join and
-// stops at the first row that answers it, which on a settled store is no rows
-// and an index seek.
+// Probe distinct room ids through the covering message index before looking
+// for a missing session. A settled store still reads that index, but it pays
+// one session lookup per room rather than one per message. Keep this probe on
+// every open: an older writer can introduce missing rows after migration.
 func backfillSessions(db *sql.DB) error {
 	var pending bool
 	if err := db.QueryRow(`
 		SELECT EXISTS (
-			SELECT 1 FROM messages m
-			WHERE m.session_id <> ''
-			  AND NOT EXISTS (SELECT 1 FROM sessions s WHERE s.id = m.session_id)
+			SELECT 1 FROM (
+				SELECT DISTINCT session_id FROM messages WHERE session_id <> ''
+			) m
+			WHERE NOT EXISTS (SELECT 1 FROM sessions s WHERE s.id = m.session_id)
 		)`).Scan(&pending); err != nil {
 		return err
 	}