diff --git a/README.md b/README.md index 9467aa6..af85f89 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ Run without `--apply` first to see what a command would do. | `doctor` | report broken, foreign, or mismatched links and stale captures; never writes | | `reconcile` | act on that drift, gated on what agentsync itself recorded | | `inventory` | regenerate `inventory.json`, a record of every agent on the machine | +| `optimize` | ask an OpenRouter advisor which model and effort each agent should use; report only | `agentsync help` lists every flag. @@ -80,6 +81,7 @@ Run without `--apply` first to see what a command would do. - [doctor](docs/doctor.md) - [reconcile](docs/reconcile.md) - [inventory](docs/inventory.md) +- [optimize](docs/optimize.md) - [State file](docs/state-file.md) - [Exit codes](docs/exit-codes.md) - [Development](docs/development.md): toolchain, CI, releases diff --git a/cmd/agentsync/config_test.go b/cmd/agentsync/config_test.go index 7ba6719..a564470 100644 --- a/cmd/agentsync/config_test.go +++ b/cmd/agentsync/config_test.go @@ -10,7 +10,9 @@ import ( ) // TestMain points HOME and XDG_CONFIG_HOME at empty temp dirs, so no test can -// read the developer's real config file and pass or fail because of it. +// read the developer's real config file and pass or fail because of it. It +// also clears the OpenRouter key, so no test can spend money, and the Claude +// overrides optimize reports. func TestMain(m *testing.M) { dir, err := os.MkdirTemp("", "agentsync-test-") if err != nil { @@ -22,12 +24,17 @@ func TestMain(m *testing.M) { os.Mkdir(home, 0o755), os.Setenv("HOME", home), os.Setenv("XDG_CONFIG_HOME", filepath.Join(dir, "xdg")), + os.Unsetenv(envOpenRouterKey), + os.Unsetenv("CLAUDE_CODE_SUBAGENT_MODEL"), + os.Unsetenv("CLAUDE_CODE_EFFORT_LEVEL"), } { if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(exitFailure) } } + // A test that sets a key but forgets its fake server fails here, offline. + openrouterURL = "http://127.0.0.1:0" code := m.Run() _ = os.RemoveAll(dir) os.Exit(code) diff --git a/cmd/agentsync/main.go b/cmd/agentsync/main.go index eac66a0..51ef0b4 100644 --- a/cmd/agentsync/main.go +++ b/cmd/agentsync/main.go @@ -35,6 +35,9 @@ // // init writes only agentsync's own config file, from values the user named or // confirmed, and refuses to replace an existing one without --force. +// +// optimize is the only command that uses the network: it asks an OpenRouter +// advisor about each agent's model and effort, and writes nothing. package main import ( @@ -62,6 +65,8 @@ Usage: agentsync doctor [flags] report broken/foreign/mismatched links and stale captures agentsync reconcile [flags] act on that drift, gated on what this tool recorded agentsync inventory [flags] regenerate inventory.json: every agent on this machine + agentsync optimize [flags] ask an OpenRouter advisor which model and effort each + agent should use (report only; the one networked command) agentsync version Common flags: @@ -216,6 +221,21 @@ inventory flags: is a real change in the tree, which is what makes this a freshness test rather than a clock reading. +optimize flags: + --advisor MODEL OpenRouter decision model to ask (default ~typesafe/jev-latest; + cloudflare/clef also works) + --max-cost USD skip any request that could take reported spend past this + (default 0.50) + --min-confidence P + least probability a pick needs before it is suggested + (default 0.6); below it the row says keep (low confidence) + --agent NAME ask only about this agent (repeatable) + +optimize needs $OPENROUTER_API_KEY and sends each agent file (first 24 KB) to +OpenRouter, one request per agent with no retries. It reads model: and effort: +from each agent's frontmatter and writes nothing. Exit 1 when a row is +unresolved or skipped. + scan and doctor are read-only. import writes only into this repo, only with --apply, and never into a project's .claude/agents directory. link is the one command that writes into a project, only with --apply, and it refuses any @@ -284,6 +304,8 @@ func run(args []string, stdout, stderr io.Writer) int { return runReconcile(args[1:], stdout, stderr) case "inventory": return runInventory(args[1:], stdout, stderr) + case "optimize": + return runOptimize(args[1:], stdout, stderr) case "-h", "--help", "help": fmt.Fprint(stdout, usage) return exitOK diff --git a/cmd/agentsync/optimize.go b/cmd/agentsync/optimize.go new file mode 100644 index 0000000..398b92d --- /dev/null +++ b/cmd/agentsync/optimize.go @@ -0,0 +1,232 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "math" + "os" + "path/filepath" + "strings" + "text/tabwriter" + + "github.com/agentic-tooling/agentsync/internal/discover" + "github.com/agentic-tooling/agentsync/internal/openrouter" + "github.com/agentic-tooling/agentsync/internal/optimize" + "github.com/agentic-tooling/agentsync/internal/scan" +) + +// openrouterURL is where optimize sends its requests; tests point it at an +// httptest server. optimize is the only command that uses the network. +var openrouterURL = openrouter.DefaultBaseURL + +const ( + envOpenRouterKey = "OPENROUTER_API_KEY" + defaultAdvisor = "~typesafe/jev-latest" +) + +// Claude Code reads these at run time and lets them beat the frontmatter. +var claudeOverrides = []string{"CLAUDE_CODE_SUBAGENT_MODEL", "CLAUDE_CODE_EFFORT_LEVEL"} + +// runOptimize asks an advisor model about each Claude agent under the root +// and reports its advice. It writes nothing. +func runOptimize(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("optimize", flag.ContinueOnError) + fs.SetOutput(stderr) + fs.Usage = func() { fmt.Fprint(stderr, usage) } + + rootF := registerRoot(fs) + repoPath := fs.String("repo", "", "canonical agent repo") + asJSON := fs.Bool("json", false, "emit JSON") + advisor := fs.String("advisor", defaultAdvisor, "OpenRouter decision model to ask") + maxCost := fs.Float64("max-cost", 0.50, "most this run may spend, in USD") + minConf := fs.Float64("min-confidence", 0.6, "least probability a pick needs to be suggested") + var names stringList + fs.Var(&names, "agent", "ask only about this agent (repeatable)") + + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return exitOK + } + return exitFailure + } + if err := optimizeFlagsErr(fs, *advisor, *maxCost, *minConf); err != nil { + fmt.Fprintf(stderr, "agentsync: %v\n", err) + return exitFailure + } + key := strings.TrimSpace(os.Getenv(envOpenRouterKey)) + if key == "" { + fmt.Fprintf(stderr, "agentsync: optimize needs an OpenRouter API key in $%s\n", envOpenRouterKey) + return exitFailure + } + + repo, repoFrom, err := openRepo(*repoPath) + if err != nil { + fmt.Fprintf(stderr, "agentsync: %v\n", err) + return exitFailure + } + root, err := rootF.resolve(repo.Home) + if err != nil { + fmt.Fprintf(stderr, "agentsync: %v\n", err) + return exitFailure + } + res, err := repo.Scan(root.Path) + if err == nil { + err = scanProblemsErr(res.Problems) + } + var agents []optimize.Agent + if err == nil { + agents, err = optimizeAgents(res, names) + } + if err != nil { + fmt.Fprintf(stderr, "agentsync: %v\n", err) + return exitFailure + } + + client := openrouter.New(key) + client.BaseURL = openrouterURL + ctx := context.Background() + cfg := optimize.Config{Advisor: *advisor, MaxCost: *maxCost, MinConfidence: *minConf} + models, err := client.Models(ctx) + if err == nil { + cfg.Price, err = optimize.InputPrice(models, *advisor) + } + if err == nil { + if cfg.Families = optimize.Available(models); len(cfg.Families) == 0 { + err = errors.New("the OpenRouter models list has no anthropic/claude-- model, so there is nothing to suggest") + } + } + if err != nil { + fmt.Fprintf(stderr, "agentsync: OpenRouter models list: %v\n", err) + return exitFailure + } + + notes := reportStream(*asJSON, stdout, stderr) + emitRepoChoice(notes, repo, repoFrom) + emitRootChoice(notes, repo.Home, root) + var overrides []string + for _, v := range claudeOverrides { + if os.Getenv(v) != "" { + overrides = append(overrides, v) + fmt.Fprintf(notes, "Note: $%s is set; Claude Code uses it over the frontmatter at run time.\n", v) + } + } + + rep := optimize.Run(ctx, client, agents, cfg) + rep.Overrides = overrides + if *asJSON { + enc := json.NewEncoder(stdout) + enc.SetIndent("", " ") + if err := enc.Encode(rep); err != nil { + fmt.Fprintf(stderr, "agentsync: %v\n", err) + return exitFailure + } + } else { + emitOptimizeText(stdout, rep, cfg) + } + if rep.Count(optimize.StatusUnresolved)+rep.Count(optimize.StatusSkipped) > 0 { + return exitAtRisk + } + return exitOK +} + +func optimizeFlagsErr(fs *flag.FlagSet, advisor string, maxCost, minConf float64) error { + switch { + case fs.NArg() > 0: + return fmt.Errorf("optimize takes no arguments, got %q", fs.Arg(0)) + case strings.TrimSpace(advisor) == "": + return errors.New("--advisor is empty") + case !(maxCost >= 0) || math.IsInf(maxCost, 1): + return fmt.Errorf("--max-cost must be a finite amount of USD at or above 0, got %v", maxCost) + case !(minConf >= 0 && minConf <= 1): + return fmt.Errorf("--min-confidence must be between 0 and 1, got %v", minConf) + } + return nil +} + +// scanProblemsErr refuses a scan that could not read everything: an agent it +// missed is one the report would silently leave out. +func scanProblemsErr(problems []scan.ScanProblem) error { + if len(problems) == 0 { + return nil + } + lines := make([]string, len(problems)) + for i, p := range problems { + lines[i] = " " + p.String() + } + return fmt.Errorf("could not read every agent under the root:\n%s", strings.Join(lines, "\n")) +} + +// optimizeAgents reads the Claude agents scan found, once per real file, so +// an agent linked into many projects costs one request. +func optimizeAgents(res scan.Result, names []string) ([]optimize.Agent, error) { + want := map[string]bool{} + for _, n := range names { + want[n] = false + } + seen := map[string]bool{} + var out []optimize.Agent + for _, e := range res.Entries { + if _, ok := want[e.Name]; len(names) > 0 && !ok { + continue + } + data, err := os.ReadFile(e.Path) + if err != nil { + return nil, fmt.Errorf("could not read agent %s (%w); agentsync doctor reports broken links", e.Path, err) + } + if !discover.IsAgent(string(data)) { + continue + } + want[e.Name] = true + real := e.Path + if r, err := filepath.EvalSymlinks(e.Path); err == nil { + real = r + } + if seen[real] { + continue + } + seen[real] = true + label := e.Name + if len(res.Dirs) > 1 { + label = e.Project + "/" + e.Name + } + out = append(out, optimize.Agent{Label: label, Path: e.Path, Content: string(data)}) + } + for _, n := range names { + if !want[n] { + return nil, fmt.Errorf("--agent %s: no Claude agent by that name under the root", n) + } + } + return out, nil +} + +func emitOptimizeText(w io.Writer, rep optimize.Report, cfg optimize.Config) { + skipped := rep.Count(optimize.StatusSkipped) + asked := len(rep.Rows) - skipped + fmt.Fprintf(w, "Asked %s about %d agent%s", rep.Advisor, asked, plural(asked, "", "s")) + if skipped > 0 { + fmt.Fprintf(w, ", %d skipped", skipped) + } + fmt.Fprintf(w, " (cap $%.2f, min confidence %.2f).\n\n", cfg.MaxCost, cfg.MinConfidence) + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + // Typed as io.Writer so errcheck's report-writer exemption applies. + var table io.Writer = tw + fmt.Fprintln(table, "AGENT\tNOW\tSUGGESTED\tCONFIDENCE") + for _, r := range rep.Rows { + fmt.Fprintf(table, "%s\t%s\t%s\t%s\n", r.Agent, r.Now(), r.Suggested(), r.Confidence()) + } + _ = tw.Flush() + fmt.Fprintln(w) + for _, r := range rep.Rows { + if r.Reason != "" { + fmt.Fprintf(w, "%s: %s: %s\n", r.Agent, r.Status, r.Reason) + } + } + if skipped > 0 { + fmt.Fprintf(w, "%d agent%s skipped (reasons above).\n", skipped, plural(skipped, " was", "s were")) + } + fmt.Fprintf(w, "Spent $%.6f (reported by OpenRouter). Dry run: nothing written.\n", rep.Spent) +} diff --git a/cmd/agentsync/optimize_test.go b/cmd/agentsync/optimize_test.go new file mode 100644 index 0000000..ad74ae9 --- /dev/null +++ b/cmd/agentsync/optimize_test.go @@ -0,0 +1,331 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + + "github.com/agentic-tooling/agentsync/internal/optimize" +) + +const orTestdata = "../../internal/openrouter/testdata/" + +func orFixture(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(orTestdata + name) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func agentFile(name, model, effort string) string { + return "---\nname: " + name + "\ndescription: d\nmodel: " + model + "\neffort: " + effort + "\n---\nbody\n" +} + +// fakeOpenRouter serves models (status 200 unless modelsStatus says +// otherwise) and the decisions in order, and points optimize at itself with +// testKey set. It counts both kinds of request. +type fakeOpenRouter struct { + models string + modelsStatus int + decisions []string + mu sync.Mutex + listed, sent int +} + +const testKey = "sk-test-never-printed" + +func assertNoKey(t *testing.T, out ...string) { + t.Helper() + if s := strings.Join(out, ""); strings.Contains(s, testKey) { + t.Errorf("output leaks the key:\n%s", s) + } +} + +func (f *fakeOpenRouter) start(t *testing.T) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + if r.Method == http.MethodGet { + f.listed++ + if f.modelsStatus != 0 { + w.WriteHeader(f.modelsStatus) + } + _, _ = io.WriteString(w, f.models) + return + } + f.sent++ + if f.sent > len(f.decisions) { + w.WriteHeader(http.StatusTeapot) + return + } + _, _ = io.WriteString(w, f.decisions[f.sent-1]) + })) + t.Cleanup(srv.Close) + old := openrouterURL + openrouterURL = srv.URL + t.Cleanup(func() { openrouterURL = old }) + t.Setenv(envOpenRouterKey, testKey) +} + +// requests is how many decision requests arrived. +func (f *fakeOpenRouter) requests() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.sent +} + +func (f *fakeOpenRouter) lists() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.listed +} + +// optimizeProject makes a repo and a project whose agents dir holds files, +// and returns both paths. +func optimizeProject(t *testing.T, files map[string]string) (project, repoRoot string) { + t.Helper() + home := mustResolveDir(t, t.TempDir()) + t.Setenv("HOME", home) + repoRoot = filepath.Join(home, "Web", "agents-repo") + mustMkdirT(t, filepath.Join(repoRoot, "agents")) + project = filepath.Join(home, "Web", "proj") + mustMkdirT(t, filepath.Join(project, ".claude", "agents")) + for name, content := range files { + mustWriteT(t, filepath.Join(project, ".claude", "agents", name), content) + } + return project, repoRoot +} + +func TestOptimize_Report(t *testing.T) { + project, repoRoot := optimizeProject(t, map[string]string{ + "go-merge-reviewer.md": agentFile("go-merge-reviewer", "opus", "xhigh"), + "README.md": "not an agent\n", + }) + f := &fakeOpenRouter{models: orFixture(t, "models.json"), decisions: []string{orFixture(t, "decision-jev.json")}} + f.start(t) + t.Setenv("CLAUDE_CODE_EFFORT_LEVEL", "low") + + code, stdout, stderr := runCLI(t, "optimize", "--root", project, "--repo", repoRoot) + if code != exitOK { + t.Fatalf("exit = %d, want %d\n%s", code, exitOK, stderr) + } + for _, w := range []string{ + "Repo: ~/Web/agents-repo (from --repo).", + "Root: ~/Web/proj (from --root).", + "Note: $CLAUDE_CODE_EFFORT_LEVEL is set", + "Asked ~typesafe/jev-latest about 1 agent (cap $0.50, min confidence 0.60).", + "AGENT NOW SUGGESTED CONFIDENCE", + "go-merge-reviewer opus / xhigh keep (low confidence) sonnet 0.57 / xhigh 0.89", + "Spent $0.000082 (reported by OpenRouter). Dry run: nothing written.", + } { + if !strings.Contains(stdout, w) { + t.Errorf("report is missing %q:\n%s", w, stdout) + } + } + if strings.Contains(stdout, "SUBAGENT_MODEL") { + t.Errorf("unexpected note:\n%s", stdout) + } + assertNoKey(t, stdout, stderr) + if f.requests() != 1 { + t.Errorf("%d decision requests, want 1 (README.md is not an agent)", f.requests()) + } +} + +func TestOptimize_JSON(t *testing.T) { + project, repoRoot := optimizeProject(t, map[string]string{"a.md": agentFile("a", "sonnet", "high")}) + f := &fakeOpenRouter{models: orFixture(t, "models.json"), decisions: []string{orFixture(t, "decision-clef.json")}} + f.start(t) + t.Setenv("CLAUDE_CODE_SUBAGENT_MODEL", "haiku") + + code, stdout, stderr := runCLI(t, "optimize", "--json", "--advisor", "cloudflare/clef", "--root", project, "--repo", repoRoot) + if code != exitOK { + t.Fatalf("exit = %d\n%s", code, stderr) + } + var rep optimize.Report + if err := json.Unmarshal([]byte(stdout), &rep); err != nil { + t.Fatalf("stdout is not JSON: %v\n%s", err, stdout) + } + if rep.Advisor != "cloudflare/clef" || len(rep.Rows) != 1 || rep.Rows[0].SuggestedModel != "opus" || rep.Spent != 0.00045312 { + t.Errorf("report = %+v", rep) + } + if !reflect.DeepEqual(rep.Overrides, []string{"CLAUDE_CODE_SUBAGENT_MODEL"}) { + t.Errorf("overrides = %v", rep.Overrides) + } + for _, w := range []string{"Repo: ", "Root: ", "Note: $CLAUDE_CODE_SUBAGENT_MODEL is set"} { + if !strings.Contains(stderr, w) { + t.Errorf("%q belongs on stderr with --json:\n%s", w, stderr) + } + } +} + +func TestOptimize_JSONWriteFails(t *testing.T) { + project, repoRoot := optimizeProject(t, map[string]string{"a.md": agentFile("a", "opus", "high")}) + (&fakeOpenRouter{models: orFixture(t, "models.json"), decisions: []string{orFixture(t, "decision-clef.json")}}).start(t) + var stderr bytes.Buffer + if code := runOptimize([]string{"--json", "--root", project, "--repo", repoRoot}, failWriter{}, &stderr); code != exitFailure { + t.Errorf("exit = %d, want %d\n%s", code, exitFailure, stderr.String()) + } +} + +type failWriter struct{} + +func (failWriter) Write([]byte) (int, error) { return 0, errors.New("closed") } + +func TestOptimize_UnresolvedAndSkippedExitOne(t *testing.T) { + project, repoRoot := optimizeProject(t, map[string]string{ + "a.md": agentFile("a", "opus", "high"), + "b.md": agentFile("b", "opus", "high"), + }) + f := &fakeOpenRouter{models: orFixture(t, "models.json"), decisions: []string{`{"answers":{}}`}} + f.start(t) + t.Setenv("CLAUDE_CODE_SUBAGENT_MODEL", "haiku") + + code, stdout, stderr := runCLI(t, "optimize", "--root", project, "--repo", repoRoot) + if code != exitAtRisk { + t.Errorf("exit = %d, want %d", code, exitAtRisk) + } + for _, w := range []string{ + "Asked ~typesafe/jev-latest about 1 agent, 1 skipped (cap", + "a: unresolved: response reported no valid usage.cost", + "b: skipped: run stopped", + "1 agent was skipped (reasons above).", + "Note: $CLAUDE_CODE_SUBAGENT_MODEL is set", + } { + if !strings.Contains(stdout, w) { + t.Errorf("report is missing %q:\n%s", w, stdout) + } + } + assertNoKey(t, stdout, stderr) +} + +func TestOptimize_AgentSelectionAndDedup(t *testing.T) { + home := mustResolveDir(t, t.TempDir()) + t.Setenv("HOME", home) + repoRoot := filepath.Join(home, "Web", "agents-repo") + shared := filepath.Join(repoRoot, "agents", "shared.md") + mustWriteT(t, shared, agentFile("shared", "opus", "high")) + for _, p := range []string{"one", "two"} { + dir := filepath.Join(home, "Web", p, ".claude", "agents") + mustMkdirT(t, dir) + if err := os.Symlink(shared, filepath.Join(dir, "shared.md")); err != nil { + t.Fatal(err) + } + mustWriteT(t, filepath.Join(dir, "own.md"), agentFile("own", "opus", "high")) + } + clef := orFixture(t, "decision-clef.json") + f := &fakeOpenRouter{models: orFixture(t, "models.json"), decisions: []string{clef, clef, clef, clef}} + f.start(t) + web := filepath.Join(home, "Web") + + // The shared file is asked about once, under the first project linking it. + code, stdout, stderr := runCLI(t, "optimize", "--root", web, "--repo", repoRoot) + if code != exitOK { + t.Fatalf("exit = %d\n%s", code, stderr) + } + for _, w := range []string{"Web/one/own ", "Web/one/shared ", "Web/two/own "} { + if !strings.Contains(stdout, w) { + t.Errorf("report is missing %q:\n%s", w, stdout) + } + } + if f.requests() != 3 || strings.Contains(stdout, "Web/two/shared") { + t.Errorf("%d requests, want 3 with the shared file once:\n%s", f.requests(), stdout) + } + + code, stdout, stderr = runCLI(t, "optimize", "--agent", "shared", "--root", web, "--repo", repoRoot) + if code != exitOK || f.requests() != 4 || strings.Contains(stdout, "own") { + t.Errorf("--agent shared: exit %d, %d requests in all, want 4:\n%s%s", code, f.requests(), stdout, stderr) + } + + code, _, stderr = runCLI(t, "optimize", "--agent", "shared", "--agent", "nope", "--root", web, "--repo", repoRoot) + if code != exitFailure || !strings.Contains(stderr, "--agent nope: no Claude agent by that name") { + t.Errorf("exit = %d, stderr:\n%s", code, stderr) + } +} + +func TestOptimize_RefusesBeforeAnyRequest(t *testing.T) { + good := map[string]string{"a.md": agentFile("a", "opus", "high")} + for _, tc := range []struct { + name string + files map[string]string + setup func(t *testing.T, project string) + fake func(f *fakeOpenRouter) + args []string + code int + stderr string + lists int // models-list fetches; zero for every refusal before it + }{ + {name: "help", args: []string{"-h"}, code: exitOK}, + {name: "unknown flag", args: []string{"--nope"}, code: exitFailure, stderr: "flag provided but not defined"}, + {name: "positional", args: []string{"extra"}, code: exitFailure, stderr: `optimize takes no arguments, got "extra"`}, + {name: "empty advisor", args: []string{"--advisor", " "}, code: exitFailure, stderr: "--advisor is empty"}, + {name: "negative cap", args: []string{"--max-cost", "-1"}, code: exitFailure, stderr: "--max-cost must be"}, + {name: "infinite cap", args: []string{"--max-cost", "Inf"}, code: exitFailure, stderr: "--max-cost must be"}, + {name: "confidence above 1", args: []string{"--min-confidence", "1.5"}, code: exitFailure, stderr: "--min-confidence must be between 0 and 1"}, + {name: "no key", setup: func(t *testing.T, _ string) { t.Setenv(envOpenRouterKey, "") }, + code: exitFailure, stderr: "$OPENROUTER_API_KEY"}, + {name: "no repo", setup: func(t *testing.T, _ string) { t.Setenv(envRepo, "") }, + args: []string{"--repo", ""}, code: exitFailure, stderr: "no canonical agent repo configured"}, + {name: "root and all", args: []string{"--all"}, code: exitFailure, stderr: "mutually exclusive"}, + {name: "root missing", setup: func(t *testing.T, project string) { mustRemoveAll(t, project) }, + code: exitFailure, stderr: "walking"}, + {name: "scan problem", files: map[string]string{"dir.md/x": "x"}, code: exitFailure, stderr: "could not read every agent under the root"}, + {name: "broken link", setup: func(t *testing.T, project string) { + if err := os.Symlink(filepath.Join(project, "gone.md"), filepath.Join(project, ".claude", "agents", "b.md")); err != nil { + t.Fatal(err) + } + }, code: exitFailure, stderr: "could not read agent"}, + {name: "models list down", fake: func(f *fakeOpenRouter) { f.modelsStatus = 500 }, + code: exitFailure, stderr: "OpenRouter models list: OpenRouter HTTP 500", lists: 1}, + {name: "advisor unpriced", args: []string{"--advisor", "typesafe/jev-router"}, code: exitFailure, stderr: "no fixed input price", lists: 1}, + {name: "no claude family", fake: func(f *fakeOpenRouter) { + f.models = `{"data":[{"id":"~typesafe/jev-latest","pricing":{"prompt":"0.000000042"}}]}` + }, code: exitFailure, stderr: "has no anthropic/claude-- model", lists: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + files := tc.files + if files == nil { + files = good + } + project, repoRoot := optimizeProject(t, files) + f := &fakeOpenRouter{models: orFixture(t, "models.json")} + if tc.fake != nil { + tc.fake(f) + } + f.start(t) + if tc.setup != nil { + tc.setup(t, project) + } + args := append([]string{"optimize", "--root", project, "--repo", repoRoot}, tc.args...) + code, stdout, stderr := runCLI(t, args...) + if code != tc.code || !strings.Contains(stderr, tc.stderr) { + t.Errorf("exit = %d, want %d; stderr missing %q:\n%s", code, tc.code, tc.stderr, stderr) + } + if code != exitOK && stdout != "" { + t.Errorf("a refused run printed a report:\n%s", stdout) + } + if f.requests() != 0 || f.lists() != tc.lists { + t.Errorf("%d decision requests and %d models fetches; want 0 and %d", f.requests(), f.lists(), tc.lists) + } + assertNoKey(t, stdout, stderr) + }) + } +} + +func mustRemoveAll(t *testing.T, p string) { + t.Helper() + if err := os.RemoveAll(p); err != nil { + t.Fatal(err) + } +} diff --git a/coverage-floors.txt b/coverage-floors.txt index 54e23b4..f522875 100644 --- a/coverage-floors.txt +++ b/coverage-floors.txt @@ -1,12 +1,14 @@ # Minimum per-package statement coverage, checked by `just check`. # Raise a floor after a coverage pass; never lower one. -cmd/agentsync 91.2 +cmd/agentsync 92.0 internal/config 100.0 internal/discover 100.0 internal/doctor 88.1 internal/importer 88.8 internal/inventory 91.2 internal/linker 94.9 +internal/openrouter 100.0 +internal/optimize 100.0 internal/reconcile 84.3 internal/releasepolicy 96.4 internal/scan 91.6 diff --git a/docs/exit-codes.md b/docs/exit-codes.md index f26a85b..1fd43d3 100644 --- a/docs/exit-codes.md +++ b/docs/exit-codes.md @@ -17,6 +17,7 @@ What counts as exit 1, by command: | `doctor` | a broken or mismatched link, a stale capture, an orphaned capture directory, or an unreadable path | | `reconcile` | a refused selector, a conflict, a diverged or dereferenced link, or a removable link left in place | | `inventory --check` | the file is stale or absent | +| `optimize` | an agent's advice is unresolved, or it was skipped by the cost cap or a stopped run | `doctor`'s exit code is the one that asserts nothing has drifted; `reconcile`'s is deliberately narrower (see [reconcile](reconcile.md)). Tests assert these diff --git a/docs/optimize.md b/docs/optimize.md new file mode 100644 index 0000000..b244825 --- /dev/null +++ b/docs/optimize.md @@ -0,0 +1,121 @@ +# optimize + +`agentsync optimize` asks an OpenRouter decision model which Claude model and +reasoning effort each of a project's Claude agents should run on, and reports +the advice. It writes nothing. It is the only agentsync command that uses the +network. + +```bash +export OPENROUTER_API_KEY=... # required +agentsync optimize # the project you are in +agentsync optimize --agent go-reviewer # one agent +agentsync optimize --advisor cloudflare/clef --max-cost 0.05 +agentsync optimize --json +``` + +Use a dedicated OpenRouter key with a credit limit set on OpenRouter's side. +That limit is the backstop if anything here undercounts. + +## What it reads and sends + +The agents are the ones [scan](scan.md) finds under the root (the project's +`.claude/agents/`, linked or local); root and repo resolve as for every other +command (see [configuration](configuration.md)). An agent file linked into +several projects is asked about once. A `.md` file without `name` and +`description` frontmatter is not an agent and is skipped. + +For each agent it shows `model:` and `effort:` from the frontmatter as written +(without quotes or a trailing `# comment`), or `(not set)`. If `$CLAUDE_CODE_SUBAGENT_MODEL` or `$CLAUDE_CODE_EFFORT_LEVEL` +is set, it notes that Claude Code uses it over the frontmatter at run time. + +Each agent costs one `POST /api/alpha/decisions` request, with no retries. The +request holds one framing sentence, the first 24 KB of the agent file and two +questions: + +- `model`: a choice among the Claude Code aliases `haiku`, `sonnet`, `opus` and + `fable`, each with a one-line description. An alias is offered only if + OpenRouter's public models list has an `anthropic/claude--` + model. Claude Code resolves an alias to the newest model in its family. +- `effort`: a score over `low`, `medium`, `high`, `xhigh`, `max`, read as the + nearest level. Its confidence is the probability the advisor gave that level, + not the score's own `confidence` field. + +Nothing else from the project is sent. + +## Flags + +| Flag | Meaning | +|---|---| +| `--advisor MODEL` | the decision model to ask; default `~typesafe/jev-latest`, and `cloudflare/clef` works too | +| `--max-cost USD` | the most the run may spend; default `0.50` | +| `--min-confidence P` | the probability a pick needs before it is suggested; default `0.6` | +| `--agent NAME` | ask only about this agent; repeatable | +| `--json` | print the report as JSON on stdout (see below), with the Repo, Root and override notes on stderr | + +## Reading the report + +``` +AGENT NOW SUGGESTED CONFIDENCE +go-merge-reviewer opus / xhigh keep (low confidence) sonnet 0.57 / xhigh 0.89 +``` + +Model and effort are judged separately. A part is suggested only when the +advisor's pick differs from the current value and its probability is at least +`--min-confidence`. A differing pick under the threshold is dropped: a row that +still changes the other part names it, as in `opus / xhigh (effort: low +confidence)`, and a row with nothing left to change says `keep (low +confidence)`. `CONFIDENCE` shows both picks whether or not they were suggested. + +A row is `unresolved` when the request failed (OpenRouter's error message is +shown, including a guardrail's reason and settings link), the response was not +valid JSON, an answer or its probability was missing, or the advisor picked +something it was not offered. An unresolved row suggests nothing. + +## JSON + +`--json` prints one object: `advisor`, `rows`, `spent_usd`, and `overrides` +(the `$CLAUDE_CODE_*` variables that are set; omitted when none). Each row has: + +| Field | Meaning | +|---|---| +| `agent`, `path` | the agent's label and file | +| `model`, `effort` | the current values, or `(not set)` | +| `status` | `change`, `keep`, `unresolved` or `skipped` | +| `suggested_model`, `suggested_effort` | set only for a part that changes | +| `model_pick`, `effort_pick` | `{value, probability}` as the advisor answered; absent unless both were read | +| `model_low_confidence`, `effort_low_confidence` | `true` when that part's differing pick was dropped; omitted otherwise | +| `reason` | why a row is unresolved or skipped | +| `cost_usd` | the `usage.cost` OpenRouter reported for this row | + +## Cost + +The advisor's input price comes from OpenRouter's public models list; an +advisor that is not listed, or lists no fixed price, is refused before any +request. Before each request the cost is estimated as request bytes / 4 times +that price, or at the highest cost per request byte OpenRouter has reported so +far in the run if that is higher. The request is skipped if the estimate would +take the spend so far past `--max-cost`. Spend so far is the sum of the +`usage.cost` OpenRouter reports on each response. + +The run stops, and the remaining agents are skipped, whenever a request may +have been billed without a usable cost: a response with no or a negative +`usage.cost`, a successful response that was cut short or was not valid JSON, +or a connection error or timeout after sending. The row it happened on is +unresolved. An error status from OpenRouter is not billed and does not stop +the run. The last line prints the total reported spend. + +A model's billed price can differ from its listed one. The estimate learns +the billed rate from each response, so it is mostly the first request that +can be underestimated. The cap still holds to within one request, because +spend is what OpenRouter reports. + +## Exit codes + +- `0`: every agent got advice. +- `1`: a row is unresolved or skipped. +- `2`: no `$OPENROUTER_API_KEY`, bad flags, no repo or root configured, a scan + that could not read everything, an agent file that could not be read, or a + models list that is unreachable, has no fixed price for the advisor, or has + no Claude family. + +See [exit codes](exit-codes.md). diff --git a/internal/discover/discover.go b/internal/discover/discover.go index 124cf3f..5bef568 100644 --- a/internal/discover/discover.go +++ b/internal/discover/discover.go @@ -124,7 +124,7 @@ func agentCount(dir string) (int, error) { if err == nil && e.Type().IsRegular() && filepath.Ext(e.Name()) == ".md" { var data []byte data, err = os.ReadFile(filepath.Join(dir, "agents", e.Name())) - if err == nil && isAgent(string(data)) { + if err == nil && IsAgent(string(data)) { n++ } } @@ -132,21 +132,36 @@ func agentCount(dir string) (int, error) { return n, err } -// isAgent reports whether s opens with YAML frontmatter carrying top-level +// IsAgent reports whether s opens with YAML frontmatter carrying top-level // name and description keys, which is what Claude Code requires of an agent. -func isAgent(s string) bool { +func IsAgent(s string) bool { + fm, ok := Frontmatter(s) + _, name := fm["name"] + _, desc := fm["description"] + return ok && name && desc +} + +// Frontmatter returns the top-level keys of s's YAML frontmatter with their +// raw values trimmed, and false when s has none. Nested and list lines are +// skipped; the first occurrence of a key wins. +func Frontmatter(s string) (map[string]string, bool) { front, ok := strings.CutPrefix(strings.ReplaceAll(s, "\r\n", "\n"), "---\n") if !ok { - return false + return nil, false } front, _, ok = strings.Cut(front, "\n---") if !ok { - return false + return nil, false } - var name, desc bool + fm := map[string]string{} for _, line := range strings.Split(front, "\n") { - name = name || strings.HasPrefix(line, "name:") - desc = desc || strings.HasPrefix(line, "description:") + key, value, found := strings.Cut(line, ":") + if !found || key == "" || strings.ContainsAny(key[:1], " \t-#") { + continue + } + if _, dup := fm[key]; !dup { + fm[key] = strings.TrimSpace(value) + } } - return name && desc + return fm, true } diff --git a/internal/discover/discover_test.go b/internal/discover/discover_test.go index f601d7e..da152f8 100644 --- a/internal/discover/discover_test.go +++ b/internal/discover/discover_test.go @@ -284,13 +284,25 @@ func TestFind_StopsAtTheLimit(t *testing.T) { func TestIsAgent(t *testing.T) { for _, s := range []string{agentMD, "---\ndescription: d\nname: n\n---"} { - if !isAgent(s) { - t.Errorf("isAgent(%q) = false", s) + if !IsAgent(s) { + t.Errorf("IsAgent(%q) = false", s) } } for _, s := range []string{"", "---\n", "---\nname: n\n---\n", " ---\nname: n\ndescription: d\n---\n"} { - if isAgent(s) { - t.Errorf("isAgent(%q) = true", s) + if IsAgent(s) { + t.Errorf("IsAgent(%q) = true", s) } } } + +func TestFrontmatter(t *testing.T) { + s := "---\r\nname: n\r\nmodel: opus \r\nmodel: sonnet\r\n effort: max\r\n- effort: low\r\n# effort: low\r\n: x\r\ncolor\r\neffort:\r\ntools: Read # and Grep\r\n---\r\nmodel: haiku\r\n" + fm, ok := Frontmatter(s) + want := map[string]string{"name": "n", "model": "opus", "effort": "", "tools": "Read # and Grep"} + if !ok || !reflect.DeepEqual(fm, want) { + t.Errorf("Frontmatter = %v, %v; want %v, true", fm, ok, want) + } + if _, ok := Frontmatter("model: opus\n"); ok { + t.Error("Frontmatter found frontmatter in a file without any") + } +} diff --git a/internal/openrouter/openrouter.go b/internal/openrouter/openrouter.go new file mode 100644 index 0000000..27155d7 --- /dev/null +++ b/internal/openrouter/openrouter.go @@ -0,0 +1,174 @@ +// Package openrouter is the small slice of OpenRouter's API that optimize +// needs: the public models list and the alpha Decisions endpoint. +package openrouter + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// DefaultBaseURL is OpenRouter's API host. Tests point Client.BaseURL at an +// httptest server instead. +const DefaultBaseURL = "https://openrouter.ai" + +// maxBody bounds what is read from any response; the full models list is +// under 1 MB. +const maxBody = 16 << 20 + +// Client calls OpenRouter. Key is sent only on Decide; the models list is +// public. +type Client struct { + BaseURL string + Key string + HTTP *http.Client +} + +// New returns a client for the real API with a per-request timeout. +func New(key string) *Client { + return &Client{BaseURL: DefaultBaseURL, Key: key, HTTP: &http.Client{Timeout: 60 * time.Second}} +} + +// Model is one entry of the models list. Prices are USD per token, as the +// decimal strings OpenRouter sends; "-1" means variable. +type Model struct { + ID string `json:"id"` + Pricing struct { + Prompt string `json:"prompt"` + } `json:"pricing"` +} + +// Question is one typed question. Criteria is a map of option to description +// for "choice" and an ordered list of levels for "score". +type Question struct { + Type string `json:"type"` + Instructions string `json:"instructions"` + Criteria any `json:"criteria"` +} + +// DecisionRequest is the body of POST /api/alpha/decisions. +type DecisionRequest struct { + Model string `json:"model"` + State []string `json:"state"` + Questions map[string]Question `json:"questions"` +} + +// Answer is one question's answer. Pointers distinguish a missing field from +// a zero, which callers must refuse rather than read as an answer. +type Answer struct { + Type string `json:"type"` + Choice string `json:"choice"` + Score *float64 `json:"score"` + Probabilities map[string]float64 `json:"probabilities"` +} + +// Decision is a Decisions response. +type Decision struct { + Model string `json:"model"` + Answers map[string]Answer `json:"answers"` + Usage struct { + Cost *float64 `json:"cost"` + } `json:"usage"` +} + +// APIError is a non-2xx response, carrying OpenRouter's own message. +type APIError struct { + Status int + Message string + Metadata map[string]any +} + +func (e *APIError) Error() string { + msg := fmt.Sprintf("OpenRouter HTTP %d: %s", e.Status, e.Message) + // A guardrail refusal names why and where to change it; both are the fix. + if r, ok := e.Metadata["ineligibility_reasons"]; ok { + if b, err := json.Marshal(r); err == nil { + msg += "; reasons: " + string(b) + } + } + if u, ok := e.Metadata["configure_url"].(string); ok { + msg += "; configure at " + u + } + return msg +} + +// Models fetches the public models list. output_modalities=all is needed: +// the default list leaves out decision models such as Jev and Clef. +func (c *Client) Models(ctx context.Context) ([]Model, error) { + var out struct { + Data []Model `json:"data"` + } + if err := c.do(ctx, http.MethodGet, "/api/v1/models?output_modalities=all", nil, &out); err != nil { + return nil, err + } + if len(out.Data) == 0 { + return nil, fmt.Errorf("OpenRouter models list is empty") + } + return out.Data, nil +} + +// Decide posts one already-encoded DecisionRequest. The caller encodes it so +// the bytes it estimated the cost of are the bytes sent. +func (c *Client) Decide(ctx context.Context, body []byte) (Decision, error) { + var d Decision + err := c.do(ctx, http.MethodPost, "/api/alpha/decisions", body, &d) + return d, err +} + +func (c *Client) do(ctx context.Context, method, path string, body []byte, out any) error { + var rd io.Reader + if body != nil { + rd = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, c.BaseURL+path, rd) + if err != nil { + return err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+c.Key) + } + resp, err := c.HTTP.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + data, err := io.ReadAll(io.LimitReader(resp.Body, maxBody)) + if err != nil { + return fmt.Errorf("reading OpenRouter response: %w", err) + } + if resp.StatusCode/100 != 2 { + return apiError(resp.StatusCode, data) + } + if err := json.Unmarshal(data, out); err != nil { + return fmt.Errorf("OpenRouter sent malformed JSON: %w", err) + } + return nil +} + +func apiError(status int, data []byte) error { + var body struct { + Error struct { + Message string `json:"message"` + Metadata map[string]any `json:"metadata"` + } `json:"error"` + } + e := &APIError{Status: status} + if json.Unmarshal(data, &body) == nil && body.Error.Message != "" { + e.Message, e.Metadata = body.Error.Message, body.Error.Metadata + return e + } + e.Message = strings.TrimSpace(string(data)) + if len(e.Message) > 200 { + e.Message = e.Message[:200] + "..." + } + if e.Message == "" { + e.Message = http.StatusText(status) + } + return e +} diff --git a/internal/openrouter/openrouter_test.go b/internal/openrouter/openrouter_test.go new file mode 100644 index 0000000..c3f60f6 --- /dev/null +++ b/internal/openrouter/openrouter_test.go @@ -0,0 +1,207 @@ +package openrouter + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" +) + +func fixture(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile("testdata/" + name) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// serve answers every request with status and body. last returns the most +// recent request; the mutex is what the race detector sees, not the socket. +func serve(t *testing.T, status int, body string) (c *Client, last func() *http.Request) { + t.Helper() + var ( + mu sync.Mutex + got *http.Request + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + got = r.Clone(context.Background()) + mu.Unlock() + w.WriteHeader(status) + _, _ = io.WriteString(w, body) + })) + t.Cleanup(srv.Close) + c = New("sk-test") + c.BaseURL = srv.URL + return c, func() *http.Request { mu.Lock(); defer mu.Unlock(); return got } +} + +func TestModels_ParsesTheRealListAndSendsNoKey(t *testing.T) { + c, last := serve(t, 200, fixture(t, "models.json")) + models, err := c.Models(context.Background()) + if err != nil { + t.Fatal(err) + } + got := last() + if got.URL.String() != "/api/v1/models?output_modalities=all" || got.Method != http.MethodGet { + t.Errorf("request = %s %s", got.Method, got.URL) + } + if h := got.Header.Get("Authorization"); h != "" { + t.Errorf("the public list was sent Authorization %q", h) + } + var jev string + for _, m := range models { + if m.ID == "~typesafe/jev-latest" { + jev = m.Pricing.Prompt + } + } + if jev != "0.000000042" { + t.Errorf("jev-latest prompt price = %q, want 0.000000042", jev) + } +} + +func TestModels_EmptyListRefuses(t *testing.T) { + c, _ := serve(t, 200, `{"data":[]}`) + if _, err := c.Models(context.Background()); err == nil || !strings.Contains(err.Error(), "empty") { + t.Errorf("err = %v, want an empty-list refusal", err) + } +} + +func TestDecide_ParsesRealResponses(t *testing.T) { + for _, tc := range []struct { + file, choice string + prob, cost float64 + }{ + {"decision-jev.json", "sonnet", 0.57, 0.000082152}, + {"decision-clef.json", "opus", 0.668, 0.00045312}, + } { + t.Run(tc.file, func(t *testing.T) { + c, last := serve(t, 200, fixture(t, tc.file)) + d, err := c.Decide(context.Background(), []byte(`{}`)) + if err != nil { + t.Fatal(err) + } + got := last() + if got.Header.Get("Authorization") != "Bearer sk-test" || got.URL.Path != "/api/alpha/decisions" || got.Method != http.MethodPost { + t.Errorf("request = %s %s auth %q", got.Method, got.URL, got.Header.Get("Authorization")) + } + m := d.Answers["model"] + if m.Choice != tc.choice || m.Probabilities[tc.choice] != tc.prob { + t.Errorf("model answer = %+v", m) + } + if e := d.Answers["effort"]; e.Score == nil || len(e.Probabilities) != 5 { + t.Errorf("effort answer lost its score or probabilities: %+v", e) + } + if d.Usage.Cost == nil || *d.Usage.Cost != tc.cost { + t.Errorf("usage.cost = %v, want %v", d.Usage.Cost, tc.cost) + } + }) + } +} + +func TestDecide_ErrorsCarryOpenRouterMessage(t *testing.T) { + guardrail := `{"error":{"message":"Provider not allowed by guardrail","code":404,"metadata":{"ineligibility_reasons":["provider anthropic blocked"],"configure_url":"https://openrouter.ai/settings/guardrails"}}}` + for _, tc := range []struct { + name string + status int + body string + want []string + }{ + {"unknown model", 400, fixture(t, "error-unknown-model.json"), []string{"HTTP 400", "Model openai/gpt-6-luna does not exist"}}, + {"guardrail", 404, guardrail, []string{"HTTP 404", "Provider not allowed by guardrail", "provider anthropic blocked", "configure at https://openrouter.ai/settings/guardrails"}}, + {"plain 5xx", 502, "bad gateway\n", []string{"HTTP 502: bad gateway"}}, + {"empty 5xx", 503, "", []string{"HTTP 503: Service Unavailable"}}, + {"long body", 500, strings.Repeat("x", 300), []string{strings.Repeat("x", 200) + "..."}}, + } { + t.Run(tc.name, func(t *testing.T) { + c, _ := serve(t, tc.status, tc.body) + _, err := c.Decide(context.Background(), []byte(`{}`)) + var apiErr *APIError + if !errors.As(err, &apiErr) || apiErr.Status != tc.status { + t.Fatalf("err = %v, want an APIError with status %d", err, tc.status) + } + for _, w := range tc.want { + if !strings.Contains(err.Error(), w) { + t.Errorf("error %q is missing %q", err, w) + } + } + if strings.Contains(err.Error(), "sk-test") { + t.Errorf("error leaks the key: %v", err) + } + }) + } +} + +func TestDecide_MalformedJSON(t *testing.T) { + c, _ := serve(t, 200, `{"answers":`) + if _, err := c.Decide(context.Background(), []byte(`{}`)); err == nil || !strings.Contains(err.Error(), "malformed JSON") { + t.Errorf("err = %v, want malformed JSON", err) + } +} + +func TestDecide_TimesOut(t *testing.T) { + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-release: + case <-r.Context().Done(): + } + })) + t.Cleanup(func() { close(release); srv.Close() }) + c := &Client{BaseURL: srv.URL, Key: "sk-test", HTTP: &http.Client{Timeout: 50 * time.Millisecond}} + _, err := c.Decide(context.Background(), []byte(`{}`)) + if err == nil || !strings.Contains(err.Error(), "Timeout") { + t.Errorf("err = %v, want a client timeout", err) + } + if err != nil && strings.Contains(err.Error(), "sk-test") { + t.Errorf("error leaks the key: %v", err) + } +} + +func TestNew_HasATimeout(t *testing.T) { + if New("").HTTP.Timeout <= 0 { + t.Error("New's client can wait forever") + } +} + +func TestDo_TransportAndRequestErrors(t *testing.T) { + c := &Client{BaseURL: "http://127.0.0.1:0", Key: "sk-test", HTTP: &http.Client{}} + if _, err := c.Models(context.Background()); err == nil { + t.Error("a refused connection returned no error") + } + _, err := c.Decide(context.Background(), []byte(`{}`)) + if err == nil || strings.Contains(err.Error(), "sk-test") { + t.Errorf("err = %v, want a refused connection that does not name the key", err) + } + c.BaseURL = "://bad" + if _, err := c.Models(context.Background()); err == nil { + t.Error("an unparseable URL returned no error") + } +} + +func TestDo_CapsTheBody(t *testing.T) { + c, _ := serve(t, 200, `{"data":[{"id":"x"}],"pad":"`+strings.Repeat("a", maxBody)+`"}`) + if _, err := c.Models(context.Background()); err == nil || !strings.Contains(err.Error(), "malformed JSON") { + t.Errorf("err = %v, want the oversized body cut short", err) + } +} + +func TestDo_TruncatedBody(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "100") + _, _ = io.WriteString(w, `{"data":`) + })) + t.Cleanup(srv.Close) + c := New("") + c.BaseURL = srv.URL + if _, err := c.Models(context.Background()); err == nil || !strings.Contains(err.Error(), "reading OpenRouter response") { + t.Errorf("err = %v, want a read error", err) + } +} diff --git a/internal/openrouter/testdata/decision-clef.json b/internal/openrouter/testdata/decision-clef.json new file mode 100644 index 0000000..f48e68c --- /dev/null +++ b/internal/openrouter/testdata/decision-clef.json @@ -0,0 +1,42 @@ +{ + "model": "cloudflare/clef", + "answers": { + "model": { + "type": "choice", + "choice": "opus", + "probabilities": { + "haiku": 0.0153, + "sonnet": 0.2806, + "opus": 0.668, + "fable": 0.0361 + }, + "confidence": 0.3687 + }, + "effort": { + "type": "score", + "score": 2.2992, + "legend": { + "0": "low", + "1": "medium", + "2": "high", + "3": "xhigh", + "4": "max" + }, + "probabilities": { + "0": 0.0138, + "1": 0.0774, + "2": 0.5636, + "3": 0.2861, + "4": 0.0591 + }, + "confidence": 0.2615 + } + }, + "usage": { + "input_tokens": 1888, + "output_tokens": 0, + "cost": 0.00045312 + }, + "id": "gen-dec-1792341389-TqBYOiBGoPYLoycdCH2R", + "provider": "Cloudflare" +} diff --git a/internal/openrouter/testdata/decision-jev.json b/internal/openrouter/testdata/decision-jev.json new file mode 100644 index 0000000..ddf1a22 --- /dev/null +++ b/internal/openrouter/testdata/decision-jev.json @@ -0,0 +1,42 @@ +{ + "model": "typesafe/jev-1.14-20261014", + "answers": { + "model": { + "type": "choice", + "choice": "sonnet", + "probabilities": { + "haiku": 0, + "sonnet": 0.57, + "opus": 0.43, + "fable": 0 + }, + "confidence": 0.43 + }, + "effort": { + "type": "score", + "score": 2.91, + "legend": { + "0": "low", + "1": "medium", + "2": "high", + "3": "xhigh", + "4": "max" + }, + "probabilities": { + "0": 0, + "1": 0, + "2": 0.09, + "3": 0.89, + "4": 0.02 + }, + "confidence": 0.91 + } + }, + "usage": { + "input_tokens": 1956, + "output_tokens": 64, + "cost": 0.000082152 + }, + "id": "gen-dec-1792341390-hlnW8eAck1t4e6SsowM0", + "provider": "TypeSafe" +} diff --git a/internal/openrouter/testdata/decision-request-questions.json b/internal/openrouter/testdata/decision-request-questions.json new file mode 100644 index 0000000..b3f8490 --- /dev/null +++ b/internal/openrouter/testdata/decision-request-questions.json @@ -0,0 +1,23 @@ +{ + "model": { + "type": "choice", + "instructions": "Which Claude model family should this subagent run on?", + "criteria": { + "haiku": "Claude Haiku: fastest and cheapest, for simple, mechanical or high-volume work", + "sonnet": "Claude Sonnet: balanced quality and cost, strong at most coding and review work", + "opus": "Claude Opus: highest-quality reasoning for hard, high-stakes judgement", + "fable": "Claude Fable: frontier model for long autonomous knowledge work, most expensive" + } + }, + "effort": { + "type": "score", + "instructions": "How much reasoning effort does this subagent need?", + "criteria": [ + "low", + "medium", + "high", + "xhigh", + "max" + ] + } +} diff --git a/internal/openrouter/testdata/error-unknown-model.json b/internal/openrouter/testdata/error-unknown-model.json new file mode 100644 index 0000000..f9d500a --- /dev/null +++ b/internal/openrouter/testdata/error-unknown-model.json @@ -0,0 +1,6 @@ +{ + "error": { + "message": "Model openai/gpt-6-luna does not exist", + "code": 400 + } +} diff --git a/internal/openrouter/testdata/models.json b/internal/openrouter/testdata/models.json new file mode 100644 index 0000000..af1c1b0 --- /dev/null +++ b/internal/openrouter/testdata/models.json @@ -0,0 +1,76 @@ +{ + "data": [ + { + "id": "typesafe/jev-1.14", + "name": "TypeSafe: Jev 1.14", + "pricing": { + "prompt": "0.000000042", + "completion": "0" + } + }, + { + "id": "anthropic/claude-haiku-5.5", + "name": "Anthropic: Claude Haiku 5.5", + "pricing": { + "prompt": "0.0000001", + "completion": "0.0000005" + } + }, + { + "id": "cloudflare/clef", + "name": "Cloudflare: Clef", + "pricing": { + "prompt": "0.000000042", + "completion": "0" + } + }, + { + "id": "anthropic/claude-sonnet-5.5", + "name": "Anthropic: Claude Sonnet 5.5", + "pricing": { + "prompt": "0.000002", + "completion": "0.00001" + } + }, + { + "id": "typesafe/jev-router", + "name": "TypeSafe: Jev Router", + "pricing": { + "prompt": "-1", + "completion": "-1" + } + }, + { + "id": "anthropic/claude-opus-5.5", + "name": "Anthropic: Claude Opus 5.5", + "pricing": { + "prompt": "0.000004", + "completion": "0.00002" + } + }, + { + "id": "~typesafe/jev-latest", + "name": "TypeSafe: Jev Latest", + "pricing": { + "prompt": "0.000000042", + "completion": "0" + } + }, + { + "id": "anthropic/claude-fable-5.1", + "name": "Anthropic: Claude Fable 5.1", + "pricing": { + "prompt": "0.00001", + "completion": "0.00005" + } + }, + { + "id": "anthropic/claude-opus-4.8", + "name": "Anthropic: Claude Opus 4.8", + "pricing": { + "prompt": "0.000005", + "completion": "0.000025" + } + } + ] +} diff --git a/internal/optimize/optimize.go b/internal/optimize/optimize.go new file mode 100644 index 0000000..0302d26 --- /dev/null +++ b/internal/optimize/optimize.go @@ -0,0 +1,349 @@ +// Package optimize asks an OpenRouter decision model which Claude model and +// effort each agent should run on, and decides which answers are confident +// enough to suggest. It writes nothing. +package optimize + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "strconv" + "strings" + + "github.com/agentic-tooling/agentsync/internal/discover" + "github.com/agentic-tooling/agentsync/internal/openrouter" +) + +// NotSet is shown for a model or effort key the frontmatter lacks. +const NotSet = "(not set)" + +// stateCap keeps the agent file under Jev's long-context threshold of ~32 KB. +const stateCap = 24 << 10 + +const framing = "A Claude Code subagent definition follows. Decide which Claude model and reasoning effort it should run on: the cheapest choice that keeps quality for its role." + +// Family is a Claude Code model alias, which Claude Code resolves to the +// newest model of that family, so advice never names a stale version. +type Family struct { + Alias string + Description string +} + +var families = []Family{ + {"haiku", "Claude Haiku: fastest and cheapest, for simple, mechanical or high-volume work"}, + {"sonnet", "Claude Sonnet: balanced quality and cost, strong at most coding and review work"}, + {"opus", "Claude Opus: highest-quality reasoning for hard, high-stakes judgement"}, + {"fable", "Claude Fable: frontier model for long autonomous knowledge work, most expensive"}, +} + +// Efforts are Claude Code's effort levels in order; a score answer indexes it. +var Efforts = []string{"low", "medium", "high", "xhigh", "max"} + +// Status is a row's outcome. Only StatusChange suggests anything; the zero +// value is none of these and suggests nothing. +type Status string + +const ( + StatusChange Status = "change" + StatusKeep Status = "keep" + StatusUnresolved Status = "unresolved" + StatusSkipped Status = "skipped" +) + +// Agent is one agent file to ask about. +type Agent struct { + Label, Path, Content string +} + +// Pick is one answer and the probability the advisor gave it; for effort, the +// probability of the level nearest the score. +type Pick struct { + Value string `json:"value"` + Probability float64 `json:"probability"` +} + +// Row is the advice for one agent. Suggested fields are set only for the +// parts that change; a LowConfidence flag marks a differing pick that was +// dropped for falling under the threshold. +type Row struct { + Agent string `json:"agent"` + Path string `json:"path"` + Model string `json:"model"` + Effort string `json:"effort"` + Status Status `json:"status"` + SuggestedModel string `json:"suggested_model,omitempty"` + SuggestedEffort string `json:"suggested_effort,omitempty"` + ModelPick *Pick `json:"model_pick,omitempty"` + EffortPick *Pick `json:"effort_pick,omitempty"` + ModelLowConfidence bool `json:"model_low_confidence,omitempty"` + EffortLowConfidence bool `json:"effort_low_confidence,omitempty"` + Reason string `json:"reason,omitempty"` + Cost float64 `json:"cost_usd"` +} + +// Report is one run's rows and the spend OpenRouter reported for them. +// Overrides names the set environment variables Claude Code uses over the +// frontmatter; the caller fills it. +type Report struct { + Advisor string `json:"advisor"` + Rows []Row `json:"rows"` + Spent float64 `json:"spent_usd"` + Overrides []string `json:"overrides,omitempty"` +} + +// Count returns how many rows have status s. +func (r Report) Count(s Status) int { + n := 0 + for _, row := range r.Rows { + if row.Status == s { + n++ + } + } + return n +} + +// Config is what Run needs beyond the agents. Price is the advisor's input +// price in USD per token. +type Config struct { + Advisor string + Price float64 + MaxCost float64 + MinConfidence float64 + Families []Family +} + +// Current returns an agent's model and effort as written, without a trailing +// YAML comment or quotes, or NotSet. +func Current(content string) (model, effort string) { + fm, _ := discover.Frontmatter(content) + get := func(k string) string { + v := " " + fm[k] + for _, sep := range []string{" #", "\t#"} { + v, _, _ = strings.Cut(v, sep) + } + if v = strings.Trim(strings.TrimSpace(v), `"'`); v != "" { + return v + } + return NotSet + } + return get("model"), get("effort") +} + +// Available keeps the aliases OpenRouter lists an anthropic/claude-- model for. +func Available(models []openrouter.Model) []Family { + var out []Family + for _, f := range families { + for _, m := range models { + v, ok := strings.CutPrefix(m.ID, "anthropic/claude-"+f.Alias+"-") + if ok && v != "" && v[0] >= '0' && v[0] <= '9' { + out = append(out, f) + break + } + } + } + return out +} + +// InputPrice is the advisor's input price per token. A missing, variable +// ("-1") or unparseable price refuses: the cost cap cannot be kept without it. +func InputPrice(models []openrouter.Model, advisor string) (float64, error) { + for _, m := range models { + if m.ID != advisor { + continue + } + p, err := strconv.ParseFloat(m.Pricing.Prompt, 64) + if err != nil || !(p >= 0) || math.IsInf(p, 1) { + return 0, fmt.Errorf("advisor %s has no fixed input price in the OpenRouter models list (got %q), so the cost cap cannot be kept", advisor, m.Pricing.Prompt) + } + return p, nil + } + return 0, fmt.Errorf("advisor %s is not in the OpenRouter models list", advisor) +} + +// Body encodes the Decisions request for one agent file. +func Body(advisor, content string, fams []Family) []byte { + if len(content) > stateCap { + content = strings.ToValidUTF8(content[:stateCap], "") + } + criteria := make(map[string]string, len(fams)) + for _, f := range fams { + criteria[f.Alias] = f.Description + } + // Strings, string maps and slices always encode; there is no error to handle. + b, _ := json.Marshal(openrouter.DecisionRequest{ + Model: advisor, + State: []string{framing, content}, + Questions: map[string]openrouter.Question{ + "model": {Type: "choice", Instructions: "Which Claude model family should this subagent run on?", Criteria: criteria}, + "effort": {Type: "score", Instructions: "How much reasoning effort does this subagent need?", Criteria: Efforts}, + }, + }) + return b +} + +// Run asks about each agent in turn, one request each and no retries. A +// request whose estimate would take the reported spend past MaxCost is +// skipped. Anything that may have billed without a usable usage.cost stops +// the run, because spend can no longer be tracked. +func Run(ctx context.Context, c *openrouter.Client, agents []Agent, cfg Config) Report { + rep := Report{Advisor: cfg.Advisor, Rows: make([]Row, 0, len(agents))} + stopped := "" + // Listed price at bytes/4 tokens, raised to the highest cost per byte + // OpenRouter has reported, so an advisor billed above its listed price is + // underestimated mostly on the first request. + rate := cfg.Price / 4 + for _, a := range agents { + row := Row{Agent: a.Label, Path: a.Path} + row.Model, row.Effort = Current(a.Content) + body := Body(cfg.Advisor, a.Content, cfg.Families) + est := float64(len(body)) * rate + switch { + case stopped != "": + row.Status, row.Reason = StatusSkipped, stopped + case rep.Spent+est > cfg.MaxCost: + row.Status = StatusSkipped + row.Reason = fmt.Sprintf("estimated $%.6f would pass the $%.2f cap", est, cfg.MaxCost) + default: + d, err := c.Decide(ctx, body) + var apiErr *openrouter.APIError + switch { + case errors.As(err, &apiErr): + // OpenRouter does not bill a request it answers with an error status. + row.Status, row.Reason = StatusUnresolved, err.Error() + case err != nil: + stopped = "run stopped: spend can no longer be tracked" + row.Status, row.Reason = StatusUnresolved, err.Error()+"; the request may have been billed, so spend can no longer be tracked" + case d.Usage.Cost == nil || !(*d.Usage.Cost >= 0) || math.IsInf(*d.Usage.Cost, 1): + stopped = "run stopped: spend can no longer be tracked" + row.Status, row.Reason = StatusUnresolved, "response reported no valid usage.cost, so spend can no longer be tracked" + default: + row.Cost = *d.Usage.Cost + rep.Spent += row.Cost + rate = max(rate, row.Cost/float64(len(body))) + row = judge(row, d, cfg) + } + } + rep.Rows = append(rep.Rows, row) + } + return rep +} + +// judge maps the answers onto row, refusing anything not offered. +func judge(row Row, d openrouter.Decision, cfg Config) Row { + m, err := modelPick(d.Answers, cfg.Families) + if err == nil { + var e Pick + if e, err = effortPick(d.Answers); err == nil { + row.ModelPick, row.EffortPick = &m, &e + } + } + if err != nil { + row.Status, row.Reason = StatusUnresolved, err.Error() + return row + } + row.SuggestedModel, row.ModelLowConfidence = suggest(row.Model, m, cfg.MinConfidence) + row.SuggestedEffort, row.EffortLowConfidence = suggest(row.Effort, *row.EffortPick, cfg.MinConfidence) + row.Status = StatusKeep + if row.SuggestedModel != "" || row.SuggestedEffort != "" { + row.Status = StatusChange + } + return row +} + +// suggest returns the pick when it differs from current and clears min, and +// whether a differing pick was dropped for low confidence. +func suggest(current string, p Pick, min float64) (string, bool) { + switch { + case p.Value == current: + return "", false + case p.Probability < min: + return "", true + } + return p.Value, false +} + +func modelPick(answers map[string]openrouter.Answer, fams []Family) (Pick, error) { + a, ok := answers["model"] + if !ok || a.Type != "choice" { + return Pick{}, fmt.Errorf("no choice answer for model (got type %q)", a.Type) + } + offered := false + for _, f := range fams { + offered = offered || f.Alias == a.Choice + } + if !offered { + return Pick{}, fmt.Errorf("model answer %q is not an offered option", a.Choice) + } + p, ok := a.Probabilities[a.Choice] + if !ok { + return Pick{}, fmt.Errorf("model answer %q has no probability", a.Choice) + } + return Pick{a.Choice, p}, probability("model", p) +} + +func effortPick(answers map[string]openrouter.Answer) (Pick, error) { + a, ok := answers["effort"] + if !ok || a.Type != "score" || a.Score == nil { + return Pick{}, fmt.Errorf("no score answer for effort (got type %q)", a.Type) + } + i := math.Round(*a.Score) + if !(i >= 0 && i < float64(len(Efforts))) { + return Pick{}, fmt.Errorf("effort score %v is outside 0..%d", *a.Score, len(Efforts)-1) + } + level := Efforts[int(i)] + p, ok := a.Probabilities[strconv.Itoa(int(i))] + if !ok { + return Pick{}, fmt.Errorf("effort level %s has no probability", level) + } + return Pick{level, p}, probability("effort", p) +} + +// probability refuses a value outside [0, 1], which would otherwise clear any +// threshold. +func probability(q string, p float64) error { + if !(p >= 0 && p <= 1) { + return fmt.Errorf("%s probability %v is outside 0..1", q, p) + } + return nil +} + +// Now is the current setting, "model / effort". +func (r Row) Now() string { return r.Model + " / " + r.Effort } + +// Suggested is the setting to move to, naming a part dropped for low +// confidence, or why there is none. +func (r Row) Suggested() string { + switch r.Status { + case StatusChange: + m, e, note := r.Model, r.Effort, "" + if r.SuggestedModel != "" { + m = r.SuggestedModel + } + if r.SuggestedEffort != "" { + e = r.SuggestedEffort + } + if r.ModelLowConfidence { + note = " (model: low confidence)" + } + if r.EffortLowConfidence { + note = " (effort: low confidence)" + } + return m + " / " + e + note + case StatusKeep: + if r.ModelLowConfidence || r.EffortLowConfidence { + return "keep (low confidence)" + } + return "keep" + } + return string(r.Status) +} + +// Confidence is each pick with its probability, or "-" when nothing was picked. +func (r Row) Confidence() string { + if r.ModelPick == nil || r.EffortPick == nil { + return "-" + } + return fmt.Sprintf("%s %.2f / %s %.2f", r.ModelPick.Value, r.ModelPick.Probability, r.EffortPick.Value, r.EffortPick.Probability) +} diff --git a/internal/optimize/optimize_test.go b/internal/optimize/optimize_test.go new file mode 100644 index 0000000..6f46561 --- /dev/null +++ b/internal/optimize/optimize_test.go @@ -0,0 +1,345 @@ +package optimize + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/agentic-tooling/agentsync/internal/openrouter" +) + +const testdata = "../openrouter/testdata/" + +func fixture(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(testdata + name) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func models(t *testing.T) []openrouter.Model { + t.Helper() + var out struct{ Data []openrouter.Model } + if err := json.Unmarshal([]byte(fixture(t, "models.json")), &out); err != nil { + t.Fatal(err) + } + return out.Data +} + +// reply is one canned response; serve, when set, replaces status and body. +type reply struct { + status int + body string + serve http.HandlerFunc +} + +// advisor serves replies in order and counts the requests it was sent. A +// request past the last reply gets 418, so an unexpected one fails a count +// assertion instead of hanging the test. +func advisor(t *testing.T, replies ...reply) (*openrouter.Client, func() int) { + t.Helper() + var ( + mu sync.Mutex + n int + ) + next := func() (reply, bool) { + mu.Lock() + defer mu.Unlock() + n++ + if n > len(replies) { + return reply{}, false + } + return replies[n-1], true + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rep, ok := next() + switch { + case !ok: + w.WriteHeader(http.StatusTeapot) + case rep.serve != nil: + rep.serve(w, r) + default: + w.WriteHeader(rep.status) + _, _ = io.WriteString(w, rep.body) + } + })) + t.Cleanup(srv.Close) + c := openrouter.New("sk-test") + c.BaseURL = srv.URL + return c, func() int { mu.Lock(); defer mu.Unlock(); return n } +} + +func agent(label, model, effort string) Agent { + fm := "---\nname: " + label + "\ndescription: d\n" + if model != "" { + fm += "model: " + model + "\n" + } + if effort != "" { + fm += "effort: " + effort + "\n" + } + return Agent{Label: label, Path: "/p/" + label + ".md", Content: fm + "---\nbody\n"} +} + +func config() Config { + return Config{Advisor: "~typesafe/jev-latest", Price: 4.2e-8, MaxCost: 0.5, MinConfidence: 0.6, Families: families} +} + +func TestCurrent(t *testing.T) { + for _, tc := range []struct{ content, model, effort string }{ + {"---\nname: n\nmodel: opus\neffort: xhigh\n---\n", "opus", "xhigh"}, + {"---\nmodel: \"claude-opus-4-5\"\neffort: 'low'\n---\n", "claude-opus-4-5", "low"}, + {"---\nmodel: inherit\n---\n", "inherit", NotSet}, + {"---\nmodel:\neffort: \"\"\n---\n", NotSet, NotSet}, + {"no frontmatter\nmodel: opus\n", NotSet, NotSet}, + {"---\nmodel: opus # cheap enough\neffort: \"high\"\t# for now\n---\n", "opus", "high"}, + {"---\nmodel: # none yet\n---\n", NotSet, NotSet}, + } { + if m, e := Current(tc.content); m != tc.model || e != tc.effort { + t.Errorf("Current(%q) = %q, %q; want %q, %q", tc.content, m, e, tc.model, tc.effort) + } + } +} + +func TestAvailable(t *testing.T) { + if got := Available(models(t)); !reflect.DeepEqual(got, families) { + t.Errorf("real list: %v, want all four", got) + } + got := Available([]openrouter.Model{{ID: "anthropic/claude-opus-5.5"}, {ID: "anthropic/claude-sonnet-x"}, {ID: "anthropic/claude-haiku-"}, {ID: "openai/fable-5"}}) + if len(got) != 1 || got[0].Alias != "opus" { + t.Errorf("got %v, want opus alone", got) + } +} + +func TestInputPrice(t *testing.T) { + ms := append(models(t), openrouter.Model{ID: "nan"}, openrouter.Model{ID: "inf"}) + ms[len(ms)-2].Pricing.Prompt, ms[len(ms)-1].Pricing.Prompt = "NaN", "+Inf" + for _, id := range []string{"~typesafe/jev-latest", "cloudflare/clef"} { + if p, err := InputPrice(ms, id); err != nil || p != 4.2e-8 { + t.Errorf("%s: %v, %v; want 4.2e-8", id, p, err) + } + } + for id, want := range map[string]string{ + "typesafe/jev-router": "no fixed input price", + "nan": "no fixed input price", + "inf": "no fixed input price", + "openai/gpt-6-luna": "not in the OpenRouter models list", + } { + if _, err := InputPrice(ms, id); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%s: err = %v, want %q", id, err, want) + } + } +} + +func TestBody_MatchesTheSpikeRequestAndCapsState(t *testing.T) { + var req openrouter.DecisionRequest + content := strings.Repeat("a", stateCap-1) + "é" + "tail" + if err := json.Unmarshal(Body("cloudflare/clef", content, families), &req); err != nil { + t.Fatal(err) + } + if req.Model != "cloudflare/clef" || len(req.State) != 2 || req.State[0] != framing { + t.Errorf("model %q, state %d entries", req.Model, len(req.State)) + } + if req.State[1] != strings.Repeat("a", stateCap-1) { + t.Errorf("state not cut cleanly at %d bytes: %d bytes, tail %q", stateCap, len(req.State[1]), req.State[1][len(req.State[1])-3:]) + } + var got, want any + b, _ := json.Marshal(req.Questions) + _ = json.Unmarshal(b, &got) + _ = json.Unmarshal([]byte(fixture(t, "decision-request-questions.json")), &want) + if !reflect.DeepEqual(got, want) { + t.Errorf("questions differ from the request that worked:\n%s", b) + } +} + +func TestRun_RealResponses(t *testing.T) { + jev, clef := fixture(t, "decision-jev.json"), fixture(t, "decision-clef.json") + for _, tc := range []struct { + name, body, model, effort string + min float64 + status Status + suggested string + lowM, lowE bool + }{ + {"jev below threshold keeps", jev, "opus", "xhigh", 0.6, StatusKeep, "keep (low confidence)", true, false}, + {"jev at threshold changes", jev, "opus", "xhigh", 0.57, StatusChange, "sonnet / xhigh", false, false}, + {"jev above threshold changes", jev, "opus", "xhigh", 0.5, StatusChange, "sonnet / xhigh", false, false}, + {"jev agrees", jev, "sonnet", "xhigh", 0.6, StatusKeep, "keep", false, false}, + {"jev both parts low", jev, "opus", "low", 0.95, StatusKeep, "keep (low confidence)", true, true}, + {"clef changes model, effort low confidence", clef, "sonnet", "xhigh", 0.6, StatusChange, "opus / xhigh (effort: low confidence)", false, true}, + {"clef effort is the nearest level's probability", clef, "opus", "", 0.56, StatusChange, "opus / high", false, false}, + {"jev changes effort, model low confidence", jev, "opus", "medium", 0.6, StatusChange, "opus / xhigh (model: low confidence)", true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + c, _ := advisor(t, reply{status: 200, body: tc.body}) + cfg := config() + cfg.MinConfidence = tc.min + rep := Run(context.Background(), c, []Agent{agent("go-merge-reviewer", tc.model, tc.effort)}, cfg) + r := rep.Rows[0] + if r.Status != tc.status || r.Suggested() != tc.suggested || r.ModelLowConfidence != tc.lowM || r.EffortLowConfidence != tc.lowE { + t.Errorf("status %q suggested %q low %v/%v; want %q %q %v/%v (%s)", r.Status, r.Suggested(), + r.ModelLowConfidence, r.EffortLowConfidence, tc.status, tc.suggested, tc.lowM, tc.lowE, r.Reason) + } + if rep.Spent != r.Cost || r.Cost == 0 { + t.Errorf("spent %v, row cost %v", rep.Spent, r.Cost) + } + }) + } +} + +func TestRun_FailsClosed(t *testing.T) { + answers := func(model, effort string) string { + return `{"answers":{` + model + `,` + effort + `},"usage":{"cost":0.0001}}` + } + goodM := `"model":{"type":"choice","choice":"opus","probabilities":{"opus":0.9}}` + goodE := `"effort":{"type":"score","score":2,"probabilities":{"2":0.9}}` + for _, tc := range []struct { + name string + r reply + want string + }{ + {"unknown model 400", reply{status: 400, body: fixture(t, "error-unknown-model.json")}, "Model openai/gpt-6-luna does not exist"}, + {"5xx", reply{status: 503, body: "down"}, "HTTP 503: down"}, + {"malformed", reply{status: 200, body: `{"answers":`}, "malformed JSON"}, + {"no answers", reply{status: 200, body: `{"usage":{"cost":0.0001}}`}, "no choice answer for model"}, + {"option not offered", reply{status: 200, body: answers(`"model":{"type":"choice","choice":"gpt","probabilities":{"gpt":1}}`, goodE)}, `"gpt" is not an offered option`}, + {"wrong type", reply{status: 200, body: answers(`"model":{"type":"score","choice":"opus"}`, goodE)}, `got type "score"`}, + {"no probability", reply{status: 200, body: answers(`"model":{"type":"choice","choice":"opus","probabilities":{}}`, goodE)}, "has no probability"}, + {"probability above 1", reply{status: 200, body: answers(`"model":{"type":"choice","choice":"opus","probabilities":{"opus":1.5}}`, goodE)}, "outside 0..1"}, + {"no effort score", reply{status: 200, body: answers(goodM, `"effort":{"type":"score","probabilities":{"2":0.9}}`)}, "no score answer for effort"}, + {"no probability for the nearest level", reply{status: 200, body: answers(goodM, `"effort":{"type":"score","score":2.6,"confidence":0.9,"probabilities":{"2":0.9}}`)}, "effort level xhigh has no probability"}, + {"effort out of range", reply{status: 200, body: answers(goodM, `"effort":{"type":"score","score":4.6,"probabilities":{"4":0.9}}`)}, "outside 0..4"}, + {"negative effort probability", reply{status: 200, body: answers(goodM, `"effort":{"type":"score","score":1,"probabilities":{"1":-0.1}}`)}, "effort probability"}, + } { + t.Run(tc.name, func(t *testing.T) { + c, _ := advisor(t, tc.r) + r := Run(context.Background(), c, []Agent{agent("a", "opus", "high")}, config()).Rows[0] + if r.Status != StatusUnresolved || !strings.Contains(r.Reason, tc.want) { + t.Errorf("status %q reason %q; want unresolved with %q", r.Status, r.Reason, tc.want) + } + if r.Suggested() != "unresolved" || r.Confidence() != "-" || r.SuggestedModel != "" || r.SuggestedEffort != "" { + t.Errorf("an unresolved row suggests something: %+v", r) + } + }) + } +} + +func TestRun_UntrackedSpendStopsTheRun(t *testing.T) { + // The server notices the client hanging up only once the body is read. + hang := func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + select { + case <-r.Context().Done(): + case <-time.After(5 * time.Second): + } + } + truncate := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "100") + _, _ = io.WriteString(w, `{"answers":`) + } + for _, tc := range []struct { + name string + r reply + want string + }{ + {"missing cost", reply{status: 200, body: `{"answers":{}}`}, "no valid usage.cost"}, + {"negative cost", reply{status: 200, body: `{"answers":{},"usage":{"cost":-1}}`}, "no valid usage.cost"}, + {"malformed 200", reply{status: 200, body: `{"answers":`}, "malformed JSON"}, + {"truncated 200", reply{serve: truncate}, "reading OpenRouter response"}, + {"timeout", reply{serve: hang}, "Timeout"}, + } { + t.Run(tc.name, func(t *testing.T) { + c, sent := advisor(t, tc.r) + c.HTTP.Timeout = 200 * time.Millisecond + rep := Run(context.Background(), c, []Agent{agent("a", "", ""), agent("b", "", ""), agent("c", "", "")}, config()) + if sent() != 1 { + t.Errorf("%d requests sent, want 1", sent()) + } + r := rep.Rows[0] + if r.Status != StatusUnresolved || !strings.Contains(r.Reason, tc.want) || !strings.Contains(r.Reason, "spend can no longer be tracked") { + t.Errorf("first row %+v, want unresolved with %q", r, tc.want) + } + if rep.Count(StatusSkipped) != 2 || !strings.Contains(rep.Rows[2].Reason, "run stopped") { + t.Errorf("rest %+v", rep.Rows[1:]) + } + if rep.Spent != 0 || strings.Contains(r.Reason, "sk-test") { + t.Errorf("spent %v; reason %q", rep.Spent, r.Reason) + } + }) + } + t.Run("an error status does not stop the run", func(t *testing.T) { + c, sent := advisor(t, reply{status: 503, body: "down"}) + rep := Run(context.Background(), c, []Agent{agent("a", "", ""), agent("b", "", "")}, config()) + if sent() != 2 || rep.Count(StatusUnresolved) != 2 { + t.Errorf("sent %d, rows %+v", sent(), rep.Rows) + } + }) +} + +func TestRun_CostCap(t *testing.T) { + costly := `{"answers":{"model":{"type":"choice","choice":"opus","probabilities":{"opus":0.9}},"effort":{"type":"score","score":2,"probabilities":{"2":0.9}}},"usage":{"cost":0.2}}` + cheap := strings.Replace(costly, `"cost":0.2`, `"cost":0.1`, 1) + agents := []Agent{agent("a", "opus", "high"), agent("b", "opus", "high"), agent("c", "opus", "high")} + t.Run("running total of usage.cost", func(t *testing.T) { + c, sent := advisor(t, reply{status: 200, body: cheap}, reply{status: 200, body: cheap}) + cfg := config() + cfg.MaxCost = 0.25 + rep := Run(context.Background(), c, agents, cfg) + // a and b fit; c's $0.10 estimate does not fit on top of $0.20 spent. + if sent() != 2 || rep.Spent != 0.2 { + t.Errorf("sent %d, spent %v", sent(), rep.Spent) + } + if rep.Count(StatusKeep) != 2 || rep.Count(StatusSkipped) != 1 { + t.Errorf("rows %+v", rep.Rows) + } + }) + t.Run("estimate learns the billed rate", func(t *testing.T) { + c, sent := advisor(t, reply{status: 200, body: costly}) + cfg := config() + cfg.MaxCost = 0.3 + rep := Run(context.Background(), c, agents, cfg) + // At the listed price b would look free; a's reported $0.20 says it costs as much again. + if sent() != 1 || rep.Spent != 0.2 || rep.Count(StatusSkipped) != 2 { + t.Errorf("sent %d, spent %v, rows %+v", sent(), rep.Spent, rep.Rows) + } + if !strings.Contains(rep.Rows[1].Reason, "estimated $0.2") { + t.Errorf("b's reason %q", rep.Rows[1].Reason) + } + }) + t.Run("estimate alone", func(t *testing.T) { + c, sent := advisor(t) + cfg := config() + cfg.MaxCost = 0 + rep := Run(context.Background(), c, []Agent{agent("a", "", "")}, cfg) + if sent() != 0 || rep.Rows[0].Status != StatusSkipped || !strings.Contains(rep.Rows[0].Reason, "would pass the $0.00 cap") { + t.Errorf("sent %d, row %+v", sent(), rep.Rows[0]) + } + if rep.Rows[0].Suggested() != "skipped" { + t.Errorf("suggested %q", rep.Rows[0].Suggested()) + } + }) +} + +func TestRow_Display(t *testing.T) { + r := Row{Model: NotSet, Effort: "high", Status: StatusChange, SuggestedModel: "opus", + ModelPick: &Pick{"opus", 0.668}, EffortPick: &Pick{"high", 0.2615}} + if r.Now() != "(not set) / high" || r.Suggested() != "opus / high" || r.Confidence() != "opus 0.67 / high 0.26" { + t.Errorf("now %q suggested %q confidence %q", r.Now(), r.Suggested(), r.Confidence()) + } + r.SuggestedModel, r.SuggestedEffort = "", "max" + if r.Suggested() != "(not set) / max" { + t.Errorf("effort-only change shows %q", r.Suggested()) + } + if (Row{}).Suggested() != "" { + t.Error("the zero row suggests something") + } +}