From c2f94c4d2b18a246be9419d1c8e73fe1068666e2 Mon Sep 17 00:00:00 2001 From: Logan Lindquist Land Date: Sun, 18 Oct 2026 13:25:05 -0500 Subject: [PATCH] feat: agentsync optimize suggests model and effort for agents `agentsync optimize` asks a decision model (TypeSafe Jev by default, Cloudflare Clef optional) through OpenRouter which Claude model family and effort each agent should run on, and reports current against suggested with the advisor's confidence. It only suggests a change above --min-confidence, refuses to exceed --max-cost, and stops when spend can't be tracked. Report only; nothing is written. Needs $OPENROUTER_API_KEY; every other command stays offline. --- README.md | 2 + cmd/agentsync/config_test.go | 9 +- cmd/agentsync/main.go | 22 ++ cmd/agentsync/optimize.go | 232 ++++++++++++ cmd/agentsync/optimize_test.go | 331 +++++++++++++++++ coverage-floors.txt | 4 +- docs/exit-codes.md | 1 + docs/optimize.md | 121 ++++++ internal/discover/discover.go | 33 +- internal/discover/discover_test.go | 20 +- internal/openrouter/openrouter.go | 174 +++++++++ internal/openrouter/openrouter_test.go | 207 +++++++++++ .../openrouter/testdata/decision-clef.json | 42 +++ .../openrouter/testdata/decision-jev.json | 42 +++ .../testdata/decision-request-questions.json | 23 ++ .../testdata/error-unknown-model.json | 6 + internal/openrouter/testdata/models.json | 76 ++++ internal/optimize/optimize.go | 349 ++++++++++++++++++ internal/optimize/optimize_test.go | 345 +++++++++++++++++ 19 files changed, 2024 insertions(+), 15 deletions(-) create mode 100644 cmd/agentsync/optimize.go create mode 100644 cmd/agentsync/optimize_test.go create mode 100644 docs/optimize.md create mode 100644 internal/openrouter/openrouter.go create mode 100644 internal/openrouter/openrouter_test.go create mode 100644 internal/openrouter/testdata/decision-clef.json create mode 100644 internal/openrouter/testdata/decision-jev.json create mode 100644 internal/openrouter/testdata/decision-request-questions.json create mode 100644 internal/openrouter/testdata/error-unknown-model.json create mode 100644 internal/openrouter/testdata/models.json create mode 100644 internal/optimize/optimize.go create mode 100644 internal/optimize/optimize_test.go diff --git a/README.md b/README.md index 9467aa6..af85f89 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ Run without `--apply` first to see what a command would do. | `doctor` | report broken, foreign, or mismatched links and stale captures; never writes | | `reconcile` | act on that drift, gated on what agentsync itself recorded | | `inventory` | regenerate `inventory.json`, a record of every agent on the machine | +| `optimize` | ask an OpenRouter advisor which model and effort each agent should use; report only | `agentsync help` lists every flag. @@ -80,6 +81,7 @@ Run without `--apply` first to see what a command would do. - [doctor](docs/doctor.md) - [reconcile](docs/reconcile.md) - [inventory](docs/inventory.md) +- [optimize](docs/optimize.md) - [State file](docs/state-file.md) - [Exit codes](docs/exit-codes.md) - [Development](docs/development.md): toolchain, CI, releases diff --git a/cmd/agentsync/config_test.go b/cmd/agentsync/config_test.go index 7ba6719..a564470 100644 --- a/cmd/agentsync/config_test.go +++ b/cmd/agentsync/config_test.go @@ -10,7 +10,9 @@ import ( ) // TestMain points HOME and XDG_CONFIG_HOME at empty temp dirs, so no test can -// read the developer's real config file and pass or fail because of it. +// read the developer's real config file and pass or fail because of it. It +// also clears the OpenRouter key, so no test can spend money, and the Claude +// overrides optimize reports. func TestMain(m *testing.M) { dir, err := os.MkdirTemp("", "agentsync-test-") if err != nil { @@ -22,12 +24,17 @@ func TestMain(m *testing.M) { os.Mkdir(home, 0o755), os.Setenv("HOME", home), os.Setenv("XDG_CONFIG_HOME", filepath.Join(dir, "xdg")), + os.Unsetenv(envOpenRouterKey), + os.Unsetenv("CLAUDE_CODE_SUBAGENT_MODEL"), + os.Unsetenv("CLAUDE_CODE_EFFORT_LEVEL"), } { if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(exitFailure) } } + // A test that sets a key but forgets its fake server fails here, offline. + openrouterURL = "http://127.0.0.1:0" code := m.Run() _ = os.RemoveAll(dir) os.Exit(code) diff --git a/cmd/agentsync/main.go b/cmd/agentsync/main.go index eac66a0..51ef0b4 100644 --- a/cmd/agentsync/main.go +++ b/cmd/agentsync/main.go @@ -35,6 +35,9 @@ // // init writes only agentsync's own config file, from values the user named or // confirmed, and refuses to replace an existing one without --force. +// +// optimize is the only command that uses the network: it asks an OpenRouter +// advisor about each agent's model and effort, and writes nothing. package main import ( @@ -62,6 +65,8 @@ Usage: agentsync doctor [flags] report broken/foreign/mismatched links and stale captures agentsync reconcile [flags] act on that drift, gated on what this tool recorded agentsync inventory [flags] regenerate inventory.json: every agent on this machine + agentsync optimize [flags] ask an OpenRouter advisor which model and effort each + agent should use (report only; the one networked command) agentsync version Common flags: @@ -216,6 +221,21 @@ inventory flags: is a real change in the tree, which is what makes this a freshness test rather than a clock reading. +optimize flags: + --advisor MODEL OpenRouter decision model to ask (default ~typesafe/jev-latest; + cloudflare/clef also works) + --max-cost USD skip any request that could take reported spend past this + (default 0.50) + --min-confidence P + least probability a pick needs before it is suggested + (default 0.6); below it the row says keep (low confidence) + --agent NAME ask only about this agent (repeatable) + +optimize needs $OPENROUTER_API_KEY and sends each agent file (first 24 KB) to +OpenRouter, one request per agent with no retries. It reads model: and effort: +from each agent's frontmatter and writes nothing. Exit 1 when a row is +unresolved or skipped. + scan and doctor are read-only. import writes only into this repo, only with --apply, and never into a project's .claude/agents directory. link is the one command that writes into a project, only with --apply, and it refuses any @@ -284,6 +304,8 @@ func run(args []string, stdout, stderr io.Writer) int { return runReconcile(args[1:], stdout, stderr) case "inventory": return runInventory(args[1:], stdout, stderr) + case "optimize": + return runOptimize(args[1:], stdout, stderr) case "-h", "--help", "help": fmt.Fprint(stdout, usage) return exitOK diff --git a/cmd/agentsync/optimize.go b/cmd/agentsync/optimize.go new file mode 100644 index 0000000..398b92d --- /dev/null +++ b/cmd/agentsync/optimize.go @@ -0,0 +1,232 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "math" + "os" + "path/filepath" + "strings" + "text/tabwriter" + + "github.com/agentic-tooling/agentsync/internal/discover" + "github.com/agentic-tooling/agentsync/internal/openrouter" + "github.com/agentic-tooling/agentsync/internal/optimize" + "github.com/agentic-tooling/agentsync/internal/scan" +) + +// openrouterURL is where optimize sends its requests; tests point it at an +// httptest server. optimize is the only command that uses the network. +var openrouterURL = openrouter.DefaultBaseURL + +const ( + envOpenRouterKey = "OPENROUTER_API_KEY" + defaultAdvisor = "~typesafe/jev-latest" +) + +// Claude Code reads these at run time and lets them beat the frontmatter. +var claudeOverrides = []string{"CLAUDE_CODE_SUBAGENT_MODEL", "CLAUDE_CODE_EFFORT_LEVEL"} + +// runOptimize asks an advisor model about each Claude agent under the root +// and reports its advice. It writes nothing. +func runOptimize(args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("optimize", flag.ContinueOnError) + fs.SetOutput(stderr) + fs.Usage = func() { fmt.Fprint(stderr, usage) } + + rootF := registerRoot(fs) + repoPath := fs.String("repo", "", "canonical agent repo") + asJSON := fs.Bool("json", false, "emit JSON") + advisor := fs.String("advisor", defaultAdvisor, "OpenRouter decision model to ask") + maxCost := fs.Float64("max-cost", 0.50, "most this run may spend, in USD") + minConf := fs.Float64("min-confidence", 0.6, "least probability a pick needs to be suggested") + var names stringList + fs.Var(&names, "agent", "ask only about this agent (repeatable)") + + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return exitOK + } + return exitFailure + } + if err := optimizeFlagsErr(fs, *advisor, *maxCost, *minConf); err != nil { + fmt.Fprintf(stderr, "agentsync: %v\n", err) + return exitFailure + } + key := strings.TrimSpace(os.Getenv(envOpenRouterKey)) + if key == "" { + fmt.Fprintf(stderr, "agentsync: optimize needs an OpenRouter API key in $%s\n", envOpenRouterKey) + return exitFailure + } + + repo, repoFrom, err := openRepo(*repoPath) + if err != nil { + fmt.Fprintf(stderr, "agentsync: %v\n", err) + return exitFailure + } + root, err := rootF.resolve(repo.Home) + if err != nil { + fmt.Fprintf(stderr, "agentsync: %v\n", err) + return exitFailure + } + res, err := repo.Scan(root.Path) + if err == nil { + err = scanProblemsErr(res.Problems) + } + var agents []optimize.Agent + if err == nil { + agents, err = optimizeAgents(res, names) + } + if err != nil { + fmt.Fprintf(stderr, "agentsync: %v\n", err) + return exitFailure + } + + client := openrouter.New(key) + client.BaseURL = openrouterURL + ctx := context.Background() + cfg := optimize.Config{Advisor: *advisor, MaxCost: *maxCost, MinConfidence: *minConf} + models, err := client.Models(ctx) + if err == nil { + cfg.Price, err = optimize.InputPrice(models, *advisor) + } + if err == nil { + if cfg.Families = optimize.Available(models); len(cfg.Families) == 0 { + err = errors.New("the OpenRouter models list has no anthropic/claude-- model, so there is nothing to suggest") + } + } + if err != nil { + fmt.Fprintf(stderr, "agentsync: OpenRouter models list: %v\n", err) + return exitFailure + } + + notes := reportStream(*asJSON, stdout, stderr) + emitRepoChoice(notes, repo, repoFrom) + emitRootChoice(notes, repo.Home, root) + var overrides []string + for _, v := range claudeOverrides { + if os.Getenv(v) != "" { + overrides = append(overrides, v) + fmt.Fprintf(notes, "Note: $%s is set; Claude Code uses it over the frontmatter at run time.\n", v) + } + } + + rep := optimize.Run(ctx, client, agents, cfg) + rep.Overrides = overrides + if *asJSON { + enc := json.NewEncoder(stdout) + enc.SetIndent("", " ") + if err := enc.Encode(rep); err != nil { + fmt.Fprintf(stderr, "agentsync: %v\n", err) + return exitFailure + } + } else { + emitOptimizeText(stdout, rep, cfg) + } + if rep.Count(optimize.StatusUnresolved)+rep.Count(optimize.StatusSkipped) > 0 { + return exitAtRisk + } + return exitOK +} + +func optimizeFlagsErr(fs *flag.FlagSet, advisor string, maxCost, minConf float64) error { + switch { + case fs.NArg() > 0: + return fmt.Errorf("optimize takes no arguments, got %q", fs.Arg(0)) + case strings.TrimSpace(advisor) == "": + return errors.New("--advisor is empty") + case !(maxCost >= 0) || math.IsInf(maxCost, 1): + return fmt.Errorf("--max-cost must be a finite amount of USD at or above 0, got %v", maxCost) + case !(minConf >= 0 && minConf <= 1): + return fmt.Errorf("--min-confidence must be between 0 and 1, got %v", minConf) + } + return nil +} + +// scanProblemsErr refuses a scan that could not read everything: an agent it +// missed is one the report would silently leave out. +func scanProblemsErr(problems []scan.ScanProblem) error { + if len(problems) == 0 { + return nil + } + lines := make([]string, len(problems)) + for i, p := range problems { + lines[i] = " " + p.String() + } + return fmt.Errorf("could not read every agent under the root:\n%s", strings.Join(lines, "\n")) +} + +// optimizeAgents reads the Claude agents scan found, once per real file, so +// an agent linked into many projects costs one request. +func optimizeAgents(res scan.Result, names []string) ([]optimize.Agent, error) { + want := map[string]bool{} + for _, n := range names { + want[n] = false + } + seen := map[string]bool{} + var out []optimize.Agent + for _, e := range res.Entries { + if _, ok := want[e.Name]; len(names) > 0 && !ok { + continue + } + data, err := os.ReadFile(e.Path) + if err != nil { + return nil, fmt.Errorf("could not read agent %s (%w); agentsync doctor reports broken links", e.Path, err) + } + if !discover.IsAgent(string(data)) { + continue + } + want[e.Name] = true + real := e.Path + if r, err := filepath.EvalSymlinks(e.Path); err == nil { + real = r + } + if seen[real] { + continue + } + seen[real] = true + label := e.Name + if len(res.Dirs) > 1 { + label = e.Project + "/" + e.Name + } + out = append(out, optimize.Agent{Label: label, Path: e.Path, Content: string(data)}) + } + for _, n := range names { + if !want[n] { + return nil, fmt.Errorf("--agent %s: no Claude agent by that name under the root", n) + } + } + return out, nil +} + +func emitOptimizeText(w io.Writer, rep optimize.Report, cfg optimize.Config) { + skipped := rep.Count(optimize.StatusSkipped) + asked := len(rep.Rows) - skipped + fmt.Fprintf(w, "Asked %s about %d agent%s", rep.Advisor, asked, plural(asked, "", "s")) + if skipped > 0 { + fmt.Fprintf(w, ", %d skipped", skipped) + } + fmt.Fprintf(w, " (cap $%.2f, min confidence %.2f).\n\n", cfg.MaxCost, cfg.MinConfidence) + tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0) + // Typed as io.Writer so errcheck's report-writer exemption applies. + var table io.Writer = tw + fmt.Fprintln(table, "AGENT\tNOW\tSUGGESTED\tCONFIDENCE") + for _, r := range rep.Rows { + fmt.Fprintf(table, "%s\t%s\t%s\t%s\n", r.Agent, r.Now(), r.Suggested(), r.Confidence()) + } + _ = tw.Flush() + fmt.Fprintln(w) + for _, r := range rep.Rows { + if r.Reason != "" { + fmt.Fprintf(w, "%s: %s: %s\n", r.Agent, r.Status, r.Reason) + } + } + if skipped > 0 { + fmt.Fprintf(w, "%d agent%s skipped (reasons above).\n", skipped, plural(skipped, " was", "s were")) + } + fmt.Fprintf(w, "Spent $%.6f (reported by OpenRouter). Dry run: nothing written.\n", rep.Spent) +} diff --git a/cmd/agentsync/optimize_test.go b/cmd/agentsync/optimize_test.go new file mode 100644 index 0000000..ad74ae9 --- /dev/null +++ b/cmd/agentsync/optimize_test.go @@ -0,0 +1,331 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + + "github.com/agentic-tooling/agentsync/internal/optimize" +) + +const orTestdata = "../../internal/openrouter/testdata/" + +func orFixture(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(orTestdata + name) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func agentFile(name, model, effort string) string { + return "---\nname: " + name + "\ndescription: d\nmodel: " + model + "\neffort: " + effort + "\n---\nbody\n" +} + +// fakeOpenRouter serves models (status 200 unless modelsStatus says +// otherwise) and the decisions in order, and points optimize at itself with +// testKey set. It counts both kinds of request. +type fakeOpenRouter struct { + models string + modelsStatus int + decisions []string + mu sync.Mutex + listed, sent int +} + +const testKey = "sk-test-never-printed" + +func assertNoKey(t *testing.T, out ...string) { + t.Helper() + if s := strings.Join(out, ""); strings.Contains(s, testKey) { + t.Errorf("output leaks the key:\n%s", s) + } +} + +func (f *fakeOpenRouter) start(t *testing.T) { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + if r.Method == http.MethodGet { + f.listed++ + if f.modelsStatus != 0 { + w.WriteHeader(f.modelsStatus) + } + _, _ = io.WriteString(w, f.models) + return + } + f.sent++ + if f.sent > len(f.decisions) { + w.WriteHeader(http.StatusTeapot) + return + } + _, _ = io.WriteString(w, f.decisions[f.sent-1]) + })) + t.Cleanup(srv.Close) + old := openrouterURL + openrouterURL = srv.URL + t.Cleanup(func() { openrouterURL = old }) + t.Setenv(envOpenRouterKey, testKey) +} + +// requests is how many decision requests arrived. +func (f *fakeOpenRouter) requests() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.sent +} + +func (f *fakeOpenRouter) lists() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.listed +} + +// optimizeProject makes a repo and a project whose agents dir holds files, +// and returns both paths. +func optimizeProject(t *testing.T, files map[string]string) (project, repoRoot string) { + t.Helper() + home := mustResolveDir(t, t.TempDir()) + t.Setenv("HOME", home) + repoRoot = filepath.Join(home, "Web", "agents-repo") + mustMkdirT(t, filepath.Join(repoRoot, "agents")) + project = filepath.Join(home, "Web", "proj") + mustMkdirT(t, filepath.Join(project, ".claude", "agents")) + for name, content := range files { + mustWriteT(t, filepath.Join(project, ".claude", "agents", name), content) + } + return project, repoRoot +} + +func TestOptimize_Report(t *testing.T) { + project, repoRoot := optimizeProject(t, map[string]string{ + "go-merge-reviewer.md": agentFile("go-merge-reviewer", "opus", "xhigh"), + "README.md": "not an agent\n", + }) + f := &fakeOpenRouter{models: orFixture(t, "models.json"), decisions: []string{orFixture(t, "decision-jev.json")}} + f.start(t) + t.Setenv("CLAUDE_CODE_EFFORT_LEVEL", "low") + + code, stdout, stderr := runCLI(t, "optimize", "--root", project, "--repo", repoRoot) + if code != exitOK { + t.Fatalf("exit = %d, want %d\n%s", code, exitOK, stderr) + } + for _, w := range []string{ + "Repo: ~/Web/agents-repo (from --repo).", + "Root: ~/Web/proj (from --root).", + "Note: $CLAUDE_CODE_EFFORT_LEVEL is set", + "Asked ~typesafe/jev-latest about 1 agent (cap $0.50, min confidence 0.60).", + "AGENT NOW SUGGESTED CONFIDENCE", + "go-merge-reviewer opus / xhigh keep (low confidence) sonnet 0.57 / xhigh 0.89", + "Spent $0.000082 (reported by OpenRouter). Dry run: nothing written.", + } { + if !strings.Contains(stdout, w) { + t.Errorf("report is missing %q:\n%s", w, stdout) + } + } + if strings.Contains(stdout, "SUBAGENT_MODEL") { + t.Errorf("unexpected note:\n%s", stdout) + } + assertNoKey(t, stdout, stderr) + if f.requests() != 1 { + t.Errorf("%d decision requests, want 1 (README.md is not an agent)", f.requests()) + } +} + +func TestOptimize_JSON(t *testing.T) { + project, repoRoot := optimizeProject(t, map[string]string{"a.md": agentFile("a", "sonnet", "high")}) + f := &fakeOpenRouter{models: orFixture(t, "models.json"), decisions: []string{orFixture(t, "decision-clef.json")}} + f.start(t) + t.Setenv("CLAUDE_CODE_SUBAGENT_MODEL", "haiku") + + code, stdout, stderr := runCLI(t, "optimize", "--json", "--advisor", "cloudflare/clef", "--root", project, "--repo", repoRoot) + if code != exitOK { + t.Fatalf("exit = %d\n%s", code, stderr) + } + var rep optimize.Report + if err := json.Unmarshal([]byte(stdout), &rep); err != nil { + t.Fatalf("stdout is not JSON: %v\n%s", err, stdout) + } + if rep.Advisor != "cloudflare/clef" || len(rep.Rows) != 1 || rep.Rows[0].SuggestedModel != "opus" || rep.Spent != 0.00045312 { + t.Errorf("report = %+v", rep) + } + if !reflect.DeepEqual(rep.Overrides, []string{"CLAUDE_CODE_SUBAGENT_MODEL"}) { + t.Errorf("overrides = %v", rep.Overrides) + } + for _, w := range []string{"Repo: ", "Root: ", "Note: $CLAUDE_CODE_SUBAGENT_MODEL is set"} { + if !strings.Contains(stderr, w) { + t.Errorf("%q belongs on stderr with --json:\n%s", w, stderr) + } + } +} + +func TestOptimize_JSONWriteFails(t *testing.T) { + project, repoRoot := optimizeProject(t, map[string]string{"a.md": agentFile("a", "opus", "high")}) + (&fakeOpenRouter{models: orFixture(t, "models.json"), decisions: []string{orFixture(t, "decision-clef.json")}}).start(t) + var stderr bytes.Buffer + if code := runOptimize([]string{"--json", "--root", project, "--repo", repoRoot}, failWriter{}, &stderr); code != exitFailure { + t.Errorf("exit = %d, want %d\n%s", code, exitFailure, stderr.String()) + } +} + +type failWriter struct{} + +func (failWriter) Write([]byte) (int, error) { return 0, errors.New("closed") } + +func TestOptimize_UnresolvedAndSkippedExitOne(t *testing.T) { + project, repoRoot := optimizeProject(t, map[string]string{ + "a.md": agentFile("a", "opus", "high"), + "b.md": agentFile("b", "opus", "high"), + }) + f := &fakeOpenRouter{models: orFixture(t, "models.json"), decisions: []string{`{"answers":{}}`}} + f.start(t) + t.Setenv("CLAUDE_CODE_SUBAGENT_MODEL", "haiku") + + code, stdout, stderr := runCLI(t, "optimize", "--root", project, "--repo", repoRoot) + if code != exitAtRisk { + t.Errorf("exit = %d, want %d", code, exitAtRisk) + } + for _, w := range []string{ + "Asked ~typesafe/jev-latest about 1 agent, 1 skipped (cap", + "a: unresolved: response reported no valid usage.cost", + "b: skipped: run stopped", + "1 agent was skipped (reasons above).", + "Note: $CLAUDE_CODE_SUBAGENT_MODEL is set", + } { + if !strings.Contains(stdout, w) { + t.Errorf("report is missing %q:\n%s", w, stdout) + } + } + assertNoKey(t, stdout, stderr) +} + +func TestOptimize_AgentSelectionAndDedup(t *testing.T) { + home := mustResolveDir(t, t.TempDir()) + t.Setenv("HOME", home) + repoRoot := filepath.Join(home, "Web", "agents-repo") + shared := filepath.Join(repoRoot, "agents", "shared.md") + mustWriteT(t, shared, agentFile("shared", "opus", "high")) + for _, p := range []string{"one", "two"} { + dir := filepath.Join(home, "Web", p, ".claude", "agents") + mustMkdirT(t, dir) + if err := os.Symlink(shared, filepath.Join(dir, "shared.md")); err != nil { + t.Fatal(err) + } + mustWriteT(t, filepath.Join(dir, "own.md"), agentFile("own", "opus", "high")) + } + clef := orFixture(t, "decision-clef.json") + f := &fakeOpenRouter{models: orFixture(t, "models.json"), decisions: []string{clef, clef, clef, clef}} + f.start(t) + web := filepath.Join(home, "Web") + + // The shared file is asked about once, under the first project linking it. + code, stdout, stderr := runCLI(t, "optimize", "--root", web, "--repo", repoRoot) + if code != exitOK { + t.Fatalf("exit = %d\n%s", code, stderr) + } + for _, w := range []string{"Web/one/own ", "Web/one/shared ", "Web/two/own "} { + if !strings.Contains(stdout, w) { + t.Errorf("report is missing %q:\n%s", w, stdout) + } + } + if f.requests() != 3 || strings.Contains(stdout, "Web/two/shared") { + t.Errorf("%d requests, want 3 with the shared file once:\n%s", f.requests(), stdout) + } + + code, stdout, stderr = runCLI(t, "optimize", "--agent", "shared", "--root", web, "--repo", repoRoot) + if code != exitOK || f.requests() != 4 || strings.Contains(stdout, "own") { + t.Errorf("--agent shared: exit %d, %d requests in all, want 4:\n%s%s", code, f.requests(), stdout, stderr) + } + + code, _, stderr = runCLI(t, "optimize", "--agent", "shared", "--agent", "nope", "--root", web, "--repo", repoRoot) + if code != exitFailure || !strings.Contains(stderr, "--agent nope: no Claude agent by that name") { + t.Errorf("exit = %d, stderr:\n%s", code, stderr) + } +} + +func TestOptimize_RefusesBeforeAnyRequest(t *testing.T) { + good := map[string]string{"a.md": agentFile("a", "opus", "high")} + for _, tc := range []struct { + name string + files map[string]string + setup func(t *testing.T, project string) + fake func(f *fakeOpenRouter) + args []string + code int + stderr string + lists int // models-list fetches; zero for every refusal before it + }{ + {name: "help", args: []string{"-h"}, code: exitOK}, + {name: "unknown flag", args: []string{"--nope"}, code: exitFailure, stderr: "flag provided but not defined"}, + {name: "positional", args: []string{"extra"}, code: exitFailure, stderr: `optimize takes no arguments, got "extra"`}, + {name: "empty advisor", args: []string{"--advisor", " "}, code: exitFailure, stderr: "--advisor is empty"}, + {name: "negative cap", args: []string{"--max-cost", "-1"}, code: exitFailure, stderr: "--max-cost must be"}, + {name: "infinite cap", args: []string{"--max-cost", "Inf"}, code: exitFailure, stderr: "--max-cost must be"}, + {name: "confidence above 1", args: []string{"--min-confidence", "1.5"}, code: exitFailure, stderr: "--min-confidence must be between 0 and 1"}, + {name: "no key", setup: func(t *testing.T, _ string) { t.Setenv(envOpenRouterKey, "") }, + code: exitFailure, stderr: "$OPENROUTER_API_KEY"}, + {name: "no repo", setup: func(t *testing.T, _ string) { t.Setenv(envRepo, "") }, + args: []string{"--repo", ""}, code: exitFailure, stderr: "no canonical agent repo configured"}, + {name: "root and all", args: []string{"--all"}, code: exitFailure, stderr: "mutually exclusive"}, + {name: "root missing", setup: func(t *testing.T, project string) { mustRemoveAll(t, project) }, + code: exitFailure, stderr: "walking"}, + {name: "scan problem", files: map[string]string{"dir.md/x": "x"}, code: exitFailure, stderr: "could not read every agent under the root"}, + {name: "broken link", setup: func(t *testing.T, project string) { + if err := os.Symlink(filepath.Join(project, "gone.md"), filepath.Join(project, ".claude", "agents", "b.md")); err != nil { + t.Fatal(err) + } + }, code: exitFailure, stderr: "could not read agent"}, + {name: "models list down", fake: func(f *fakeOpenRouter) { f.modelsStatus = 500 }, + code: exitFailure, stderr: "OpenRouter models list: OpenRouter HTTP 500", lists: 1}, + {name: "advisor unpriced", args: []string{"--advisor", "typesafe/jev-router"}, code: exitFailure, stderr: "no fixed input price", lists: 1}, + {name: "no claude family", fake: func(f *fakeOpenRouter) { + f.models = `{"data":[{"id":"~typesafe/jev-latest","pricing":{"prompt":"0.000000042"}}]}` + }, code: exitFailure, stderr: "has no anthropic/claude-- model", lists: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + files := tc.files + if files == nil { + files = good + } + project, repoRoot := optimizeProject(t, files) + f := &fakeOpenRouter{models: orFixture(t, "models.json")} + if tc.fake != nil { + tc.fake(f) + } + f.start(t) + if tc.setup != nil { + tc.setup(t, project) + } + args := append([]string{"optimize", "--root", project, "--repo", repoRoot}, tc.args...) + code, stdout, stderr := runCLI(t, args...) + if code != tc.code || !strings.Contains(stderr, tc.stderr) { + t.Errorf("exit = %d, want %d; stderr missing %q:\n%s", code, tc.code, tc.stderr, stderr) + } + if code != exitOK && stdout != "" { + t.Errorf("a refused run printed a report:\n%s", stdout) + } + if f.requests() != 0 || f.lists() != tc.lists { + t.Errorf("%d decision requests and %d models fetches; want 0 and %d", f.requests(), f.lists(), tc.lists) + } + assertNoKey(t, stdout, stderr) + }) + } +} + +func mustRemoveAll(t *testing.T, p string) { + t.Helper() + if err := os.RemoveAll(p); err != nil { + t.Fatal(err) + } +} diff --git a/coverage-floors.txt b/coverage-floors.txt index 54e23b4..f522875 100644 --- a/coverage-floors.txt +++ b/coverage-floors.txt @@ -1,12 +1,14 @@ # Minimum per-package statement coverage, checked by `just check`. # Raise a floor after a coverage pass; never lower one. -cmd/agentsync 91.2 +cmd/agentsync 92.0 internal/config 100.0 internal/discover 100.0 internal/doctor 88.1 internal/importer 88.8 internal/inventory 91.2 internal/linker 94.9 +internal/openrouter 100.0 +internal/optimize 100.0 internal/reconcile 84.3 internal/releasepolicy 96.4 internal/scan 91.6 diff --git a/docs/exit-codes.md b/docs/exit-codes.md index f26a85b..1fd43d3 100644 --- a/docs/exit-codes.md +++ b/docs/exit-codes.md @@ -17,6 +17,7 @@ What counts as exit 1, by command: | `doctor` | a broken or mismatched link, a stale capture, an orphaned capture directory, or an unreadable path | | `reconcile` | a refused selector, a conflict, a diverged or dereferenced link, or a removable link left in place | | `inventory --check` | the file is stale or absent | +| `optimize` | an agent's advice is unresolved, or it was skipped by the cost cap or a stopped run | `doctor`'s exit code is the one that asserts nothing has drifted; `reconcile`'s is deliberately narrower (see [reconcile](reconcile.md)). Tests assert these diff --git a/docs/optimize.md b/docs/optimize.md new file mode 100644 index 0000000..b244825 --- /dev/null +++ b/docs/optimize.md @@ -0,0 +1,121 @@ +# optimize + +`agentsync optimize` asks an OpenRouter decision model which Claude model and +reasoning effort each of a project's Claude agents should run on, and reports +the advice. It writes nothing. It is the only agentsync command that uses the +network. + +```bash +export OPENROUTER_API_KEY=... # required +agentsync optimize # the project you are in +agentsync optimize --agent go-reviewer # one agent +agentsync optimize --advisor cloudflare/clef --max-cost 0.05 +agentsync optimize --json +``` + +Use a dedicated OpenRouter key with a credit limit set on OpenRouter's side. +That limit is the backstop if anything here undercounts. + +## What it reads and sends + +The agents are the ones [scan](scan.md) finds under the root (the project's +`.claude/agents/`, linked or local); root and repo resolve as for every other +command (see [configuration](configuration.md)). An agent file linked into +several projects is asked about once. A `.md` file without `name` and +`description` frontmatter is not an agent and is skipped. + +For each agent it shows `model:` and `effort:` from the frontmatter as written +(without quotes or a trailing `# comment`), or `(not set)`. If `$CLAUDE_CODE_SUBAGENT_MODEL` or `$CLAUDE_CODE_EFFORT_LEVEL` +is set, it notes that Claude Code uses it over the frontmatter at run time. + +Each agent costs one `POST /api/alpha/decisions` request, with no retries. The +request holds one framing sentence, the first 24 KB of the agent file and two +questions: + +- `model`: a choice among the Claude Code aliases `haiku`, `sonnet`, `opus` and + `fable`, each with a one-line description. An alias is offered only if + OpenRouter's public models list has an `anthropic/claude--` + model. Claude Code resolves an alias to the newest model in its family. +- `effort`: a score over `low`, `medium`, `high`, `xhigh`, `max`, read as the + nearest level. Its confidence is the probability the advisor gave that level, + not the score's own `confidence` field. + +Nothing else from the project is sent. + +## Flags + +| Flag | Meaning | +|---|---| +| `--advisor MODEL` | the decision model to ask; default `~typesafe/jev-latest`, and `cloudflare/clef` works too | +| `--max-cost USD` | the most the run may spend; default `0.50` | +| `--min-confidence P` | the probability a pick needs before it is suggested; default `0.6` | +| `--agent NAME` | ask only about this agent; repeatable | +| `--json` | print the report as JSON on stdout (see below), with the Repo, Root and override notes on stderr | + +## Reading the report + +``` +AGENT NOW SUGGESTED CONFIDENCE +go-merge-reviewer opus / xhigh keep (low confidence) sonnet 0.57 / xhigh 0.89 +``` + +Model and effort are judged separately. A part is suggested only when the +advisor's pick differs from the current value and its probability is at least +`--min-confidence`. A differing pick under the threshold is dropped: a row that +still changes the other part names it, as in `opus / xhigh (effort: low +confidence)`, and a row with nothing left to change says `keep (low +confidence)`. `CONFIDENCE` shows both picks whether or not they were suggested. + +A row is `unresolved` when the request failed (OpenRouter's error message is +shown, including a guardrail's reason and settings link), the response was not +valid JSON, an answer or its probability was missing, or the advisor picked +something it was not offered. An unresolved row suggests nothing. + +## JSON + +`--json` prints one object: `advisor`, `rows`, `spent_usd`, and `overrides` +(the `$CLAUDE_CODE_*` variables that are set; omitted when none). Each row has: + +| Field | Meaning | +|---|---| +| `agent`, `path` | the agent's label and file | +| `model`, `effort` | the current values, or `(not set)` | +| `status` | `change`, `keep`, `unresolved` or `skipped` | +| `suggested_model`, `suggested_effort` | set only for a part that changes | +| `model_pick`, `effort_pick` | `{value, probability}` as the advisor answered; absent unless both were read | +| `model_low_confidence`, `effort_low_confidence` | `true` when that part's differing pick was dropped; omitted otherwise | +| `reason` | why a row is unresolved or skipped | +| `cost_usd` | the `usage.cost` OpenRouter reported for this row | + +## Cost + +The advisor's input price comes from OpenRouter's public models list; an +advisor that is not listed, or lists no fixed price, is refused before any +request. Before each request the cost is estimated as request bytes / 4 times +that price, or at the highest cost per request byte OpenRouter has reported so +far in the run if that is higher. The request is skipped if the estimate would +take the spend so far past `--max-cost`. Spend so far is the sum of the +`usage.cost` OpenRouter reports on each response. + +The run stops, and the remaining agents are skipped, whenever a request may +have been billed without a usable cost: a response with no or a negative +`usage.cost`, a successful response that was cut short or was not valid JSON, +or a connection error or timeout after sending. The row it happened on is +unresolved. An error status from OpenRouter is not billed and does not stop +the run. The last line prints the total reported spend. + +A model's billed price can differ from its listed one. The estimate learns +the billed rate from each response, so it is mostly the first request that +can be underestimated. The cap still holds to within one request, because +spend is what OpenRouter reports. + +## Exit codes + +- `0`: every agent got advice. +- `1`: a row is unresolved or skipped. +- `2`: no `$OPENROUTER_API_KEY`, bad flags, no repo or root configured, a scan + that could not read everything, an agent file that could not be read, or a + models list that is unreachable, has no fixed price for the advisor, or has + no Claude family. + +See [exit codes](exit-codes.md). diff --git a/internal/discover/discover.go b/internal/discover/discover.go index 124cf3f..5bef568 100644 --- a/internal/discover/discover.go +++ b/internal/discover/discover.go @@ -124,7 +124,7 @@ func agentCount(dir string) (int, error) { if err == nil && e.Type().IsRegular() && filepath.Ext(e.Name()) == ".md" { var data []byte data, err = os.ReadFile(filepath.Join(dir, "agents", e.Name())) - if err == nil && isAgent(string(data)) { + if err == nil && IsAgent(string(data)) { n++ } } @@ -132,21 +132,36 @@ func agentCount(dir string) (int, error) { return n, err } -// isAgent reports whether s opens with YAML frontmatter carrying top-level +// IsAgent reports whether s opens with YAML frontmatter carrying top-level // name and description keys, which is what Claude Code requires of an agent. -func isAgent(s string) bool { +func IsAgent(s string) bool { + fm, ok := Frontmatter(s) + _, name := fm["name"] + _, desc := fm["description"] + return ok && name && desc +} + +// Frontmatter returns the top-level keys of s's YAML frontmatter with their +// raw values trimmed, and false when s has none. Nested and list lines are +// skipped; the first occurrence of a key wins. +func Frontmatter(s string) (map[string]string, bool) { front, ok := strings.CutPrefix(strings.ReplaceAll(s, "\r\n", "\n"), "---\n") if !ok { - return false + return nil, false } front, _, ok = strings.Cut(front, "\n---") if !ok { - return false + return nil, false } - var name, desc bool + fm := map[string]string{} for _, line := range strings.Split(front, "\n") { - name = name || strings.HasPrefix(line, "name:") - desc = desc || strings.HasPrefix(line, "description:") + key, value, found := strings.Cut(line, ":") + if !found || key == "" || strings.ContainsAny(key[:1], " \t-#") { + continue + } + if _, dup := fm[key]; !dup { + fm[key] = strings.TrimSpace(value) + } } - return name && desc + return fm, true } diff --git a/internal/discover/discover_test.go b/internal/discover/discover_test.go index f601d7e..da152f8 100644 --- a/internal/discover/discover_test.go +++ b/internal/discover/discover_test.go @@ -284,13 +284,25 @@ func TestFind_StopsAtTheLimit(t *testing.T) { func TestIsAgent(t *testing.T) { for _, s := range []string{agentMD, "---\ndescription: d\nname: n\n---"} { - if !isAgent(s) { - t.Errorf("isAgent(%q) = false", s) + if !IsAgent(s) { + t.Errorf("IsAgent(%q) = false", s) } } for _, s := range []string{"", "---\n", "---\nname: n\n---\n", " ---\nname: n\ndescription: d\n---\n"} { - if isAgent(s) { - t.Errorf("isAgent(%q) = true", s) + if IsAgent(s) { + t.Errorf("IsAgent(%q) = true", s) } } } + +func TestFrontmatter(t *testing.T) { + s := "---\r\nname: n\r\nmodel: opus \r\nmodel: sonnet\r\n effort: max\r\n- effort: low\r\n# effort: low\r\n: x\r\ncolor\r\neffort:\r\ntools: Read # and Grep\r\n---\r\nmodel: haiku\r\n" + fm, ok := Frontmatter(s) + want := map[string]string{"name": "n", "model": "opus", "effort": "", "tools": "Read # and Grep"} + if !ok || !reflect.DeepEqual(fm, want) { + t.Errorf("Frontmatter = %v, %v; want %v, true", fm, ok, want) + } + if _, ok := Frontmatter("model: opus\n"); ok { + t.Error("Frontmatter found frontmatter in a file without any") + } +} diff --git a/internal/openrouter/openrouter.go b/internal/openrouter/openrouter.go new file mode 100644 index 0000000..27155d7 --- /dev/null +++ b/internal/openrouter/openrouter.go @@ -0,0 +1,174 @@ +// Package openrouter is the small slice of OpenRouter's API that optimize +// needs: the public models list and the alpha Decisions endpoint. +package openrouter + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// DefaultBaseURL is OpenRouter's API host. Tests point Client.BaseURL at an +// httptest server instead. +const DefaultBaseURL = "https://openrouter.ai" + +// maxBody bounds what is read from any response; the full models list is +// under 1 MB. +const maxBody = 16 << 20 + +// Client calls OpenRouter. Key is sent only on Decide; the models list is +// public. +type Client struct { + BaseURL string + Key string + HTTP *http.Client +} + +// New returns a client for the real API with a per-request timeout. +func New(key string) *Client { + return &Client{BaseURL: DefaultBaseURL, Key: key, HTTP: &http.Client{Timeout: 60 * time.Second}} +} + +// Model is one entry of the models list. Prices are USD per token, as the +// decimal strings OpenRouter sends; "-1" means variable. +type Model struct { + ID string `json:"id"` + Pricing struct { + Prompt string `json:"prompt"` + } `json:"pricing"` +} + +// Question is one typed question. Criteria is a map of option to description +// for "choice" and an ordered list of levels for "score". +type Question struct { + Type string `json:"type"` + Instructions string `json:"instructions"` + Criteria any `json:"criteria"` +} + +// DecisionRequest is the body of POST /api/alpha/decisions. +type DecisionRequest struct { + Model string `json:"model"` + State []string `json:"state"` + Questions map[string]Question `json:"questions"` +} + +// Answer is one question's answer. Pointers distinguish a missing field from +// a zero, which callers must refuse rather than read as an answer. +type Answer struct { + Type string `json:"type"` + Choice string `json:"choice"` + Score *float64 `json:"score"` + Probabilities map[string]float64 `json:"probabilities"` +} + +// Decision is a Decisions response. +type Decision struct { + Model string `json:"model"` + Answers map[string]Answer `json:"answers"` + Usage struct { + Cost *float64 `json:"cost"` + } `json:"usage"` +} + +// APIError is a non-2xx response, carrying OpenRouter's own message. +type APIError struct { + Status int + Message string + Metadata map[string]any +} + +func (e *APIError) Error() string { + msg := fmt.Sprintf("OpenRouter HTTP %d: %s", e.Status, e.Message) + // A guardrail refusal names why and where to change it; both are the fix. + if r, ok := e.Metadata["ineligibility_reasons"]; ok { + if b, err := json.Marshal(r); err == nil { + msg += "; reasons: " + string(b) + } + } + if u, ok := e.Metadata["configure_url"].(string); ok { + msg += "; configure at " + u + } + return msg +} + +// Models fetches the public models list. output_modalities=all is needed: +// the default list leaves out decision models such as Jev and Clef. +func (c *Client) Models(ctx context.Context) ([]Model, error) { + var out struct { + Data []Model `json:"data"` + } + if err := c.do(ctx, http.MethodGet, "/api/v1/models?output_modalities=all", nil, &out); err != nil { + return nil, err + } + if len(out.Data) == 0 { + return nil, fmt.Errorf("OpenRouter models list is empty") + } + return out.Data, nil +} + +// Decide posts one already-encoded DecisionRequest. The caller encodes it so +// the bytes it estimated the cost of are the bytes sent. +func (c *Client) Decide(ctx context.Context, body []byte) (Decision, error) { + var d Decision + err := c.do(ctx, http.MethodPost, "/api/alpha/decisions", body, &d) + return d, err +} + +func (c *Client) do(ctx context.Context, method, path string, body []byte, out any) error { + var rd io.Reader + if body != nil { + rd = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, c.BaseURL+path, rd) + if err != nil { + return err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+c.Key) + } + resp, err := c.HTTP.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + data, err := io.ReadAll(io.LimitReader(resp.Body, maxBody)) + if err != nil { + return fmt.Errorf("reading OpenRouter response: %w", err) + } + if resp.StatusCode/100 != 2 { + return apiError(resp.StatusCode, data) + } + if err := json.Unmarshal(data, out); err != nil { + return fmt.Errorf("OpenRouter sent malformed JSON: %w", err) + } + return nil +} + +func apiError(status int, data []byte) error { + var body struct { + Error struct { + Message string `json:"message"` + Metadata map[string]any `json:"metadata"` + } `json:"error"` + } + e := &APIError{Status: status} + if json.Unmarshal(data, &body) == nil && body.Error.Message != "" { + e.Message, e.Metadata = body.Error.Message, body.Error.Metadata + return e + } + e.Message = strings.TrimSpace(string(data)) + if len(e.Message) > 200 { + e.Message = e.Message[:200] + "..." + } + if e.Message == "" { + e.Message = http.StatusText(status) + } + return e +} diff --git a/internal/openrouter/openrouter_test.go b/internal/openrouter/openrouter_test.go new file mode 100644 index 0000000..c3f60f6 --- /dev/null +++ b/internal/openrouter/openrouter_test.go @@ -0,0 +1,207 @@ +package openrouter + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" +) + +func fixture(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile("testdata/" + name) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// serve answers every request with status and body. last returns the most +// recent request; the mutex is what the race detector sees, not the socket. +func serve(t *testing.T, status int, body string) (c *Client, last func() *http.Request) { + t.Helper() + var ( + mu sync.Mutex + got *http.Request + ) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + got = r.Clone(context.Background()) + mu.Unlock() + w.WriteHeader(status) + _, _ = io.WriteString(w, body) + })) + t.Cleanup(srv.Close) + c = New("sk-test") + c.BaseURL = srv.URL + return c, func() *http.Request { mu.Lock(); defer mu.Unlock(); return got } +} + +func TestModels_ParsesTheRealListAndSendsNoKey(t *testing.T) { + c, last := serve(t, 200, fixture(t, "models.json")) + models, err := c.Models(context.Background()) + if err != nil { + t.Fatal(err) + } + got := last() + if got.URL.String() != "/api/v1/models?output_modalities=all" || got.Method != http.MethodGet { + t.Errorf("request = %s %s", got.Method, got.URL) + } + if h := got.Header.Get("Authorization"); h != "" { + t.Errorf("the public list was sent Authorization %q", h) + } + var jev string + for _, m := range models { + if m.ID == "~typesafe/jev-latest" { + jev = m.Pricing.Prompt + } + } + if jev != "0.000000042" { + t.Errorf("jev-latest prompt price = %q, want 0.000000042", jev) + } +} + +func TestModels_EmptyListRefuses(t *testing.T) { + c, _ := serve(t, 200, `{"data":[]}`) + if _, err := c.Models(context.Background()); err == nil || !strings.Contains(err.Error(), "empty") { + t.Errorf("err = %v, want an empty-list refusal", err) + } +} + +func TestDecide_ParsesRealResponses(t *testing.T) { + for _, tc := range []struct { + file, choice string + prob, cost float64 + }{ + {"decision-jev.json", "sonnet", 0.57, 0.000082152}, + {"decision-clef.json", "opus", 0.668, 0.00045312}, + } { + t.Run(tc.file, func(t *testing.T) { + c, last := serve(t, 200, fixture(t, tc.file)) + d, err := c.Decide(context.Background(), []byte(`{}`)) + if err != nil { + t.Fatal(err) + } + got := last() + if got.Header.Get("Authorization") != "Bearer sk-test" || got.URL.Path != "/api/alpha/decisions" || got.Method != http.MethodPost { + t.Errorf("request = %s %s auth %q", got.Method, got.URL, got.Header.Get("Authorization")) + } + m := d.Answers["model"] + if m.Choice != tc.choice || m.Probabilities[tc.choice] != tc.prob { + t.Errorf("model answer = %+v", m) + } + if e := d.Answers["effort"]; e.Score == nil || len(e.Probabilities) != 5 { + t.Errorf("effort answer lost its score or probabilities: %+v", e) + } + if d.Usage.Cost == nil || *d.Usage.Cost != tc.cost { + t.Errorf("usage.cost = %v, want %v", d.Usage.Cost, tc.cost) + } + }) + } +} + +func TestDecide_ErrorsCarryOpenRouterMessage(t *testing.T) { + guardrail := `{"error":{"message":"Provider not allowed by guardrail","code":404,"metadata":{"ineligibility_reasons":["provider anthropic blocked"],"configure_url":"https://openrouter.ai/settings/guardrails"}}}` + for _, tc := range []struct { + name string + status int + body string + want []string + }{ + {"unknown model", 400, fixture(t, "error-unknown-model.json"), []string{"HTTP 400", "Model openai/gpt-6-luna does not exist"}}, + {"guardrail", 404, guardrail, []string{"HTTP 404", "Provider not allowed by guardrail", "provider anthropic blocked", "configure at https://openrouter.ai/settings/guardrails"}}, + {"plain 5xx", 502, "bad gateway\n", []string{"HTTP 502: bad gateway"}}, + {"empty 5xx", 503, "", []string{"HTTP 503: Service Unavailable"}}, + {"long body", 500, strings.Repeat("x", 300), []string{strings.Repeat("x", 200) + "..."}}, + } { + t.Run(tc.name, func(t *testing.T) { + c, _ := serve(t, tc.status, tc.body) + _, err := c.Decide(context.Background(), []byte(`{}`)) + var apiErr *APIError + if !errors.As(err, &apiErr) || apiErr.Status != tc.status { + t.Fatalf("err = %v, want an APIError with status %d", err, tc.status) + } + for _, w := range tc.want { + if !strings.Contains(err.Error(), w) { + t.Errorf("error %q is missing %q", err, w) + } + } + if strings.Contains(err.Error(), "sk-test") { + t.Errorf("error leaks the key: %v", err) + } + }) + } +} + +func TestDecide_MalformedJSON(t *testing.T) { + c, _ := serve(t, 200, `{"answers":`) + if _, err := c.Decide(context.Background(), []byte(`{}`)); err == nil || !strings.Contains(err.Error(), "malformed JSON") { + t.Errorf("err = %v, want malformed JSON", err) + } +} + +func TestDecide_TimesOut(t *testing.T) { + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-release: + case <-r.Context().Done(): + } + })) + t.Cleanup(func() { close(release); srv.Close() }) + c := &Client{BaseURL: srv.URL, Key: "sk-test", HTTP: &http.Client{Timeout: 50 * time.Millisecond}} + _, err := c.Decide(context.Background(), []byte(`{}`)) + if err == nil || !strings.Contains(err.Error(), "Timeout") { + t.Errorf("err = %v, want a client timeout", err) + } + if err != nil && strings.Contains(err.Error(), "sk-test") { + t.Errorf("error leaks the key: %v", err) + } +} + +func TestNew_HasATimeout(t *testing.T) { + if New("").HTTP.Timeout <= 0 { + t.Error("New's client can wait forever") + } +} + +func TestDo_TransportAndRequestErrors(t *testing.T) { + c := &Client{BaseURL: "http://127.0.0.1:0", Key: "sk-test", HTTP: &http.Client{}} + if _, err := c.Models(context.Background()); err == nil { + t.Error("a refused connection returned no error") + } + _, err := c.Decide(context.Background(), []byte(`{}`)) + if err == nil || strings.Contains(err.Error(), "sk-test") { + t.Errorf("err = %v, want a refused connection that does not name the key", err) + } + c.BaseURL = "://bad" + if _, err := c.Models(context.Background()); err == nil { + t.Error("an unparseable URL returned no error") + } +} + +func TestDo_CapsTheBody(t *testing.T) { + c, _ := serve(t, 200, `{"data":[{"id":"x"}],"pad":"`+strings.Repeat("a", maxBody)+`"}`) + if _, err := c.Models(context.Background()); err == nil || !strings.Contains(err.Error(), "malformed JSON") { + t.Errorf("err = %v, want the oversized body cut short", err) + } +} + +func TestDo_TruncatedBody(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "100") + _, _ = io.WriteString(w, `{"data":`) + })) + t.Cleanup(srv.Close) + c := New("") + c.BaseURL = srv.URL + if _, err := c.Models(context.Background()); err == nil || !strings.Contains(err.Error(), "reading OpenRouter response") { + t.Errorf("err = %v, want a read error", err) + } +} diff --git a/internal/openrouter/testdata/decision-clef.json b/internal/openrouter/testdata/decision-clef.json new file mode 100644 index 0000000..f48e68c --- /dev/null +++ b/internal/openrouter/testdata/decision-clef.json @@ -0,0 +1,42 @@ +{ + "model": "cloudflare/clef", + "answers": { + "model": { + "type": "choice", + "choice": "opus", + "probabilities": { + "haiku": 0.0153, + "sonnet": 0.2806, + "opus": 0.668, + "fable": 0.0361 + }, + "confidence": 0.3687 + }, + "effort": { + "type": "score", + "score": 2.2992, + "legend": { + "0": "low", + "1": "medium", + "2": "high", + "3": "xhigh", + "4": "max" + }, + "probabilities": { + "0": 0.0138, + "1": 0.0774, + "2": 0.5636, + "3": 0.2861, + "4": 0.0591 + }, + "confidence": 0.2615 + } + }, + "usage": { + "input_tokens": 1888, + "output_tokens": 0, + "cost": 0.00045312 + }, + "id": "gen-dec-1792341389-TqBYOiBGoPYLoycdCH2R", + "provider": "Cloudflare" +} diff --git a/internal/openrouter/testdata/decision-jev.json b/internal/openrouter/testdata/decision-jev.json new file mode 100644 index 0000000..ddf1a22 --- /dev/null +++ b/internal/openrouter/testdata/decision-jev.json @@ -0,0 +1,42 @@ +{ + "model": "typesafe/jev-1.14-20261014", + "answers": { + "model": { + "type": "choice", + "choice": "sonnet", + "probabilities": { + "haiku": 0, + "sonnet": 0.57, + "opus": 0.43, + "fable": 0 + }, + "confidence": 0.43 + }, + "effort": { + "type": "score", + "score": 2.91, + "legend": { + "0": "low", + "1": "medium", + "2": "high", + "3": "xhigh", + "4": "max" + }, + "probabilities": { + "0": 0, + "1": 0, + "2": 0.09, + "3": 0.89, + "4": 0.02 + }, + "confidence": 0.91 + } + }, + "usage": { + "input_tokens": 1956, + "output_tokens": 64, + "cost": 0.000082152 + }, + "id": "gen-dec-1792341390-hlnW8eAck1t4e6SsowM0", + "provider": "TypeSafe" +} diff --git a/internal/openrouter/testdata/decision-request-questions.json b/internal/openrouter/testdata/decision-request-questions.json new file mode 100644 index 0000000..b3f8490 --- /dev/null +++ b/internal/openrouter/testdata/decision-request-questions.json @@ -0,0 +1,23 @@ +{ + "model": { + "type": "choice", + "instructions": "Which Claude model family should this subagent run on?", + "criteria": { + "haiku": "Claude Haiku: fastest and cheapest, for simple, mechanical or high-volume work", + "sonnet": "Claude Sonnet: balanced quality and cost, strong at most coding and review work", + "opus": "Claude Opus: highest-quality reasoning for hard, high-stakes judgement", + "fable": "Claude Fable: frontier model for long autonomous knowledge work, most expensive" + } + }, + "effort": { + "type": "score", + "instructions": "How much reasoning effort does this subagent need?", + "criteria": [ + "low", + "medium", + "high", + "xhigh", + "max" + ] + } +} diff --git a/internal/openrouter/testdata/error-unknown-model.json b/internal/openrouter/testdata/error-unknown-model.json new file mode 100644 index 0000000..f9d500a --- /dev/null +++ b/internal/openrouter/testdata/error-unknown-model.json @@ -0,0 +1,6 @@ +{ + "error": { + "message": "Model openai/gpt-6-luna does not exist", + "code": 400 + } +} diff --git a/internal/openrouter/testdata/models.json b/internal/openrouter/testdata/models.json new file mode 100644 index 0000000..af1c1b0 --- /dev/null +++ b/internal/openrouter/testdata/models.json @@ -0,0 +1,76 @@ +{ + "data": [ + { + "id": "typesafe/jev-1.14", + "name": "TypeSafe: Jev 1.14", + "pricing": { + "prompt": "0.000000042", + "completion": "0" + } + }, + { + "id": "anthropic/claude-haiku-5.5", + "name": "Anthropic: Claude Haiku 5.5", + "pricing": { + "prompt": "0.0000001", + "completion": "0.0000005" + } + }, + { + "id": "cloudflare/clef", + "name": "Cloudflare: Clef", + "pricing": { + "prompt": "0.000000042", + "completion": "0" + } + }, + { + "id": "anthropic/claude-sonnet-5.5", + "name": "Anthropic: Claude Sonnet 5.5", + "pricing": { + "prompt": "0.000002", + "completion": "0.00001" + } + }, + { + "id": "typesafe/jev-router", + "name": "TypeSafe: Jev Router", + "pricing": { + "prompt": "-1", + "completion": "-1" + } + }, + { + "id": "anthropic/claude-opus-5.5", + "name": "Anthropic: Claude Opus 5.5", + "pricing": { + "prompt": "0.000004", + "completion": "0.00002" + } + }, + { + "id": "~typesafe/jev-latest", + "name": "TypeSafe: Jev Latest", + "pricing": { + "prompt": "0.000000042", + "completion": "0" + } + }, + { + "id": "anthropic/claude-fable-5.1", + "name": "Anthropic: Claude Fable 5.1", + "pricing": { + "prompt": "0.00001", + "completion": "0.00005" + } + }, + { + "id": "anthropic/claude-opus-4.8", + "name": "Anthropic: Claude Opus 4.8", + "pricing": { + "prompt": "0.000005", + "completion": "0.000025" + } + } + ] +} diff --git a/internal/optimize/optimize.go b/internal/optimize/optimize.go new file mode 100644 index 0000000..0302d26 --- /dev/null +++ b/internal/optimize/optimize.go @@ -0,0 +1,349 @@ +// Package optimize asks an OpenRouter decision model which Claude model and +// effort each agent should run on, and decides which answers are confident +// enough to suggest. It writes nothing. +package optimize + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "strconv" + "strings" + + "github.com/agentic-tooling/agentsync/internal/discover" + "github.com/agentic-tooling/agentsync/internal/openrouter" +) + +// NotSet is shown for a model or effort key the frontmatter lacks. +const NotSet = "(not set)" + +// stateCap keeps the agent file under Jev's long-context threshold of ~32 KB. +const stateCap = 24 << 10 + +const framing = "A Claude Code subagent definition follows. Decide which Claude model and reasoning effort it should run on: the cheapest choice that keeps quality for its role." + +// Family is a Claude Code model alias, which Claude Code resolves to the +// newest model of that family, so advice never names a stale version. +type Family struct { + Alias string + Description string +} + +var families = []Family{ + {"haiku", "Claude Haiku: fastest and cheapest, for simple, mechanical or high-volume work"}, + {"sonnet", "Claude Sonnet: balanced quality and cost, strong at most coding and review work"}, + {"opus", "Claude Opus: highest-quality reasoning for hard, high-stakes judgement"}, + {"fable", "Claude Fable: frontier model for long autonomous knowledge work, most expensive"}, +} + +// Efforts are Claude Code's effort levels in order; a score answer indexes it. +var Efforts = []string{"low", "medium", "high", "xhigh", "max"} + +// Status is a row's outcome. Only StatusChange suggests anything; the zero +// value is none of these and suggests nothing. +type Status string + +const ( + StatusChange Status = "change" + StatusKeep Status = "keep" + StatusUnresolved Status = "unresolved" + StatusSkipped Status = "skipped" +) + +// Agent is one agent file to ask about. +type Agent struct { + Label, Path, Content string +} + +// Pick is one answer and the probability the advisor gave it; for effort, the +// probability of the level nearest the score. +type Pick struct { + Value string `json:"value"` + Probability float64 `json:"probability"` +} + +// Row is the advice for one agent. Suggested fields are set only for the +// parts that change; a LowConfidence flag marks a differing pick that was +// dropped for falling under the threshold. +type Row struct { + Agent string `json:"agent"` + Path string `json:"path"` + Model string `json:"model"` + Effort string `json:"effort"` + Status Status `json:"status"` + SuggestedModel string `json:"suggested_model,omitempty"` + SuggestedEffort string `json:"suggested_effort,omitempty"` + ModelPick *Pick `json:"model_pick,omitempty"` + EffortPick *Pick `json:"effort_pick,omitempty"` + ModelLowConfidence bool `json:"model_low_confidence,omitempty"` + EffortLowConfidence bool `json:"effort_low_confidence,omitempty"` + Reason string `json:"reason,omitempty"` + Cost float64 `json:"cost_usd"` +} + +// Report is one run's rows and the spend OpenRouter reported for them. +// Overrides names the set environment variables Claude Code uses over the +// frontmatter; the caller fills it. +type Report struct { + Advisor string `json:"advisor"` + Rows []Row `json:"rows"` + Spent float64 `json:"spent_usd"` + Overrides []string `json:"overrides,omitempty"` +} + +// Count returns how many rows have status s. +func (r Report) Count(s Status) int { + n := 0 + for _, row := range r.Rows { + if row.Status == s { + n++ + } + } + return n +} + +// Config is what Run needs beyond the agents. Price is the advisor's input +// price in USD per token. +type Config struct { + Advisor string + Price float64 + MaxCost float64 + MinConfidence float64 + Families []Family +} + +// Current returns an agent's model and effort as written, without a trailing +// YAML comment or quotes, or NotSet. +func Current(content string) (model, effort string) { + fm, _ := discover.Frontmatter(content) + get := func(k string) string { + v := " " + fm[k] + for _, sep := range []string{" #", "\t#"} { + v, _, _ = strings.Cut(v, sep) + } + if v = strings.Trim(strings.TrimSpace(v), `"'`); v != "" { + return v + } + return NotSet + } + return get("model"), get("effort") +} + +// Available keeps the aliases OpenRouter lists an anthropic/claude-- model for. +func Available(models []openrouter.Model) []Family { + var out []Family + for _, f := range families { + for _, m := range models { + v, ok := strings.CutPrefix(m.ID, "anthropic/claude-"+f.Alias+"-") + if ok && v != "" && v[0] >= '0' && v[0] <= '9' { + out = append(out, f) + break + } + } + } + return out +} + +// InputPrice is the advisor's input price per token. A missing, variable +// ("-1") or unparseable price refuses: the cost cap cannot be kept without it. +func InputPrice(models []openrouter.Model, advisor string) (float64, error) { + for _, m := range models { + if m.ID != advisor { + continue + } + p, err := strconv.ParseFloat(m.Pricing.Prompt, 64) + if err != nil || !(p >= 0) || math.IsInf(p, 1) { + return 0, fmt.Errorf("advisor %s has no fixed input price in the OpenRouter models list (got %q), so the cost cap cannot be kept", advisor, m.Pricing.Prompt) + } + return p, nil + } + return 0, fmt.Errorf("advisor %s is not in the OpenRouter models list", advisor) +} + +// Body encodes the Decisions request for one agent file. +func Body(advisor, content string, fams []Family) []byte { + if len(content) > stateCap { + content = strings.ToValidUTF8(content[:stateCap], "") + } + criteria := make(map[string]string, len(fams)) + for _, f := range fams { + criteria[f.Alias] = f.Description + } + // Strings, string maps and slices always encode; there is no error to handle. + b, _ := json.Marshal(openrouter.DecisionRequest{ + Model: advisor, + State: []string{framing, content}, + Questions: map[string]openrouter.Question{ + "model": {Type: "choice", Instructions: "Which Claude model family should this subagent run on?", Criteria: criteria}, + "effort": {Type: "score", Instructions: "How much reasoning effort does this subagent need?", Criteria: Efforts}, + }, + }) + return b +} + +// Run asks about each agent in turn, one request each and no retries. A +// request whose estimate would take the reported spend past MaxCost is +// skipped. Anything that may have billed without a usable usage.cost stops +// the run, because spend can no longer be tracked. +func Run(ctx context.Context, c *openrouter.Client, agents []Agent, cfg Config) Report { + rep := Report{Advisor: cfg.Advisor, Rows: make([]Row, 0, len(agents))} + stopped := "" + // Listed price at bytes/4 tokens, raised to the highest cost per byte + // OpenRouter has reported, so an advisor billed above its listed price is + // underestimated mostly on the first request. + rate := cfg.Price / 4 + for _, a := range agents { + row := Row{Agent: a.Label, Path: a.Path} + row.Model, row.Effort = Current(a.Content) + body := Body(cfg.Advisor, a.Content, cfg.Families) + est := float64(len(body)) * rate + switch { + case stopped != "": + row.Status, row.Reason = StatusSkipped, stopped + case rep.Spent+est > cfg.MaxCost: + row.Status = StatusSkipped + row.Reason = fmt.Sprintf("estimated $%.6f would pass the $%.2f cap", est, cfg.MaxCost) + default: + d, err := c.Decide(ctx, body) + var apiErr *openrouter.APIError + switch { + case errors.As(err, &apiErr): + // OpenRouter does not bill a request it answers with an error status. + row.Status, row.Reason = StatusUnresolved, err.Error() + case err != nil: + stopped = "run stopped: spend can no longer be tracked" + row.Status, row.Reason = StatusUnresolved, err.Error()+"; the request may have been billed, so spend can no longer be tracked" + case d.Usage.Cost == nil || !(*d.Usage.Cost >= 0) || math.IsInf(*d.Usage.Cost, 1): + stopped = "run stopped: spend can no longer be tracked" + row.Status, row.Reason = StatusUnresolved, "response reported no valid usage.cost, so spend can no longer be tracked" + default: + row.Cost = *d.Usage.Cost + rep.Spent += row.Cost + rate = max(rate, row.Cost/float64(len(body))) + row = judge(row, d, cfg) + } + } + rep.Rows = append(rep.Rows, row) + } + return rep +} + +// judge maps the answers onto row, refusing anything not offered. +func judge(row Row, d openrouter.Decision, cfg Config) Row { + m, err := modelPick(d.Answers, cfg.Families) + if err == nil { + var e Pick + if e, err = effortPick(d.Answers); err == nil { + row.ModelPick, row.EffortPick = &m, &e + } + } + if err != nil { + row.Status, row.Reason = StatusUnresolved, err.Error() + return row + } + row.SuggestedModel, row.ModelLowConfidence = suggest(row.Model, m, cfg.MinConfidence) + row.SuggestedEffort, row.EffortLowConfidence = suggest(row.Effort, *row.EffortPick, cfg.MinConfidence) + row.Status = StatusKeep + if row.SuggestedModel != "" || row.SuggestedEffort != "" { + row.Status = StatusChange + } + return row +} + +// suggest returns the pick when it differs from current and clears min, and +// whether a differing pick was dropped for low confidence. +func suggest(current string, p Pick, min float64) (string, bool) { + switch { + case p.Value == current: + return "", false + case p.Probability < min: + return "", true + } + return p.Value, false +} + +func modelPick(answers map[string]openrouter.Answer, fams []Family) (Pick, error) { + a, ok := answers["model"] + if !ok || a.Type != "choice" { + return Pick{}, fmt.Errorf("no choice answer for model (got type %q)", a.Type) + } + offered := false + for _, f := range fams { + offered = offered || f.Alias == a.Choice + } + if !offered { + return Pick{}, fmt.Errorf("model answer %q is not an offered option", a.Choice) + } + p, ok := a.Probabilities[a.Choice] + if !ok { + return Pick{}, fmt.Errorf("model answer %q has no probability", a.Choice) + } + return Pick{a.Choice, p}, probability("model", p) +} + +func effortPick(answers map[string]openrouter.Answer) (Pick, error) { + a, ok := answers["effort"] + if !ok || a.Type != "score" || a.Score == nil { + return Pick{}, fmt.Errorf("no score answer for effort (got type %q)", a.Type) + } + i := math.Round(*a.Score) + if !(i >= 0 && i < float64(len(Efforts))) { + return Pick{}, fmt.Errorf("effort score %v is outside 0..%d", *a.Score, len(Efforts)-1) + } + level := Efforts[int(i)] + p, ok := a.Probabilities[strconv.Itoa(int(i))] + if !ok { + return Pick{}, fmt.Errorf("effort level %s has no probability", level) + } + return Pick{level, p}, probability("effort", p) +} + +// probability refuses a value outside [0, 1], which would otherwise clear any +// threshold. +func probability(q string, p float64) error { + if !(p >= 0 && p <= 1) { + return fmt.Errorf("%s probability %v is outside 0..1", q, p) + } + return nil +} + +// Now is the current setting, "model / effort". +func (r Row) Now() string { return r.Model + " / " + r.Effort } + +// Suggested is the setting to move to, naming a part dropped for low +// confidence, or why there is none. +func (r Row) Suggested() string { + switch r.Status { + case StatusChange: + m, e, note := r.Model, r.Effort, "" + if r.SuggestedModel != "" { + m = r.SuggestedModel + } + if r.SuggestedEffort != "" { + e = r.SuggestedEffort + } + if r.ModelLowConfidence { + note = " (model: low confidence)" + } + if r.EffortLowConfidence { + note = " (effort: low confidence)" + } + return m + " / " + e + note + case StatusKeep: + if r.ModelLowConfidence || r.EffortLowConfidence { + return "keep (low confidence)" + } + return "keep" + } + return string(r.Status) +} + +// Confidence is each pick with its probability, or "-" when nothing was picked. +func (r Row) Confidence() string { + if r.ModelPick == nil || r.EffortPick == nil { + return "-" + } + return fmt.Sprintf("%s %.2f / %s %.2f", r.ModelPick.Value, r.ModelPick.Probability, r.EffortPick.Value, r.EffortPick.Probability) +} diff --git a/internal/optimize/optimize_test.go b/internal/optimize/optimize_test.go new file mode 100644 index 0000000..6f46561 --- /dev/null +++ b/internal/optimize/optimize_test.go @@ -0,0 +1,345 @@ +package optimize + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/agentic-tooling/agentsync/internal/openrouter" +) + +const testdata = "../openrouter/testdata/" + +func fixture(t *testing.T, name string) string { + t.Helper() + b, err := os.ReadFile(testdata + name) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func models(t *testing.T) []openrouter.Model { + t.Helper() + var out struct{ Data []openrouter.Model } + if err := json.Unmarshal([]byte(fixture(t, "models.json")), &out); err != nil { + t.Fatal(err) + } + return out.Data +} + +// reply is one canned response; serve, when set, replaces status and body. +type reply struct { + status int + body string + serve http.HandlerFunc +} + +// advisor serves replies in order and counts the requests it was sent. A +// request past the last reply gets 418, so an unexpected one fails a count +// assertion instead of hanging the test. +func advisor(t *testing.T, replies ...reply) (*openrouter.Client, func() int) { + t.Helper() + var ( + mu sync.Mutex + n int + ) + next := func() (reply, bool) { + mu.Lock() + defer mu.Unlock() + n++ + if n > len(replies) { + return reply{}, false + } + return replies[n-1], true + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rep, ok := next() + switch { + case !ok: + w.WriteHeader(http.StatusTeapot) + case rep.serve != nil: + rep.serve(w, r) + default: + w.WriteHeader(rep.status) + _, _ = io.WriteString(w, rep.body) + } + })) + t.Cleanup(srv.Close) + c := openrouter.New("sk-test") + c.BaseURL = srv.URL + return c, func() int { mu.Lock(); defer mu.Unlock(); return n } +} + +func agent(label, model, effort string) Agent { + fm := "---\nname: " + label + "\ndescription: d\n" + if model != "" { + fm += "model: " + model + "\n" + } + if effort != "" { + fm += "effort: " + effort + "\n" + } + return Agent{Label: label, Path: "/p/" + label + ".md", Content: fm + "---\nbody\n"} +} + +func config() Config { + return Config{Advisor: "~typesafe/jev-latest", Price: 4.2e-8, MaxCost: 0.5, MinConfidence: 0.6, Families: families} +} + +func TestCurrent(t *testing.T) { + for _, tc := range []struct{ content, model, effort string }{ + {"---\nname: n\nmodel: opus\neffort: xhigh\n---\n", "opus", "xhigh"}, + {"---\nmodel: \"claude-opus-4-5\"\neffort: 'low'\n---\n", "claude-opus-4-5", "low"}, + {"---\nmodel: inherit\n---\n", "inherit", NotSet}, + {"---\nmodel:\neffort: \"\"\n---\n", NotSet, NotSet}, + {"no frontmatter\nmodel: opus\n", NotSet, NotSet}, + {"---\nmodel: opus # cheap enough\neffort: \"high\"\t# for now\n---\n", "opus", "high"}, + {"---\nmodel: # none yet\n---\n", NotSet, NotSet}, + } { + if m, e := Current(tc.content); m != tc.model || e != tc.effort { + t.Errorf("Current(%q) = %q, %q; want %q, %q", tc.content, m, e, tc.model, tc.effort) + } + } +} + +func TestAvailable(t *testing.T) { + if got := Available(models(t)); !reflect.DeepEqual(got, families) { + t.Errorf("real list: %v, want all four", got) + } + got := Available([]openrouter.Model{{ID: "anthropic/claude-opus-5.5"}, {ID: "anthropic/claude-sonnet-x"}, {ID: "anthropic/claude-haiku-"}, {ID: "openai/fable-5"}}) + if len(got) != 1 || got[0].Alias != "opus" { + t.Errorf("got %v, want opus alone", got) + } +} + +func TestInputPrice(t *testing.T) { + ms := append(models(t), openrouter.Model{ID: "nan"}, openrouter.Model{ID: "inf"}) + ms[len(ms)-2].Pricing.Prompt, ms[len(ms)-1].Pricing.Prompt = "NaN", "+Inf" + for _, id := range []string{"~typesafe/jev-latest", "cloudflare/clef"} { + if p, err := InputPrice(ms, id); err != nil || p != 4.2e-8 { + t.Errorf("%s: %v, %v; want 4.2e-8", id, p, err) + } + } + for id, want := range map[string]string{ + "typesafe/jev-router": "no fixed input price", + "nan": "no fixed input price", + "inf": "no fixed input price", + "openai/gpt-6-luna": "not in the OpenRouter models list", + } { + if _, err := InputPrice(ms, id); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%s: err = %v, want %q", id, err, want) + } + } +} + +func TestBody_MatchesTheSpikeRequestAndCapsState(t *testing.T) { + var req openrouter.DecisionRequest + content := strings.Repeat("a", stateCap-1) + "é" + "tail" + if err := json.Unmarshal(Body("cloudflare/clef", content, families), &req); err != nil { + t.Fatal(err) + } + if req.Model != "cloudflare/clef" || len(req.State) != 2 || req.State[0] != framing { + t.Errorf("model %q, state %d entries", req.Model, len(req.State)) + } + if req.State[1] != strings.Repeat("a", stateCap-1) { + t.Errorf("state not cut cleanly at %d bytes: %d bytes, tail %q", stateCap, len(req.State[1]), req.State[1][len(req.State[1])-3:]) + } + var got, want any + b, _ := json.Marshal(req.Questions) + _ = json.Unmarshal(b, &got) + _ = json.Unmarshal([]byte(fixture(t, "decision-request-questions.json")), &want) + if !reflect.DeepEqual(got, want) { + t.Errorf("questions differ from the request that worked:\n%s", b) + } +} + +func TestRun_RealResponses(t *testing.T) { + jev, clef := fixture(t, "decision-jev.json"), fixture(t, "decision-clef.json") + for _, tc := range []struct { + name, body, model, effort string + min float64 + status Status + suggested string + lowM, lowE bool + }{ + {"jev below threshold keeps", jev, "opus", "xhigh", 0.6, StatusKeep, "keep (low confidence)", true, false}, + {"jev at threshold changes", jev, "opus", "xhigh", 0.57, StatusChange, "sonnet / xhigh", false, false}, + {"jev above threshold changes", jev, "opus", "xhigh", 0.5, StatusChange, "sonnet / xhigh", false, false}, + {"jev agrees", jev, "sonnet", "xhigh", 0.6, StatusKeep, "keep", false, false}, + {"jev both parts low", jev, "opus", "low", 0.95, StatusKeep, "keep (low confidence)", true, true}, + {"clef changes model, effort low confidence", clef, "sonnet", "xhigh", 0.6, StatusChange, "opus / xhigh (effort: low confidence)", false, true}, + {"clef effort is the nearest level's probability", clef, "opus", "", 0.56, StatusChange, "opus / high", false, false}, + {"jev changes effort, model low confidence", jev, "opus", "medium", 0.6, StatusChange, "opus / xhigh (model: low confidence)", true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + c, _ := advisor(t, reply{status: 200, body: tc.body}) + cfg := config() + cfg.MinConfidence = tc.min + rep := Run(context.Background(), c, []Agent{agent("go-merge-reviewer", tc.model, tc.effort)}, cfg) + r := rep.Rows[0] + if r.Status != tc.status || r.Suggested() != tc.suggested || r.ModelLowConfidence != tc.lowM || r.EffortLowConfidence != tc.lowE { + t.Errorf("status %q suggested %q low %v/%v; want %q %q %v/%v (%s)", r.Status, r.Suggested(), + r.ModelLowConfidence, r.EffortLowConfidence, tc.status, tc.suggested, tc.lowM, tc.lowE, r.Reason) + } + if rep.Spent != r.Cost || r.Cost == 0 { + t.Errorf("spent %v, row cost %v", rep.Spent, r.Cost) + } + }) + } +} + +func TestRun_FailsClosed(t *testing.T) { + answers := func(model, effort string) string { + return `{"answers":{` + model + `,` + effort + `},"usage":{"cost":0.0001}}` + } + goodM := `"model":{"type":"choice","choice":"opus","probabilities":{"opus":0.9}}` + goodE := `"effort":{"type":"score","score":2,"probabilities":{"2":0.9}}` + for _, tc := range []struct { + name string + r reply + want string + }{ + {"unknown model 400", reply{status: 400, body: fixture(t, "error-unknown-model.json")}, "Model openai/gpt-6-luna does not exist"}, + {"5xx", reply{status: 503, body: "down"}, "HTTP 503: down"}, + {"malformed", reply{status: 200, body: `{"answers":`}, "malformed JSON"}, + {"no answers", reply{status: 200, body: `{"usage":{"cost":0.0001}}`}, "no choice answer for model"}, + {"option not offered", reply{status: 200, body: answers(`"model":{"type":"choice","choice":"gpt","probabilities":{"gpt":1}}`, goodE)}, `"gpt" is not an offered option`}, + {"wrong type", reply{status: 200, body: answers(`"model":{"type":"score","choice":"opus"}`, goodE)}, `got type "score"`}, + {"no probability", reply{status: 200, body: answers(`"model":{"type":"choice","choice":"opus","probabilities":{}}`, goodE)}, "has no probability"}, + {"probability above 1", reply{status: 200, body: answers(`"model":{"type":"choice","choice":"opus","probabilities":{"opus":1.5}}`, goodE)}, "outside 0..1"}, + {"no effort score", reply{status: 200, body: answers(goodM, `"effort":{"type":"score","probabilities":{"2":0.9}}`)}, "no score answer for effort"}, + {"no probability for the nearest level", reply{status: 200, body: answers(goodM, `"effort":{"type":"score","score":2.6,"confidence":0.9,"probabilities":{"2":0.9}}`)}, "effort level xhigh has no probability"}, + {"effort out of range", reply{status: 200, body: answers(goodM, `"effort":{"type":"score","score":4.6,"probabilities":{"4":0.9}}`)}, "outside 0..4"}, + {"negative effort probability", reply{status: 200, body: answers(goodM, `"effort":{"type":"score","score":1,"probabilities":{"1":-0.1}}`)}, "effort probability"}, + } { + t.Run(tc.name, func(t *testing.T) { + c, _ := advisor(t, tc.r) + r := Run(context.Background(), c, []Agent{agent("a", "opus", "high")}, config()).Rows[0] + if r.Status != StatusUnresolved || !strings.Contains(r.Reason, tc.want) { + t.Errorf("status %q reason %q; want unresolved with %q", r.Status, r.Reason, tc.want) + } + if r.Suggested() != "unresolved" || r.Confidence() != "-" || r.SuggestedModel != "" || r.SuggestedEffort != "" { + t.Errorf("an unresolved row suggests something: %+v", r) + } + }) + } +} + +func TestRun_UntrackedSpendStopsTheRun(t *testing.T) { + // The server notices the client hanging up only once the body is read. + hang := func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + select { + case <-r.Context().Done(): + case <-time.After(5 * time.Second): + } + } + truncate := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "100") + _, _ = io.WriteString(w, `{"answers":`) + } + for _, tc := range []struct { + name string + r reply + want string + }{ + {"missing cost", reply{status: 200, body: `{"answers":{}}`}, "no valid usage.cost"}, + {"negative cost", reply{status: 200, body: `{"answers":{},"usage":{"cost":-1}}`}, "no valid usage.cost"}, + {"malformed 200", reply{status: 200, body: `{"answers":`}, "malformed JSON"}, + {"truncated 200", reply{serve: truncate}, "reading OpenRouter response"}, + {"timeout", reply{serve: hang}, "Timeout"}, + } { + t.Run(tc.name, func(t *testing.T) { + c, sent := advisor(t, tc.r) + c.HTTP.Timeout = 200 * time.Millisecond + rep := Run(context.Background(), c, []Agent{agent("a", "", ""), agent("b", "", ""), agent("c", "", "")}, config()) + if sent() != 1 { + t.Errorf("%d requests sent, want 1", sent()) + } + r := rep.Rows[0] + if r.Status != StatusUnresolved || !strings.Contains(r.Reason, tc.want) || !strings.Contains(r.Reason, "spend can no longer be tracked") { + t.Errorf("first row %+v, want unresolved with %q", r, tc.want) + } + if rep.Count(StatusSkipped) != 2 || !strings.Contains(rep.Rows[2].Reason, "run stopped") { + t.Errorf("rest %+v", rep.Rows[1:]) + } + if rep.Spent != 0 || strings.Contains(r.Reason, "sk-test") { + t.Errorf("spent %v; reason %q", rep.Spent, r.Reason) + } + }) + } + t.Run("an error status does not stop the run", func(t *testing.T) { + c, sent := advisor(t, reply{status: 503, body: "down"}) + rep := Run(context.Background(), c, []Agent{agent("a", "", ""), agent("b", "", "")}, config()) + if sent() != 2 || rep.Count(StatusUnresolved) != 2 { + t.Errorf("sent %d, rows %+v", sent(), rep.Rows) + } + }) +} + +func TestRun_CostCap(t *testing.T) { + costly := `{"answers":{"model":{"type":"choice","choice":"opus","probabilities":{"opus":0.9}},"effort":{"type":"score","score":2,"probabilities":{"2":0.9}}},"usage":{"cost":0.2}}` + cheap := strings.Replace(costly, `"cost":0.2`, `"cost":0.1`, 1) + agents := []Agent{agent("a", "opus", "high"), agent("b", "opus", "high"), agent("c", "opus", "high")} + t.Run("running total of usage.cost", func(t *testing.T) { + c, sent := advisor(t, reply{status: 200, body: cheap}, reply{status: 200, body: cheap}) + cfg := config() + cfg.MaxCost = 0.25 + rep := Run(context.Background(), c, agents, cfg) + // a and b fit; c's $0.10 estimate does not fit on top of $0.20 spent. + if sent() != 2 || rep.Spent != 0.2 { + t.Errorf("sent %d, spent %v", sent(), rep.Spent) + } + if rep.Count(StatusKeep) != 2 || rep.Count(StatusSkipped) != 1 { + t.Errorf("rows %+v", rep.Rows) + } + }) + t.Run("estimate learns the billed rate", func(t *testing.T) { + c, sent := advisor(t, reply{status: 200, body: costly}) + cfg := config() + cfg.MaxCost = 0.3 + rep := Run(context.Background(), c, agents, cfg) + // At the listed price b would look free; a's reported $0.20 says it costs as much again. + if sent() != 1 || rep.Spent != 0.2 || rep.Count(StatusSkipped) != 2 { + t.Errorf("sent %d, spent %v, rows %+v", sent(), rep.Spent, rep.Rows) + } + if !strings.Contains(rep.Rows[1].Reason, "estimated $0.2") { + t.Errorf("b's reason %q", rep.Rows[1].Reason) + } + }) + t.Run("estimate alone", func(t *testing.T) { + c, sent := advisor(t) + cfg := config() + cfg.MaxCost = 0 + rep := Run(context.Background(), c, []Agent{agent("a", "", "")}, cfg) + if sent() != 0 || rep.Rows[0].Status != StatusSkipped || !strings.Contains(rep.Rows[0].Reason, "would pass the $0.00 cap") { + t.Errorf("sent %d, row %+v", sent(), rep.Rows[0]) + } + if rep.Rows[0].Suggested() != "skipped" { + t.Errorf("suggested %q", rep.Rows[0].Suggested()) + } + }) +} + +func TestRow_Display(t *testing.T) { + r := Row{Model: NotSet, Effort: "high", Status: StatusChange, SuggestedModel: "opus", + ModelPick: &Pick{"opus", 0.668}, EffortPick: &Pick{"high", 0.2615}} + if r.Now() != "(not set) / high" || r.Suggested() != "opus / high" || r.Confidence() != "opus 0.67 / high 0.26" { + t.Errorf("now %q suggested %q confidence %q", r.Now(), r.Suggested(), r.Confidence()) + } + r.SuggestedModel, r.SuggestedEffort = "", "max" + if r.Suggested() != "(not set) / max" { + t.Errorf("effort-only change shows %q", r.Suggested()) + } + if (Row{}).Suggested() != "" { + t.Error("the zero row suggests something") + } +}