diff --git a/.github/workflows/live-tests.yml b/.github/workflows/live-tests.yml index 5bd0ed13..d210dcc6 100644 --- a/.github/workflows/live-tests.yml +++ b/.github/workflows/live-tests.yml @@ -37,8 +37,7 @@ jobs: # There is no live-anthropic job: Anthropic is not a provider of this # product (ADR 0001, decided 2026-08-21). The frozen adapter retires under - # #430. The next live job to add is live-huggingface, with the provider - # itself (#484). + # #430. live-dartmouth: name: Live Dartmouth provider (free models) @@ -74,3 +73,37 @@ jobs: # otherwise pass having proved nothing. Require real passes. grep -qE "[1-9][0-9]* passed" dartmouth-live.txt \ || { echo "No Dartmouth live test actually passed; coverage is not real."; exit 1; } + + live-huggingface: + name: Live HuggingFace Inference API (free routes) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install + run: | + python -m pip install --upgrade pip + # No provider extra: the router is OpenAI-compatible and the + # adapter speaks it with aiohttp, which is already a core dependency. + python -m pip install -e . + python -m pip install pytest pytest-asyncio pytest-timeout + + - name: Run live HuggingFace tests + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + ORCHESTRATOR_REQUIRE_LIVE: "1" + run: | + if [ -z "${HF_TOKEN}" ]; then + echo "HF_TOKEN is not configured; cannot verify HuggingFace support." + exit 1 + fi + python -m pytest -m live -k huggingface -v | tee huggingface-live.txt + # Free routes are promos and every one can legitimately be flapping + # at once, in which case every generation test skips and the job + # would otherwise pass having proved nothing. Require real passes. + grep -qE "[1-9][0-9]* passed" huggingface-live.txt \ + || { echo "No HuggingFace live test actually passed; coverage is not real."; exit 1; } diff --git a/.gitignore b/.gitignore index ed8ac2b0..d61baa37 100644 --- a/.gitignore +++ b/.gitignore @@ -286,3 +286,9 @@ tests/performance/results/ tests/performance/alerts/ tests/quality/results/ tests/scenarios/results/ + +# opencode/OMC session tooling state +.omo/ + +# minikernel prototype scratch +scripts/prototypes/**/__pycache__/ diff --git a/README.md b/README.md index 3a02172d..27d58f4f 100644 --- a/README.md +++ b/README.md @@ -30,12 +30,11 @@ provider to pass live acceptance tests** (`live-dartmouth`, 9 passed remotely on 2026-08-01) and is the one provider described here as supported. **Provider policy: Dartmouth Chat and HuggingFace (Inference API) only.** -Anthropic, OpenAI, Google and Ollama adapters remain in the tree but are -unsupported, are not advertised, and are being retired -([#430](https://github.com/ContextLab/orchestrator/issues/430)). HuggingFace -support means the hosted Inference API and is in progress -([#484](https://github.com/ContextLab/orchestrator/issues/484)); it is not -claimed to work until its live job passes. +The Anthropic, OpenAI, Google and Ollama adapters were retired +([#430](https://github.com/ContextLab/orchestrator/issues/430)) and are no +longer shipped. HuggingFace support means the hosted Inference API and is in +progress ([#484](https://github.com/ContextLab/orchestrator/issues/484)); it +is not claimed to work until its live job passes. **The wider legacy test suite is not green.** Only the marked `unit`/`contract`/`e2e` layer gates the build. The remainder were written @@ -47,8 +46,8 @@ That job is marked `continue-on-error`, which means GitHub reports it green **regardless of the result** — so its check mark says nothing about the suite. The real numbers are in the job's run summary, in a warning annotation on the run page, and in its `legacy-suite-results` artifact. -As of the most recent run: **434 failed, 248 errors, 1827 passed, 227 -skipped**. Track it in +As of the post-retirement re-baseline (full local run, 2026-08-21): +**500 failed, 93 errors, 1707 passed, 109 skipped**. Track it in [#354](https://github.com/ContextLab/orchestrator/issues/354) rather than trusting a number maintained by hand here, which has been wrong before. @@ -152,10 +151,9 @@ pip install "py-orc[all]" # every runtime extra Neither supported provider needs an extra: Dartmouth Chat and the HuggingFace Inference API are both spoken over HTTP with `aiohttp`, already a core -dependency. The `anthropic`, `openai` and `google` extras still exist for the -frozen adapters, which are unsupported and being retired -([#430](https://github.com/ContextLab/orchestrator/issues/430)) — do not build -on them. +dependency. The `anthropic`, `openai` and `google` extras were removed with +the retired adapters +([#430](https://github.com/ContextLab/orchestrator/issues/430)). A missing extra disables only the feature that needs it; it never breaks `import orchestrator`. @@ -324,12 +322,10 @@ The supported providers need no configuration file: [#484](https://github.com/ContextLab/orchestrator/issues/484); `HF_TOKEN` will be the credential. -A `~/.orchestrator/models.yaml` with `source:` entries (`ollama`, -`huggingface`, `openai`, `anthropic`, `google`) is still read, but every one -of those sources routes through the frozen adapter layer — unsupported, and -being retired under -[#430](https://github.com/ContextLab/orchestrator/issues/430). New work should -not depend on them. +A `~/.orchestrator/models.yaml` written before the provider retirement may +still name `ollama`, `huggingface`, `openai`, `anthropic` or `google` +sources; each such entry is skipped with a warning at population time — an +old config file is not an error, but it no longer registers anything. ## Advanced Example @@ -545,10 +541,10 @@ intent: |-|-|-| | Dartmouth Chat | — | **Supported** (free models). `live-dartmouth` green: 9 passed, 2026-08-01 | | HuggingFace (Inference API) | — | In progress ([#484](https://github.com/ContextLab/orchestrator/issues/484)) — not claimed to work until its live job passes | -| Anthropic | `anthropic` | Not a provider of this product — frozen adapter, retiring under [#430](https://github.com/ContextLab/orchestrator/issues/430) | -| OpenAI | `openai` | Not a provider of this product — frozen adapter, retiring under [#430](https://github.com/ContextLab/orchestrator/issues/430) | -| Google | `google` | Not a provider of this product — frozen adapter, retiring under [#430](https://github.com/ContextLab/orchestrator/issues/430) | -| Ollama (local) | — | Not a provider of this product — frozen adapter, retiring under [#430](https://github.com/ContextLab/orchestrator/issues/430) | +| Anthropic | `anthropic` | Retired under [#430](https://github.com/ContextLab/orchestrator/issues/430) — no longer shipped | +| OpenAI | `openai` | Retired under [#430](https://github.com/ContextLab/orchestrator/issues/430) — no longer shipped | +| Google | `google` | Retired under [#430](https://github.com/ContextLab/orchestrator/issues/430) — no longer shipped | +| Ollama (local) | — | Retired under [#430](https://github.com/ContextLab/orchestrator/issues/430) — no longer shipped | A provider is only called **supported** once the `live-tests` workflow passes for it remotely. "Verified locally" means its live tests were run by hand diff --git a/docs/adr/0001-product-contract.md b/docs/adr/0001-product-contract.md index 1694e89d..32f3bb89 100644 --- a/docs/adr/0001-product-contract.md +++ b/docs/adr/0001-product-contract.md @@ -80,10 +80,16 @@ passes. are not the supported path; the Inference API adapter is new work, tracked in #484. - **Anthropic, OpenAI, Google and Ollama are not providers of this product.** - Their adapters remain in the tree but are unsupported, must not be - advertised, and are retired under #430. The earlier plan to bring Anthropic - under live acceptance tests is withdrawn, and with it #432 (the - credit-blocked verification) and the `live-anthropic` CI job. + Their adapters were removed from the tree on 2026-08-21 (the first #430 + cut: `integrations/` adapters, `models/anthropic_model.py`, + `models/openai_model.py`, `models/providers/anthropic_provider.py`, the + skills-era `models/registry.py` + `models/config.py` that only ever + supported Anthropic, `tools/update_models.py`, and the packaged default + model pool in `config/models.yaml`, which listed only retired providers). + A `models.yaml` written before the retirement is skipped entry-by-entry + with a warning, never raised on. The earlier plan to bring Anthropic under + live acceptance tests is withdrawn, and with it #432 (the credit-blocked + verification) and the `live-anthropic` CI job. - A provider earns the word **supported** only when its `live-tests` job passes remotely. As of 2026-08-21: - **Dartmouth Chat: supported.** `live-dartmouth` passed with 9 tests diff --git a/docs/index.rst b/docs/index.rst index 1ca9005d..144f18fd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -57,4 +57,4 @@ Contribute Unsupported areas ----------------- -The supported providers are Dartmouth Chat (live-tested) and the HuggingFace Inference API (in progress). Anthropic, OpenAI, Google and Ollama adapters are present but frozen, and are being retired. Multimodal tools, MCP integration, monitoring, analytics, and deployment code are present but not part of the verified product surface. See the product contract for the precise boundary. +The supported providers are Dartmouth Chat (live-tested) and the HuggingFace Inference API (in progress). The Anthropic, OpenAI, Google and Ollama adapters were retired and are no longer shipped. Multimodal tools, MCP integration, monitoring, analytics, and deployment code are present but not part of the verified product surface. See the product contract for the precise boundary. diff --git a/pyproject.toml b/pyproject.toml index a8b2e965..389bfefc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,9 +48,9 @@ dependencies = [ [project.optional-dependencies] # --- Model providers ------------------------------------------------------- -anthropic = ["anthropic>=0.7.0"] -openai = ["openai>=1.0.0"] -google = ["google-generativeai>=0.3.0"] +# None. The supported providers (Dartmouth Chat, HuggingFace Inference API) +# are spoken over HTTP with aiohttp, a core dependency. The anthropic / +# openai / google extras retired with their adapters (#430). # --- Graph runtime --------------------------------------------------------- langgraph = [ @@ -127,7 +127,7 @@ notebooks = [ # import name (orchestrator). The previous value referenced a package that # does not exist on any index. all = [ - "py-orc[anthropic,openai,google,langgraph,web,multimedia,viz,infra,crypto]", + "py-orc[langgraph,web,multimedia,viz,infra,crypto]", ] all-dev = [ "py-orc[all,dev,docs,notebooks]", @@ -226,6 +226,13 @@ markers = [ "integration: needs local services (docker/redis/postgres); opt-in", "live: calls a real model provider API and costs money; opt-in", "e2e: full pipeline through the installed CLI/API", + # pytest 9 validates builtin marks applied through a module-level + # `pytestmark` list against this table, so `filterwarnings` has to be + # declared here even though pytest registers it itself. Without this the + # blocking suite fails at COLLECTION on test_failure_policy.py and + # test_supported_examples.py, which both promote teardown warnings to + # errors that way. + "filterwarnings: builtin; declared so --strict-markers accepts it in a module-level pytestmark list under pytest >= 9", # Resource requirements "local: marks tests as local-only (not run in CI)", "slow: marks tests as slow (deselect with '-m \"not slow\"')", @@ -260,8 +267,6 @@ module = [ "docker.*", "redis.*", "psycopg2.*", - "openai.*", - "anthropic.*", ] ignore_missing_imports = true diff --git a/scripts/prototypes/README.md b/scripts/prototypes/README.md new file mode 100644 index 00000000..da19d08b --- /dev/null +++ b/scripts/prototypes/README.md @@ -0,0 +1,106 @@ +# `minikernel` — an executable schematic for #485 + +A deliberately small, runnable kernel for the redesign proposed in +[#485](https://github.com/ContextLab/orchestrator/issues/485). Not a product, +and not on the ADR-0001 path: nothing under `src/` imports it, and it imports +nothing from `src/`. It exists so the issue's claims can be **executed** rather +than argued about. + +The three design reviews on #485 all reached their conclusions by modelling +mechanisms in isolation (see `scripts/simulations/`). This does the other half: +it puts every mechanism the issue proposes into one running system, in its +cheapest honest form, and reports what breaks. + +```bash +.venv/bin/python scripts/prototypes/run_scenarios.py # 55 end-to-end checks +.venv/bin/python -m pytest tests/test_minikernel.py -q # 57 unit tests +.venv/bin/python scripts/prototypes/measure_ambiguity.py # measured f (cached) +.venv/bin/python scripts/prototypes/probe_optimism.py # is low f real? (cached) +``` + +## Modules + +| module | #485 component | what it is | +|-|-|-| +| `store.py` | 3 | one substrate: append-only event log, content-addressed blobs, sealed-segment journal, summary DAG, FTS. The scratchpad, insight pool, context tables and tool history are **queries over it**, not four stores. | +| `ir.py` | 1 | typed plan IR (sequence / branch / bounded loop / call / decompose), its validator, `Authority`, and the `Budget` ledger. No `goto`. | +| `capabilities.py` | 3 | one lifecycle for tools, skills and reusable plans: `draft → candidate → trusted`, `quarantined`, `revoked`, plus the bug-report/triage workflow. | +| `library.py` | 1 + 3 | the solved-problem library — two-key retrieval (statement similarity **and** typed I/O signature) and, added after the harness demanded it, **negative results**. | +| `review.py` | 1a | separation of duty, frozen criteria, concern ledger, evidential gate, insight pool with contradiction detection. | +| `planner.py` | 1 | `StubPlanner` (deterministic, for sweeps) and `LLMPlanner` (a real model over stdlib `urllib`, for measurement). | +| `runtime.py` | 2 | the durable executor: nested runs, crash-resume, budget escalation, addressed message bus, admission control. | + +## What the harness is for + +Each scenario in `run_scenarios.py` asserts a property the design needs. When a +scenario failed, the **kernel** was changed, not the assertion — and six of +those changes are design findings, not typos: + +1. **The library must store negative results.** A mission that ends at the depth + cap taught the system nothing, so re-running it cost exactly as much, + forever. The one regime where learning matters was the one regime where + learning could not start. With dead-end memory a repeated unreachable + mission costs 5 nodes instead of 20 — and still never reports success. +2. **An escalating sibling must not cancel the others.** Returning on the first + unreachable subtree threw away every sibling that was still solvable, and + with them everything the run would have learned. +3. **A wildcard in a stored signature defeats the two-key match.** A solution + published as `any->any` matches every later query, so text similarity + silently becomes the only key. Untyped solutions are no longer published. +4. **Budget exhaustion must not count against a cached plan.** It says nothing + about whether the plan was right, but counting it as a failure dropped + reliability below the retrieval floor after one unlucky mission. +5. **A planner must be a function of its inputs.** The first `StubPlanner` + carried one RNG stream across calls, so asking the same question twice gave + different plans and the library's benefit was unmeasurable. +6. **`atomic` has to be a checked claim, not a label** — see below. + +## The measurement that matters + +`f` — the fraction of steps a planner marks ambiguous — is the load-bearing +parameter: recursion is finite in expectation iff `m = b·f < 1`. Every prior +review had to guess it. Measured on 16 real problems across four library tiers +(`measurements/ambiguity.json`): + +``` +model library #caps mean b mean f m (correct) regime +gpt-5.4-mini L0_bare 0 2.50 1.000 2.50 SUPERCRITICAL +gpt-5.4-mini L1_minimal 4 2.73 0.483 1.33 SUPERCRITICAL +gpt-5.4-mini L2_working 12 3.38 0.302 0.88 subcritical +gpt-5.4-mini L3_mature 30 4.31 0.489 1.88 SUPERCRITICAL +gpt-5.6-sol L2_working 12 4.80 0.013 0.07 subcritical +``` + +Two things to note. **The library helps and then hurts** — a bigger menu drives +`f` down but drives `b` up, and `m` is their product. And **the estimator +matters**: `m` is `E[ambiguous children]`, not `mean(b)·mean(f)`; the product of +the means called L2 critical when it is subcritical. + +Then `probe_optimism.py` asks an independent judge model whether each +*declared-atomic* step can really be done by the capability it names: + +``` +planner plans mean b declared m overclaim rate corrected m +gpt-5.4-mini 8 3.00 1.88 4/9 = 44% 2.38 +gpt-5.6-sol 8 4.38 0.12 12/34 = 35% 1.62 +``` + +The stronger planner's apparent `m` of 0.12 is really **1.62**. It was not less +ambiguous; it was more optimistic. Declared `f` is not a safety metric, and the +cheapest way for any planner to look like it terminates is to lie about what is +easy. That is why `Runtime` takes an `admission` hook, and why scenario S10 +shows the same run reporting `completed` with a placeholder answer without it +and `escalated` with it. + +## Assumptions + +- Token counts are a `len/3.5` estimate, deliberately over-counting. A real + implementation must use the selected model's tokeniser. +- `ReviewBoard.detection_rate_prior` (0.6) is a **guess**, carried through to + every reported `residual_risk`. It should be replaced by a measured + per-reviewer detection rate as soon as there is one. +- The measured `f` numbers are properties of *(model, problem distribution, + library contents)* and of these 16 problems in particular. Quote all three. +- The judge in `probe_optimism.py` is one model (`gpt-5.5`). Its own error rate + is unmeasured; the overclaim rates are therefore lower-confidence than the + direction of the effect. diff --git a/scripts/prototypes/measure_ambiguity.py b/scripts/prototypes/measure_ambiguity.py new file mode 100644 index 00000000..3466fabf --- /dev/null +++ b/scripts/prototypes/measure_ambiguity.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Measure `f` -- the parameter the whole #485 design rests on. + +Recursive decomposition is a Galton-Watson branching process: a node emits `b` +steps, each ambiguous with probability `f`, and the process is finite in +expectation iff `m = b*f < 1`. Every review of #485 so far has had to GUESS +`f`. This script measures it, with a real planner, on real problems, against a +real capability library -- and measures how it moves as the library grows, +which is the claim that the library is the termination mechanism rather than an +efficiency nicety. + + .venv/bin/python scripts/prototypes/measure_ambiguity.py # cached + .venv/bin/python scripts/prototypes/measure_ambiguity.py --refresh # re-call + +Results are cached in measurements/ambiguity.json so the numbers are +reproducible without re-spending tokens. +""" + +from __future__ import annotations + +import argparse +import json +import os +import statistics +import sys +from collections import defaultdict + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from minikernel import LLMPlanner, load_env_key + +HERE = os.path.dirname(os.path.abspath(__file__)) +CACHE = os.path.join(HERE, "measurements", "ambiguity.json") + +# A capability pool that grows in realistic tiers. The point of the tiers is +# that `f` is NOT a property of the model: it is a property of +# (model, problem distribution, library contents). +TIERS: dict[str, list[str]] = { + "L0_bare": [], + "L1_minimal": ["read_file@1", "write_file@1", "run_python@1", "llm_transform@1"], + "L2_working": [ + "read_file@1", "write_file@1", "run_python@1", "llm_transform@1", + "web_search@1", "fetch_url@1", "extract_text@1", "summarize@1", + "parse_csv@1", "sql_query@1", "regex_extract@1", "plot_chart@1", + ], + "L3_mature": [ + "read_file@1", "write_file@1", "run_python@1", "llm_transform@1", + "web_search@1", "fetch_url@1", "extract_text@1", "summarize@1", + "parse_csv@1", "sql_query@1", "regex_extract@1", "plot_chart@1", + "ocr_pdf@1", "transcribe_audio@1", "translate@1", "embed_text@1", + "vector_search@1", "dedupe_records@1", "join_tables@1", + "statistical_test@1", "fit_model@1", "cross_validate@1", + "git_clone@1", "run_tests@1", "diff_files@1", "render_markdown@1", + "http_post@1", "schedule_job@1", "classify@1", "validate_schema@1", + ], +} + +PROBLEMS = [ + "Produce a literature review of recursive LLM task decomposition in which " + "every citation resolves to a real paper and supports the claim it is " + "attached to.", + "Given a directory of YAML pipeline definitions, compile them to a typed " + "intermediate representation and report which ones fail validation and why.", + "Reproduce figure 3 of a published paper from the authors' released dataset " + "and report where your numbers differ from theirs.", + "Find the median household income of every US county and render a " + "choropleth map of the result.", + "Determine whether a proposed database schema change is safe to deploy " + "against a 400M-row production table.", + "Summarise the last 500 commits of a repository into a changelog grouped by " + "user-visible behaviour change.", + "Decide which of three candidate caching strategies to adopt for a service, " + "with evidence.", + "Extract every numeric claim from a 200-page PDF report and check each one " + "against the underlying spreadsheet.", + "Write and validate a regression test that reproduces an intermittent " + "failure reported only in CI.", + "Translate a technical manual from English to Japanese preserving all code " + "blocks and cross-references verbatim.", + "Estimate the annual carbon footprint of a company's cloud infrastructure " + "from its billing exports.", + "Given a corpus of 50,000 customer support tickets, identify the five " + "product defects responsible for the most support load.", + "Convert a legacy Fortran numerical routine to Python and prove the outputs " + "agree to within floating-point tolerance.", + "Design and run an experiment that determines whether a new ranking model " + "improves user outcomes.", + "Audit a codebase for hardcoded credentials and produce a remediation plan " + "ordered by blast radius.", + "Given a city's public transit GTFS feed, compute how many residents live " + "within a 15-minute walk of frequent service.", +] + + +def measure(planner: LLMPlanner, tiers: dict[str, list[str]], + problems: list[str]) -> list[dict]: + rows = [] + for tier, caps in tiers.items(): + maturities = {c: "trusted" for c in caps} + for i, problem in enumerate(problems): + try: + draft = planner.decompose(problem, "any->report", maturities, 0) + except Exception as exc: + print(f" !! {tier} p{i}: {exc}") + continue + rows.append({ + "model": planner.model, "tier": tier, "n_caps": len(caps), + "problem_index": i, "problem": problem[:90], + "b": draft.fan_out, "ambiguous": draft.n_ambiguous, + "f": round(draft.f, 4), "m": round(draft.fan_out * draft.f, 4), + "tokens_in": draft.tokens_in, "tokens_out": draft.tokens_out, + "rationale": draft.rationale[:160], + }) + print(f" {tier:12s} p{i:<2d} b={draft.fan_out} " + f"amb={draft.n_ambiguous} f={draft.f:.2f} " + f"m={draft.fan_out * draft.f:.2f}") + return rows + + +def report(rows: list[dict]) -> None: + by = defaultdict(list) + for r in rows: + by[(r["model"], r["tier"], r["n_caps"])].append(r) + print(f"\n{'='*84}") + print("MEASURED decomposition statistics (real planner, real problems)") + print(f"{'='*84}") + # NOTE ON THE ESTIMATOR. The Galton-Watson offspring mean is + # m = E[number of ambiguous children], i.e. mean(b*f) -- NOT mean(b) * + # mean(f). The two differ whenever b and f are correlated across problems, + # and here they are: the first version of this report used the product of + # the means and called L2 "critical" when the correct estimator makes it + # subcritical. Both are printed so the difference is visible. + print(f"{'model':<16} {'library':<12} {'#caps':>5} {'mean b':>7} {'mean f':>7} " + f"{'m (correct)':>12} {'mean-b*mean-f':>14} {'P(>=1 amb)':>11} {'regime':>14}") + print("-" * 106) + for (model, tier, n), rs in sorted(by.items(), key=lambda kv: (kv[0][0], kv[0][2])): + b = statistics.mean(r["b"] for r in rs) + f = statistics.mean(r["f"] for r in rs) + m = statistics.mean(r["ambiguous"] for r in rs) + naive = b * f + pm = sum(1 for r in rs if r["ambiguous"] >= 1) / len(rs) + regime = ("SUPERCRITICAL" if m > 1.05 else + "critical" if m > 0.95 else "subcritical") + print(f"{model:<16} {tier:<12} {n:>5} {b:>7.2f} {f:>7.3f} {m:>12.2f} " + f"{naive:>14.2f} {pm:>11.0%} {regime:>14}") + print("-" * 84) + hardest = sorted(rows, key=lambda r: -r["ambiguous"])[:5] + print("\nhighest-m problems observed:") + for r in hardest: + print(f" amb={r['ambiguous']} b={r['b']} f={r['f']:.2f} [{r['tier']}] " + f"{r['problem'][:70]}") + easiest = [r for r in rows if r["ambiguous"] == 0] + print(f"\n{len(easiest)}/{len(rows)} decompositions emitted NO ambiguous step " + f"(m = 0, immediate termination)") + tin = sum(r["tokens_in"] for r in rows) + tout = sum(r["tokens_out"] for r in rows) + print(f"total planner cost: {len(rows)} calls, {tin:,} in + {tout:,} out tokens") + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--refresh", action="store_true") + ap.add_argument("--model", default="gpt-5.4-mini") + ap.add_argument("--crosscheck", default="gpt-5.6-sol") + ap.add_argument("--base-url", default="https://api.openai.com/v1") + args = ap.parse_args() + + os.makedirs(os.path.dirname(CACHE), exist_ok=True) + if os.path.exists(CACHE) and not args.refresh: + rows = json.load(open(CACHE)) + print(f"(cached: {len(rows)} measurements from {CACHE})") + report(rows) + return 0 + + key = load_env_key("OPENAI_API_KEY") + if not key: + print("No OPENAI_API_KEY reachable; cannot measure. " + "This script refuses to invent numbers.") + return 2 + rows: list[dict] = [] + print(f"measuring with {args.model} across {len(TIERS)} library tiers " + f"x {len(PROBLEMS)} problems ...") + rows += measure(LLMPlanner(args.model, key, args.base_url), TIERS, PROBLEMS) + if args.crosscheck: + print(f"\ncross-checking model dependence with {args.crosscheck} " + f"at L2_working ...") + rows += measure(LLMPlanner(args.crosscheck, key, args.base_url), + {"L2_working": TIERS["L2_working"]}, PROBLEMS) + json.dump(rows, open(CACHE, "w"), indent=1) + report(rows) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prototypes/measurements/ambiguity.json b/scripts/prototypes/measurements/ambiguity.json new file mode 100644 index 00000000..81cecf2e --- /dev/null +++ b/scripts/prototypes/measurements/ambiguity.json @@ -0,0 +1,1094 @@ +[ + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 0, + "problem": "Produce a literature review of recursive LLM task decomposition in which every citation re", + "b": 3, + "ambiguous": 3, + "f": 1.0, + "m": 3.0, + "tokens_in": 314, + "tokens_out": 214, + "rationale": "No capabilities are available, so the task must be recursively decomposed into literature identification, synthesis, and citation validation." + }, + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 1, + "problem": "Given a directory of YAML pipeline definitions, compile them to a typed intermediate repre", + "b": 4, + "ambiguous": 4, + "f": 1.0, + "m": 4.0, + "tokens_in": 310, + "tokens_out": 191, + "rationale": "No capabilities are available, so the task must be decomposed into parsing, IR compilation, validation, and reporting subproblems." + }, + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 2, + "problem": "Reproduce figure 3 of a published paper from the authors' released dataset and report wher", + "b": 3, + "ambiguous": 3, + "f": 1.0, + "m": 3.0, + "tokens_in": 310, + "tokens_out": 207, + "rationale": "No capabilities are available, so the task must be decomposed into planning, reproduction/comparison, and reporting subproblems." + }, + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 3, + "problem": "Find the median household income of every US county and render a choropleth map of the res", + "b": 4, + "ambiguous": 4, + "f": 1.0, + "m": 4.0, + "tokens_in": 306, + "tokens_out": 227, + "rationale": "No capabilities are available, so the task must be decomposed into data acquisition, geometry acquisition, joining, and map rendering." + }, + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 4, + "problem": "Determine whether a proposed database schema change is safe to deploy against a 400M-row p", + "b": 1, + "ambiguous": 1, + "f": 1.0, + "m": 1.0, + "tokens_in": 306, + "tokens_out": 107, + "rationale": "No capabilities are available, so the only valid move is to decompose the safety assessment into a lower-level analysis task." + }, + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 5, + "problem": "Summarise the last 500 commits of a repository into a changelog grouped by user-visible be", + "b": 3, + "ambiguous": 3, + "f": 1.0, + "m": 3.0, + "tokens_in": 308, + "tokens_out": 164, + "rationale": "No capabilities are available, so the task must be decomposed into data extraction, change grouping, and report generation." + }, + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 6, + "problem": "Decide which of three candidate caching strategies to adopt for a service, with evidence.", + "b": 1, + "ambiguous": 1, + "f": 1.0, + "m": 1.0, + "tokens_in": 302, + "tokens_out": 70, + "rationale": "No capabilities are available, so the task must be deferred to a recursive decomposition." + }, + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 7, + "problem": "Extract every numeric claim from a 200-page PDF report and check each one against the unde", + "b": 1, + "ambiguous": 1, + "f": 1.0, + "m": 1.0, + "tokens_in": 306, + "tokens_out": 77, + "rationale": "No capabilities are available to perform extraction or verification directly, so the task must be decomposed further." + }, + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 8, + "problem": "Write and validate a regression test that reproduces an intermittent failure reported only", + "b": 5, + "ambiguous": 5, + "f": 1.0, + "m": 5.0, + "tokens_in": 303, + "tokens_out": 274, + "rationale": "No capabilities are available, so the task must be broken into a recursive sequence of decomposition steps leading to the required report." + }, + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 9, + "problem": "Translate a technical manual from English to Japanese preserving all code blocks and cross", + "b": 1, + "ambiguous": 1, + "f": 1.0, + "m": 1.0, + "tokens_in": 305, + "tokens_out": 89, + "rationale": "No available capability can perform translation or preservation handling directly, so the task must be decomposed." + }, + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 10, + "problem": "Estimate the annual carbon footprint of a company's cloud infrastructure from its billing ", + "b": 3, + "ambiguous": 3, + "f": 1.0, + "m": 3.0, + "tokens_in": 301, + "tokens_out": 197, + "rationale": "No capabilities are available, so the task must be decomposed into analysis, estimation, and reporting substeps." + }, + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 11, + "problem": "Given a corpus of 50,000 customer support tickets, identify the five product defects respo", + "b": 1, + "ambiguous": 1, + "f": 1.0, + "m": 1.0, + "tokens_in": 310, + "tokens_out": 79, + "rationale": "No capabilities are available, so the task must be decomposed for further solving." + }, + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 12, + "problem": "Convert a legacy Fortran numerical routine to Python and prove the outputs agree to within", + "b": 3, + "ambiguous": 3, + "f": 1.0, + "m": 3.0, + "tokens_in": 306, + "tokens_out": 192, + "rationale": "No capabilities are available, so the task must be decomposed into translation, validation, and reporting subproblems." + }, + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 13, + "problem": "Design and run an experiment that determines whether a new ranking model improves user out", + "b": 3, + "ambiguous": 3, + "f": 1.0, + "m": 3.0, + "tokens_in": 302, + "tokens_out": 178, + "rationale": "No capabilities are available, so the task must be broken into the minimal recursive subproblems: design, execute, and analyze/report." + }, + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 14, + "problem": "Audit a codebase for hardcoded credentials and produce a remediation plan ordered by blast", + "b": 3, + "ambiguous": 3, + "f": 1.0, + "m": 3.0, + "tokens_in": 304, + "tokens_out": 186, + "rationale": "No capabilities are available, so the task must be decomposed into audit, risk ranking, and remediation planning subproblems." + }, + { + "model": "gpt-5.4-mini", + "tier": "L0_bare", + "n_caps": 0, + "problem_index": 15, + "problem": "Given a city's public transit GTFS feed, compute how many residents live within a 15-minut", + "b": 1, + "ambiguous": 1, + "f": 1.0, + "m": 1.0, + "tokens_in": 310, + "tokens_out": 91, + "rationale": "No capabilities are available, so the task must be decomposed before any computation can be performed." + }, + { + "model": "gpt-5.4-mini", + "tier": "L1_minimal", + "n_caps": 4, + "problem_index": 0, + "problem": "Produce a literature review of recursive LLM task decomposition in which every citation re", + "b": 3, + "ambiguous": 2, + "f": 0.6667, + "m": 2.0, + "tokens_in": 338, + "tokens_out": 183, + "rationale": "No capability can directly perform literature search or verification, so the task must be decomposed into source gathering and synthesis, then written to a repo" + }, + { + "model": "gpt-5.4-mini", + "tier": "L1_minimal", + "n_caps": 4, + "problem_index": 1, + "problem": "Given a directory of YAML pipeline definitions, compile them to a typed intermediate repre", + "b": 2, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 334, + "tokens_out": 164, + "rationale": "The task can be handled in two direct capability calls: first read the YAML inputs, then use Python to compile, validate, and summarize results into the require" + }, + { + "model": "gpt-5.4-mini", + "tier": "L1_minimal", + "n_caps": 4, + "problem_index": 2, + "problem": "Reproduce figure 3 of a published paper from the authors' released dataset and report wher", + "b": 5, + "ambiguous": 5, + "f": 1.0, + "m": 5.0, + "tokens_in": 334, + "tokens_out": 264, + "rationale": "The task requires identifying the target figure, reconstructing the analysis from released materials, and then comparing outputs, none of which can be done dire" + }, + { + "model": "gpt-5.4-mini", + "tier": "L1_minimal", + "n_caps": 4, + "problem_index": 3, + "problem": "Find the median household income of every US county and render a choropleth map of the res", + "b": 2, + "ambiguous": 1, + "f": 0.5, + "m": 1.0, + "tokens_in": 330, + "tokens_out": 148, + "rationale": "The data acquisition step may require choosing and harmonizing external sources, while the actual merge and map rendering can be handled directly in Python once" + }, + { + "model": "gpt-5.4-mini", + "tier": "L1_minimal", + "n_caps": 4, + "problem_index": 4, + "problem": "Determine whether a proposed database schema change is safe to deploy against a 400M-row p", + "b": 1, + "ambiguous": 1, + "f": 1.0, + "m": 1.0, + "tokens_in": 330, + "tokens_out": 95, + "rationale": "No available capability directly evaluates arbitrary schema-change safety, so this must be decomposed into a deeper analysis step." + }, + { + "model": "gpt-5.4-mini", + "tier": "L1_minimal", + "n_caps": 4, + "problem_index": 5, + "problem": "Summarise the last 500 commits of a repository into a changelog grouped by user-visible be", + "b": 3, + "ambiguous": 1, + "f": 0.3333, + "m": 1.0, + "tokens_in": 332, + "tokens_out": 214, + "rationale": "The repository history must first be gathered, then summarized and reorganized into behavior-based changelog groups, which can be handled by the available langu" + }, + { + "model": "gpt-5.4-mini", + "tier": "L1_minimal", + "n_caps": 4, + "problem_index": 6, + "problem": "Decide which of three candidate caching strategies to adopt for a service, with evidence.", + "b": 2, + "ambiguous": 1, + "f": 0.5, + "m": 1.0, + "tokens_in": 326, + "tokens_out": 160, + "rationale": "No direct capability can independently evaluate unknown candidate strategies from evidence, so first the problem must be decomposed to gather comparison notes, " + }, + { + "model": "gpt-5.4-mini", + "tier": "L1_minimal", + "n_caps": 4, + "problem_index": 7, + "problem": "Extract every numeric claim from a 200-page PDF report and check each one against the unde", + "b": 1, + "ambiguous": 1, + "f": 1.0, + "m": 1.0, + "tokens_in": 330, + "tokens_out": 81, + "rationale": "This task requires document understanding, spreadsheet comparison, and aggregation beyond the immediate capabilities, so it must be decomposed further." + }, + { + "model": "gpt-5.4-mini", + "tier": "L1_minimal", + "n_caps": 4, + "problem_index": 8, + "problem": "Write and validate a regression test that reproduces an intermittent failure reported only", + "b": 4, + "ambiguous": 2, + "f": 0.5, + "m": 2.0, + "tokens_in": 327, + "tokens_out": 224, + "rationale": "The problem requires first extracting failure context, then designing a test, and finally writing and validating it, but the available capabilities do not direc" + }, + { + "model": "gpt-5.4-mini", + "tier": "L1_minimal", + "n_caps": 4, + "problem_index": 9, + "problem": "Translate a technical manual from English to Japanese preserving all code blocks and cross", + "b": 1, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 329, + "tokens_out": 92, + "rationale": "The translation can be completed directly by the language transformation capability without further decomposition." + }, + { + "model": "gpt-5.4-mini", + "tier": "L1_minimal", + "n_caps": 4, + "problem_index": 10, + "problem": "Estimate the annual carbon footprint of a company's cloud infrastructure from its billing ", + "b": 3, + "ambiguous": 2, + "f": 0.6667, + "m": 2.0, + "tokens_in": 325, + "tokens_out": 187, + "rationale": "The task requires extracting and calculating from billing exports, which can be decomposed for data processing, while report writing can be handled directly by " + }, + { + "model": "gpt-5.4-mini", + "tier": "L1_minimal", + "n_caps": 4, + "problem_index": 11, + "problem": "Given a corpus of 50,000 customer support tickets, identify the five product defects respo", + "b": 3, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 334, + "tokens_out": 213, + "rationale": "Read the corpus, aggregate/group tickets into defect candidates, then use the LLM to synthesize the top five defects into the required report." + }, + { + "model": "gpt-5.4-mini", + "tier": "L1_minimal", + "n_caps": 4, + "problem_index": 12, + "problem": "Convert a legacy Fortran numerical routine to Python and prove the outputs agree to within", + "b": 3, + "ambiguous": 1, + "f": 0.3333, + "m": 1.0, + "tokens_in": 330, + "tokens_out": 192, + "rationale": "This requires both code translation and empirical verification; translation may need further decomposition, while execution and reporting can be done directly w" + }, + { + "model": "gpt-5.4-mini", + "tier": "L1_minimal", + "n_caps": 4, + "problem_index": 13, + "problem": "Design and run an experiment that determines whether a new ranking model improves user out", + "b": 4, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 326, + "tokens_out": 332, + "rationale": "The task can be handled with direct planning, analysis-code generation, and report synthesis using the available capabilities, without further decomposition." + }, + { + "model": "gpt-5.4-mini", + "tier": "L1_minimal", + "n_caps": 4, + "problem_index": 15, + "problem": "Given a city's public transit GTFS feed, compute how many residents live within a 15-minut", + "b": 4, + "ambiguous": 3, + "f": 0.75, + "m": 3.0, + "tokens_in": 334, + "tokens_out": 266, + "rationale": "The task requires data inspection and geospatial/population computation that are not directly covered by a single capability, so it is best decomposed into anal" + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 0, + "problem": "Produce a literature review of recursive LLM task decomposition in which every citation re", + "b": 4, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 393, + "tokens_out": 250, + "rationale": "A literature review can be produced by searching for relevant papers, extracting their text, and transforming it into a citation-grounded report." + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 1, + "problem": "Given a directory of YAML pipeline definitions, compile them to a typed intermediate repre", + "b": 4, + "ambiguous": 4, + "f": 1.0, + "m": 4.0, + "tokens_in": 389, + "tokens_out": 202, + "rationale": "The task requires directory inspection, YAML-to-IR compilation, validation, and report assembly, but no single available capability covers the full pipeline end" + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 2, + "problem": "Reproduce figure 3 of a published paper from the authors' released dataset and report wher", + "b": 8, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 389, + "tokens_out": 361, + "rationale": "The task can be completed by locating the paper and dataset, extracting Figure 3 details, reproducing the computation, comparing numbers, and summarizing differ" + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 3, + "problem": "Find the median household income of every US county and render a choropleth map of the res", + "b": 4, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 385, + "tokens_out": 208, + "rationale": "Use search to locate a county-level income dataset, load it, then process and plot the county choropleth in Python." + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 4, + "problem": "Determine whether a proposed database schema change is safe to deploy against a 400M-row p", + "b": 1, + "ambiguous": 1, + "f": 1.0, + "m": 1.0, + "tokens_in": 385, + "tokens_out": 107, + "rationale": "No concrete schema-change details or direct database metadata access are provided, so the task must first be decomposed into an assessment subtree rather than e" + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 5, + "problem": "Summarise the last 500 commits of a repository into a changelog grouped by user-visible be", + "b": 2, + "ambiguous": 1, + "f": 0.5, + "m": 1.0, + "tokens_in": 387, + "tokens_out": 161, + "rationale": "We need repository commit data first, then an LLM can transform it into a behavior-oriented changelog report." + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 6, + "problem": "Decide which of three candidate caching strategies to adopt for a service, with evidence.", + "b": 3, + "ambiguous": 3, + "f": 1.0, + "m": 3.0, + "tokens_in": 381, + "tokens_out": 199, + "rationale": "The request lacks the concrete candidates and evidence needed for direct capability use, so the problem must be decomposed into gathering/inferring inputs, comp" + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 7, + "problem": "Extract every numeric claim from a 200-page PDF report and check each one against the unde", + "b": 5, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 385, + "tokens_out": 297, + "rationale": "The task can be completed directly by extracting report text, structuring numeric claims, loading and parsing the spreadsheet, then programmatically checking ea" + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 8, + "problem": "Write and validate a regression test that reproduces an intermittent failure reported only", + "b": 3, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 382, + "tokens_out": 201, + "rationale": "The task is to author a regression test/report, which can be planned from the prompt and materialized directly with available text transformation and file-writi" + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 9, + "problem": "Translate a technical manual from English to Japanese preserving all code blocks and cross", + "b": 2, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 384, + "tokens_out": 154, + "rationale": "The task can be handled directly by extracting the source text if needed and then applying a translation transform with explicit preservation constraints." + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 10, + "problem": "Estimate the annual carbon footprint of a company's cloud infrastructure from its billing ", + "b": 2, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 380, + "tokens_out": 155, + "rationale": "The task is a direct analytical transformation into a report, and the available capabilities are sufficient to extract the prompt text and synthesize the estima" + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 11, + "problem": "Given a corpus of 50,000 customer support tickets, identify the five product defects respo", + "b": 3, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 389, + "tokens_out": 204, + "rationale": "The task can be completed directly by extracting the ticket text, using an LLM to group and rank defect themes by support load, and then summarizing the result " + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 12, + "problem": "Convert a legacy Fortran numerical routine to Python and prove the outputs agree to within", + "b": 4, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 385, + "tokens_out": 228, + "rationale": "The task can be completed directly with file reading, translation, execution-based validation, and report generation without further decomposition." + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 13, + "problem": "Design and run an experiment that determines whether a new ranking model improves user out", + "b": 2, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 381, + "tokens_out": 178, + "rationale": "The task can be handled directly by transforming the prompt into an experiment design and then into a concise report, without needing deeper decomposition." + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 14, + "problem": "Audit a codebase for hardcoded credentials and produce a remediation plan ordered by blast", + "b": 3, + "ambiguous": 1, + "f": 0.3333, + "m": 1.0, + "tokens_in": 383, + "tokens_out": 259, + "rationale": "First inspect the repository contents, then extract likely credential patterns, then transform the findings into an ordered remediation report." + }, + { + "model": "gpt-5.4-mini", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 15, + "problem": "Given a city's public transit GTFS feed, compute how many residents live within a 15-minut", + "b": 4, + "ambiguous": 4, + "f": 1.0, + "m": 4.0, + "tokens_in": 389, + "tokens_out": 224, + "rationale": "Computing residents within a 15-minute walk of frequent transit requires feeding GTFS and population inputs through spatial analysis, which cannot be completed " + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 0, + "problem": "Produce a literature review of recursive LLM task decomposition in which every citation re", + "b": 4, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 521, + "tokens_out": 249, + "rationale": "The task can be completed with search, verification, extraction, and summarization capabilities without further decomposition." + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 1, + "problem": "Given a directory of YAML pipeline definitions, compile them to a typed intermediate repre", + "b": 5, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 517, + "tokens_out": 285, + "rationale": "The task can be handled directly by reading the YAML files, transforming them into a typed IR, validating against a schema, and formatting the failures into a r" + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 2, + "problem": "Reproduce figure 3 of a published paper from the authors' released dataset and report wher", + "b": 5, + "ambiguous": 5, + "f": 1.0, + "m": 5.0, + "tokens_in": 517, + "tokens_out": 271, + "rationale": "This task requires paper/dataset discovery, data preparation, reproduction, comparison, and reporting, none of which can be completed directly with a single ava" + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 3, + "problem": "Find the median household income of every US county and render a choropleth map of the res", + "b": 6, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 513, + "tokens_out": 325, + "rationale": "This can be completed directly by finding a suitable county income dataset and geography source, joining them on county FIPS, plotting a choropleth, and packagi" + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 4, + "problem": "Determine whether a proposed database schema change is safe to deploy against a 400M-row p", + "b": 1, + "ambiguous": 1, + "f": 1.0, + "m": 1.0, + "tokens_in": 513, + "tokens_out": 95, + "rationale": "The problem is underspecified and requires a risk-assessment workflow rather than a direct capability call, so it must be decomposed first." + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 5, + "problem": "Summarise the last 500 commits of a repository into a changelog grouped by user-visible be", + "b": 3, + "ambiguous": 3, + "f": 1.0, + "m": 3.0, + "tokens_in": 515, + "tokens_out": 183, + "rationale": "This task requires repository inspection and semantic summarization into grouped user-facing changes, which cannot be completed directly with the listed capabil" + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 6, + "problem": "Decide which of three candidate caching strategies to adopt for a service, with evidence.", + "b": 3, + "ambiguous": 3, + "f": 1.0, + "m": 3.0, + "tokens_in": 509, + "tokens_out": 186, + "rationale": "No single capability reliably performs an evidence-based strategy decision end-to-end, so the task should be decomposed into evidence gathering, comparison, and" + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 7, + "problem": "Extract every numeric claim from a 200-page PDF report and check each one against the unde", + "b": 4, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 513, + "tokens_out": 208, + "rationale": "Extract the PDF text, identify numeric claims, load the spreadsheet, and cross-check each extracted claim against the source data to produce the required report" + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 8, + "problem": "Write and validate a regression test that reproduces an intermittent failure reported only", + "b": 7, + "ambiguous": 1, + "f": 0.1429, + "m": 1.0, + "tokens_in": 510, + "tokens_out": 441, + "rationale": "Use repository inspection and repeated test execution to isolate the flaky CI failure, then record and validate the resulting regression artifact." + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 9, + "problem": "Translate a technical manual from English to Japanese preserving all code blocks and cross", + "b": 1, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 512, + "tokens_out": 95, + "rationale": "A direct translation capability is available, and the preservation constraints can be passed through as explicit instructions without further decomposition." + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 10, + "problem": "Estimate the annual carbon footprint of a company's cloud infrastructure from its billing ", + "b": 4, + "ambiguous": 4, + "f": 1.0, + "m": 4.0, + "tokens_in": 508, + "tokens_out": 232, + "rationale": "This requires data normalization, factor mapping, aggregation, and presentation, which are best handled as a short recursive pipeline rather than a single direc" + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 11, + "problem": "Given a corpus of 50,000 customer support tickets, identify the five product defects respo", + "b": 5, + "ambiguous": 5, + "f": 1.0, + "m": 5.0, + "tokens_in": 517, + "tokens_out": 288, + "rationale": "The task requires corpus preparation, clustering, aggregation, and reporting, and no single available capability can directly identify the top five defects end-" + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 12, + "problem": "Convert a legacy Fortran numerical routine to Python and prove the outputs agree to within", + "b": 6, + "ambiguous": 1, + "f": 0.1667, + "m": 1.0, + "tokens_in": 513, + "tokens_out": 338, + "rationale": "The task requires source ingestion, code translation, executable comparison, tolerance checking, and final reporting, all of which can be handled with the avail" + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 13, + "problem": "Design and run an experiment that determines whether a new ranking model improves user out", + "b": 5, + "ambiguous": 3, + "f": 0.6, + "m": 3.0, + "tokens_in": 509, + "tokens_out": 287, + "rationale": "The task requires designing, executing, analyzing, and reporting on an experiment; available capabilities can handle statistical testing and charting, but the o" + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 14, + "problem": "Audit a codebase for hardcoded credentials and produce a remediation plan ordered by blast", + "b": 6, + "ambiguous": 1, + "f": 0.1667, + "m": 1.0, + "tokens_in": 511, + "tokens_out": 489, + "rationale": "The available capabilities support scanning files, extracting secret-like patterns, deduplicating findings, classifying them by blast radius, and turning the re" + }, + { + "model": "gpt-5.4-mini", + "tier": "L3_mature", + "n_caps": 30, + "problem_index": 15, + "problem": "Given a city's public transit GTFS feed, compute how many residents live within a 15-minut", + "b": 4, + "ambiguous": 3, + "f": 0.75, + "m": 3.0, + "tokens_in": 517, + "tokens_out": 223, + "rationale": "This requires data acquisition, GTFS accessibility analysis, spatial population overlay, and then report synthesis, which cannot be completed directly by a sing" + }, + { + "model": "gpt-5.6-sol", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 0, + "problem": "Produce a literature review of recursive LLM task decomposition in which every citation re", + "b": 6, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 393, + "tokens_out": 911, + "rationale": "The pipeline discovers candidate scholarship, verifies papers at stable sources, extracts claim-level evidence, drafts a grounded review, and audits every citat" + }, + { + "model": "gpt-5.6-sol", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 1, + "problem": "Given a directory of YAML pipeline definitions, compile them to a typed intermediate repre", + "b": 1, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 389, + "tokens_out": 410, + "rationale": "A single Python execution can traverse the directory, parse YAML, compile and validate each pipeline, and directly produce the requested structured report." + }, + { + "model": "gpt-5.6-sol", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 2, + "problem": "Reproduce figure 3 of a published paper from the authors' released dataset and report wher", + "b": 7, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 389, + "tokens_out": 1117, + "rationale": "The pipeline identifies authoritative inputs, extracts the published figure specification, recomputes it from the released data, renders the reproduction, and r" + }, + { + "model": "gpt-5.6-sol", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 4, + "problem": "Determine whether a proposed database schema change is safe to deploy against a 400M-row p", + "b": 5, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 385, + "tokens_out": 879, + "rationale": "The pipeline normalizes the proposal, derives and runs only read-only production diagnostics, checks engine-specific official guidance, and synthesizes an evide" + }, + { + "model": "gpt-5.6-sol", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 5, + "problem": "Summarise the last 500 commits of a repository into a changelog grouped by user-visible be", + "b": 3, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 387, + "tokens_out": 488, + "rationale": "Extract structured evidence from the latest 500 commits, semantically consolidate it into user-visible behavioural changes, then render the result as a readable" + }, + { + "model": "gpt-5.6-sol", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 6, + "problem": "Decide which of three candidate caching strategies to adopt for a service, with evidence.", + "b": 4, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 381, + "tokens_out": 907, + "rationale": "The pipeline structures arbitrary input, supplements evidence gaps, performs a reproducible quantitative comparison, and synthesizes a cited, risk-aware adoptio" + }, + { + "model": "gpt-5.6-sol", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 7, + "problem": "Extract every numeric claim from a 200-page PDF report and check each one against the unde", + "b": 6, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 385, + "tokens_out": 862, + "rationale": "The pipeline preserves PDF provenance, exhaustively identifies and normalizes numeric claims, structures the complete spreadsheet, programmatically verifies eac" + }, + { + "model": "gpt-5.6-sol", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 8, + "problem": "Write and validate a regression test that reproduces an intermittent failure reported only", + "b": 7, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 382, + "tokens_out": 1047, + "rationale": "Draft the CI-focused regression test, execute it under stress, revise it from observed evidence, rerun validation, and report only substantiated results." + }, + { + "model": "gpt-5.6-sol", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 9, + "problem": "Translate a technical manual from English to Japanese preserving all code blocks and cross", + "b": 3, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 384, + "tokens_out": 623, + "rationale": "The pipeline extracts the manual with formatting intact, performs a constraint-aware translation, then audits and corrects protected content before returning th" + }, + { + "model": "gpt-5.6-sol", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 10, + "problem": "Estimate the annual carbon footprint of a company's cloud infrastructure from its billing ", + "b": 5, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 380, + "tokens_out": 1072, + "rationale": "The pipeline normalizes arbitrary billing data, obtains and documents suitable emissions factors, performs reproducible annualized calculations with uncertainty" + }, + { + "model": "gpt-5.6-sol", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 11, + "problem": "Given a corpus of 50,000 customer support tickets, identify the five product defects respo", + "b": 6, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 389, + "tokens_out": 1094, + "rationale": "The pipeline ingests and cleans the corpus, identifies and normalizes genuine defect reports, quantitatively ranks their support burden, and presents the five l" + }, + { + "model": "gpt-5.6-sol", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 12, + "problem": "Convert a legacy Fortran numerical routine to Python and prove the outputs agree to within", + "b": 4, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 385, + "tokens_out": 830, + "rationale": "The pipeline extracts the legacy source, creates a semantics-preserving translation and differential harness, executes both implementations for quantitative ver" + }, + { + "model": "gpt-5.6-sol", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 13, + "problem": "Design and run an experiment that determines whether a new ranking model improves user out", + "b": 5, + "ambiguous": 1, + "f": 0.2, + "m": 1.0, + "tokens_in": 381, + "tokens_out": 901, + "rationale": "The pipeline preregisters a causal test, delegates the operational launch and data collection that available analytical tools cannot perform, then analyzes and " + }, + { + "model": "gpt-5.6-sol", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 14, + "problem": "Audit a codebase for hardcoded credentials and produce a remediation plan ordered by blast", + "b": 3, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 383, + "tokens_out": 705, + "rationale": "A recursive scanner inventories redacted candidates, an analysis pass ranks deduplicated findings by blast radius, and a final transformation turns the prioriti" + }, + { + "model": "gpt-5.6-sol", + "tier": "L2_working", + "n_caps": 12, + "problem_index": 15, + "problem": "Given a city's public transit GTFS feed, compute how many residents live within a 15-minut", + "b": 7, + "ambiguous": 0, + "f": 0.0, + "m": 0.0, + "tokens_in": 389, + "tokens_out": 1444, + "rationale": "The pipeline validates and analyzes scheduled frequency, obtains suitable population data, computes non-overlapping pedestrian-network catchments, spatially all" + } +] \ No newline at end of file diff --git a/scripts/prototypes/measurements/optimism.json b/scripts/prototypes/measurements/optimism.json new file mode 100644 index 00000000..b145dca6 --- /dev/null +++ b/scripts/prototypes/measurements/optimism.json @@ -0,0 +1,2020 @@ +[ + { + "planner": "gpt-5.4-mini", + "judge": "gpt-5.5", + "problem_index": 1, + "problem": "Given a directory of YAML pipeline definitions, compile them to a typed intermed", + "b": 3, + "declared_ambiguous": 3, + "atomic_steps": 0, + "overclaimed": 0, + "declared_f": 1.0, + "corrected_f": 1.0, + "corrected_ambiguous": 3, + "plan": { + "id": "plan-34680141", + "problem": "Given a directory of YAML pipeline definitions, compile them to a typed intermediate representation and report which ones fail validation and why.", + "steps": [ + { + "id": "s1", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "Read and enumerate the YAML pipeline definition files from the given directory/input so they can be processed one by one", + "inputs": [], + "outputs": [ + "files" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "list", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s2", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "For each YAML file, parse/compile it into the typed intermediate representation and validate it against the schema or typing rules", + "inputs": [], + "outputs": [ + "ir_results" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "list", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s3", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "Aggregate validation failures across all files and produce a report explaining which pipelines failed and why", + "inputs": [], + "outputs": [ + "report" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [] + }, + { + "planner": "gpt-5.4-mini", + "judge": "gpt-5.5", + "problem_index": 2, + "problem": "Reproduce figure 3 of a published paper from the authors' released dataset and r", + "b": 5, + "declared_ambiguous": 5, + "atomic_steps": 0, + "overclaimed": 0, + "declared_f": 1.0, + "corrected_f": 1.0, + "corrected_ambiguous": 5, + "plan": { + "id": "plan-91001540", + "problem": "Reproduce figure 3 of a published paper from the authors' released dataset and report where your numbers differ from theirs.", + "steps": [ + { + "id": "s1", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "Identify the target paper, locate the authors' released dataset and figure 3 details, and determine what data/processing is needed to reproduce the figure.", + "inputs": [], + "outputs": [ + "paper_info", + "dataset_info", + "figure_spec" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s2", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "Load and clean the released dataset, then compute the quantities needed for figure 3 reproduction using the paper's described methodology.", + "inputs": [], + "outputs": [ + "reproduction_data" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s3", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "Generate a reproduction of figure 3 from the computed data, matching the paper's visual conventions as closely as possible.", + "inputs": [], + "outputs": [ + "reproduced_figure" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s4", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "Compare reproduced numbers against the paper's reported numbers and identify any differences with likely causes.", + "inputs": [], + "outputs": [ + "difference_report" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s5", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "Compile a final report summarizing the reproduction process, the reproduced figure, and all numerical discrepancies versus the paper.", + "inputs": [], + "outputs": [ + "report" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [] + }, + { + "planner": "gpt-5.4-mini", + "judge": "gpt-5.5", + "problem_index": 4, + "problem": "Determine whether a proposed database schema change is safe to deploy against a ", + "b": 3, + "declared_ambiguous": 3, + "atomic_steps": 0, + "overclaimed": 0, + "declared_f": 1.0, + "corrected_f": 1.0, + "corrected_ambiguous": 3, + "plan": { + "id": "plan-36176905", + "problem": "Determine whether a proposed database schema change is safe to deploy against a 400M-row production table.", + "steps": [ + { + "id": "s1", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "Inspect the proposed schema change and any supporting context to identify the exact DDL, table size, indexes, constraints, and deployment intent needed to judge safety.", + "inputs": [], + "outputs": [ + "change_context" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s2", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "Assess operational and data-migration risk of the schema change for a 400M-row table, including lock time, rewrite cost, backfill behavior, compatibility, and rollback feasibility.", + "inputs": [], + "outputs": [ + "risk_assessment" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s3", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "Conclude whether the change is safe to deploy and produce a concise report with recommendation, blockers, and any required mitigations.", + "inputs": [], + "outputs": [ + "report" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [] + }, + { + "planner": "gpt-5.4-mini", + "judge": "gpt-5.5", + "problem_index": 5, + "problem": "Summarise the last 500 commits of a repository into a changelog grouped by user-", + "b": 3, + "declared_ambiguous": 1, + "atomic_steps": 2, + "overclaimed": 1, + "declared_f": 0.333, + "corrected_f": 0.667, + "corrected_ambiguous": 2, + "plan": { + "id": "plan-64550883", + "problem": "Summarise the last 500 commits of a repository into a changelog grouped by user-visible behaviour change.", + "steps": [ + { + "id": "s1", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "Obtain the last 500 commits from the repository with sufficient detail (e.g., messages, timestamps, diffs if available) to support changelog grouping by user-visible behavior change.", + "inputs": [], + "outputs": [ + "commit_data" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "structured commit list", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s2", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": "commit_data", + "task": "Cluster commits into user-visible behavior change groups, infer release-note style summaries, and produce concise changelog entries with representative commit references." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "grouped changelog entries", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s3", + "kind": "capability", + "ref": "summarize@1", + "args": { + "input": "grouped changelog entries", + "task": "Condense into a readable changelog organized by behavior change, preserving important impacts and avoiding internal implementation details." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [ + { + "id": "s2", + "atomic": false, + "why": "Requires clustering, impact inference, and release-note writing over many commits." + }, + { + "id": "s3", + "atomic": true, + "why": "Summarize can condense provided grouped entries into a readable changelog." + } + ] + }, + { + "planner": "gpt-5.4-mini", + "judge": "gpt-5.5", + "problem_index": 6, + "problem": "Decide which of three candidate caching strategies to adopt for a service, with ", + "b": 3, + "declared_ambiguous": 3, + "atomic_steps": 0, + "overclaimed": 0, + "declared_f": 1.0, + "corrected_f": 1.0, + "corrected_ambiguous": 3, + "plan": { + "id": "plan-77289456", + "problem": "Decide which of three candidate caching strategies to adopt for a service, with evidence.", + "steps": [ + { + "id": "s1", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "Identify the three candidate caching strategies and gather their evidence from the provided inputs or attached materials.", + "inputs": [], + "outputs": [ + "candidates_and_evidence" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s2", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "Compare the three candidate strategies on relevant criteria such as latency impact, hit rate, operational complexity, correctness risk, and cost, using the gathered evidence.", + "inputs": [], + "outputs": [ + "comparison" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s3", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "Decide which caching strategy to adopt based on the comparison and produce a concise evidence-backed recommendation report.", + "inputs": [], + "outputs": [ + "report" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [] + }, + { + "planner": "gpt-5.4-mini", + "judge": "gpt-5.5", + "problem_index": 10, + "problem": "Estimate the annual carbon footprint of a company's cloud infrastructure from it", + "b": 2, + "declared_ambiguous": 0, + "atomic_steps": 2, + "overclaimed": 1, + "declared_f": 0.0, + "corrected_f": 0.5, + "corrected_ambiguous": 1, + "plan": { + "id": "plan-97048116", + "problem": "Estimate the annual carbon footprint of a company's cloud infrastructure from its billing exports.", + "steps": [ + { + "id": "s1", + "kind": "capability", + "ref": "extract_text@1", + "args": { + "input": "PROBLEM: Estimate the annual carbon footprint of a company's cloud infrastructure from its billing exports." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "text", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s2", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input_from": "s1", + "instruction": "Produce a concise report estimating annual carbon footprint from cloud billing exports. Include assumptions, method, any needed conversion factors or emission factors, and a clear final estimate if possible." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [ + { + "id": "s1", + "atomic": true, + "why": "Input already contains extractable text; output is just text." + }, + { + "id": "s2", + "atomic": false, + "why": "Requires billing export data and calculations not available from problem statement alone." + } + ] + }, + { + "planner": "gpt-5.4-mini", + "judge": "gpt-5.5", + "problem_index": 11, + "problem": "Given a corpus of 50,000 customer support tickets, identify the five product def", + "b": 3, + "declared_ambiguous": 0, + "atomic_steps": 3, + "overclaimed": 2, + "declared_f": 0.0, + "corrected_f": 0.667, + "corrected_ambiguous": 2, + "plan": { + "id": "plan-38236506", + "problem": "Given a corpus of 50,000 customer support tickets, identify the five product defects responsible for the most support load.", + "steps": [ + { + "id": "s1", + "kind": "capability", + "ref": "extract_text@1", + "args": { + "input": "corpus of 50,000 customer support tickets" + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "text", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s2", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": "text", + "instruction": "Identify mentions of product defects/issues, cluster equivalent defects, estimate support load by defect from ticket frequency or explicit effort indicators if present, and produce a ranked list of the top five defects with brief evidence." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "structured_summary", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s3", + "kind": "capability", + "ref": "summarize@1", + "args": { + "input": "structured_summary", + "instruction": "Format as a concise report listing the five product defects responsible for the most support load, including ranking and evidence basis." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [ + { + "id": "s1", + "atomic": false, + "why": "Corpus access/format unspecified; may require reading/parsing many records first." + }, + { + "id": "s2", + "atomic": false, + "why": "Requires extraction, clustering, counting, and judgment over 50,000 tickets." + }, + { + "id": "s3", + "atomic": true, + "why": "Can format an existing structured summary into a concise ranked report." + } + ] + }, + { + "planner": "gpt-5.4-mini", + "judge": "gpt-5.5", + "problem_index": 13, + "problem": "Design and run an experiment that determines whether a new ranking model improve", + "b": 2, + "declared_ambiguous": 0, + "atomic_steps": 2, + "overclaimed": 0, + "declared_f": 0.0, + "corrected_f": 0.0, + "corrected_ambiguous": 0, + "plan": { + "id": "plan-89200844", + "problem": "Design and run an experiment that determines whether a new ranking model improves user outcomes.", + "steps": [ + { + "id": "s1", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": "Design an experiment to determine whether a new ranking model improves user outcomes. Produce a concise experimental plan including hypothesis, primary metrics, assignment strategy, sample size/power considerations, duration, guardrails, analysis plan, and decision criteria.", + "instruction": "Create a practical A/B experiment design and analysis plan for evaluating a ranking model change.", + "output_format": "report" + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s2", + "kind": "capability", + "ref": "summarize@1", + "args": { + "input": "report", + "instruction": "Condense the experiment design into a final actionable report with clear sections and next steps.", + "output_format": "report" + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [ + { + "id": "s1", + "atomic": true, + "why": "LLM can draft a concise experimental plan from the prompt." + }, + { + "id": "s2", + "atomic": true, + "why": "Summarizer can condense an existing report into structured actionable sections." + } + ] + }, + { + "planner": "gpt-5.6-sol", + "judge": "gpt-5.5", + "problem_index": 1, + "problem": "Given a directory of YAML pipeline definitions, compile them to a typed intermed", + "b": 1, + "declared_ambiguous": 0, + "atomic_steps": 1, + "overclaimed": 1, + "declared_f": 0.0, + "corrected_f": 1.0, + "corrected_ambiguous": 1, + "plan": { + "id": "plan-18185610", + "problem": "Given a directory of YAML pipeline definitions, compile them to a typed intermediate representation and report which ones fail validation and why.", + "steps": [ + { + "id": "s1", + "kind": "capability", + "ref": "run_python@1", + "args": { + "input": "", + "task": "Recursively find .yaml and .yml files; safely parse each document; compile valid definitions into the pipeline typed intermediate representation using the supplied schema/type rules; validate required fields, field types, enum values, references, dependency cycles, and other schema constraints; isolate errors per file and return a deterministic report containing each file's status, compiled IR when valid, and structured validation failures with document paths and reasons when invalid." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [ + { + "id": "s1", + "atomic": false, + "why": "Requires schema-specific validation and compiler logic not provided; open-ended work." + } + ] + }, + { + "planner": "gpt-5.6-sol", + "judge": "gpt-5.5", + "problem_index": 2, + "problem": "Reproduce figure 3 of a published paper from the authors' released dataset and r", + "b": 7, + "declared_ambiguous": 0, + "atomic_steps": 7, + "overclaimed": 2, + "declared_f": 0.0, + "corrected_f": 0.286, + "corrected_ambiguous": 2, + "plan": { + "id": "plan-40379116", + "problem": "Reproduce figure 3 of a published paper from the authors' released dataset and report where your numbers differ from theirs.", + "steps": [ + { + "id": "s1", + "kind": "capability", + "ref": "web_search@1", + "args": { + "query": "Using the supplied paper identification/context {{input}}, locate the published paper, its Figure 3, and the authors' official released dataset or code repository." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "search_results", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s2", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": "{{input}}\n{{s1}}", + "instruction": "Resolve the canonical paper URL and official dataset/code-release URL. Record Figure 3's caption, panel definitions, metrics, filters, aggregation rules, and reported or graphically displayed values. Return structured JSON with URLs and reproduction requirements." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s3", + "kind": "capability", + "ref": "fetch_url@1", + "args": { + "url": "{{s2.paper_url}}" + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "binary", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s4", + "kind": "capability", + "ref": "extract_text@1", + "args": { + "input": "{{s3}}", + "instruction": "Extract the full text, emphasizing Figure 3, its caption, surrounding methodology, axis labels, legends, numerical annotations, and uncertainty definitions." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "text", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s5", + "kind": "capability", + "ref": "fetch_url@1", + "args": { + "url": "{{s2.dataset_url}}" + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "binary", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s6", + "kind": "capability", + "ref": "run_python@1", + "args": { + "inputs": { + "dataset": "{{s5}}", + "paper": "{{s3}}", + "paper_text": "{{s4}}", + "figure_spec": "{{s2}}" + }, + "task": "Inspect and unpack the authors' release; identify the raw data and any relevant scripts; independently implement the Figure 3 transformations, filters, groupings, statistics, and uncertainty calculations; extract the authors' plotted numerical values where needed from the paper figure; generate a reproduction of every Figure 3 panel; and return computed values, author values, absolute and relative differences, tolerances, provenance, warnings, and paths to generated figure/data artifacts. Do not silently substitute assumptions\u2014document them." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s7", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": "Paper metadata/specification: {{s2}}\nPaper text: {{s4}}\nReproduction results: {{s6}}", + "instruction": "Write a self-contained reproducibility report identifying the paper and dataset version, explaining the exact method, linking the reproduced Figure 3 artifact, tabulating reproduced versus published values panel-by-panel with absolute and relative differences, distinguishing rounding/digitization discrepancies from substantive mismatches, and documenting assumptions, software details, and limitations." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [ + { + "id": "s1", + "atomic": true, + "why": "A web search can return relevant search results for the query." + }, + { + "id": "s2", + "atomic": false, + "why": "Requires resolving and extracting details not necessarily present in search results." + }, + { + "id": "s3", + "atomic": true, + "why": "Fetching a URL as binary is a single capability call." + }, + { + "id": "s4", + "atomic": true, + "why": "Text extraction from the fetched paper is the named capability\u2019s purpose." + }, + { + "id": "s5", + "atomic": true, + "why": "Fetching the dataset URL as binary is a single capability call." + }, + { + "id": "s6", + "atomic": false, + "why": "Requires unpacking, interpretation, analysis, and reproduction; open-ended workflow." + }, + { + "id": "s7", + "atomic": true, + "why": "Given all inputs and results, transforming them into a report is plausible." + } + ] + }, + { + "planner": "gpt-5.6-sol", + "judge": "gpt-5.5", + "problem_index": 4, + "problem": "Determine whether a proposed database schema change is safe to deploy against a ", + "b": 5, + "declared_ambiguous": 0, + "atomic_steps": 5, + "overclaimed": 1, + "declared_f": 0.0, + "corrected_f": 0.2, + "corrected_ambiguous": 1, + "plan": { + "id": "plan-41994055", + "problem": "Determine whether a proposed database schema change is safe to deploy against a 400M-row production table.", + "steps": [ + { + "id": "s1", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": "$input", + "instruction": "Normalize the proposed schema change and all supplied context into structured facts: database engine/version, exact DDL, table structure, row/byte size, indexes and constraints, traffic/SLA, replication, deployment method, rollback plan, and database access details. Explicitly list missing facts that prevent a definitive safety determination." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s2", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": "$s1", + "instruction": "Generate engine-specific, strictly read-only diagnostic SQL needed to assess rewrite/scan behavior, lock duration, disk headroom, transaction pressure, replication lag, conflicting sessions, and relevant table/index metadata. Do not emit or execute DDL, EXPLAIN ANALYZE, locks, or writes; if connection details are unavailable, return the queries as unexecuted diagnostics." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s3", + "kind": "capability", + "ref": "sql_query@1", + "args": { + "connection": "$s1.database_access", + "queries": "$s2.read_only_queries", + "read_only": true + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s4", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": { + "proposal": "$s1", + "diagnostic_plan": "$s2", + "diagnostic_results": "$s3" + }, + "instruction": "Assess deployment safety for a 400M-row production table. Classify the change as SAFE, SAFE_WITH_MITIGATIONS, UNSAFE, or INDETERMINATE; analyze table rewrite/scan, metadata and row locks, runtime, disk/WAL/binlog growth, replication impact, failure and rollback behavior, and version-specific online-DDL semantics. Never infer absent evidence; distinguish measured findings from assumptions and propose a safer migration sequence where needed." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s5", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": "$s4", + "instruction": "Produce the final deployment-safety report with executive verdict, confidence, evidence, risks, blocking unknowns, required mitigations, preflight checks, staged rollout and monitoring thresholds, abort criteria, rollback procedure, and an explicit go/no-go recommendation. State that no definitive approval is possible if critical production evidence is missing." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [ + { + "id": "s1", + "atomic": true, + "why": "Single LLM transformation can extract and normalize supplied context into structured facts." + }, + { + "id": "s2", + "atomic": true, + "why": "Single LLM call can draft read-only diagnostic SQL from structured schema context." + }, + { + "id": "s3", + "atomic": true, + "why": "sql_query can execute the provided read-only query list in one call." + }, + { + "id": "s4", + "atomic": false, + "why": "Deployment safety assessment requires expert judgment beyond a simple transform capability." + }, + { + "id": "s5", + "atomic": true, + "why": "Single LLM transformation can format an existing assessment into a final report." + } + ] + }, + { + "planner": "gpt-5.6-sol", + "judge": "gpt-5.5", + "problem_index": 5, + "problem": "Summarise the last 500 commits of a repository into a changelog grouped by user-", + "b": 3, + "declared_ambiguous": 0, + "atomic_steps": 3, + "overclaimed": 1, + "declared_f": 0.0, + "corrected_f": 0.333, + "corrected_ambiguous": 1, + "plan": { + "id": "plan-7865018", + "problem": "Summarise the last 500 commits of a repository into a changelog grouped by user-visible behaviour change.", + "steps": [ + { + "id": "s1", + "kind": "capability", + "ref": "run_python@1", + "args": { + "input": "repository", + "task": "Inspect the supplied local repository with git and extract the last 500 commits in chronological order, including commit hash, date, author, subject, body, changed paths, and diff statistics; retain enough patch context to identify user-visible behavior while truncating generated files and oversized diffs." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "array", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s2", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input_from": "s1", + "instruction": "Analyze the commits as a history rather than independently. Identify user-visible behavior changes, combine related commits (including fixes, follow-ups, and reverts) into coherent change items, distinguish additions/changes/fixes/removals, omit purely internal work unless it affects users, and preserve representative commit hashes for traceability." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "array", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s3", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input_from": "s2", + "instruction": "Produce a concise changelog report grouped by user-visible behavior change, organized into clear sections such as Added, Changed, Fixed, and Removed where applicable. Explain the resulting behavior in user-facing language, mention affected components, include representative short commit hashes, and add a brief note about excluded internal-only commits." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [ + { + "id": "s1", + "atomic": true, + "why": "A script can query git and return structured last-500 commit records." + }, + { + "id": "s2", + "atomic": false, + "why": "Requires broad judgment and clustering many commits into behavior changes." + }, + { + "id": "s3", + "atomic": true, + "why": "Can format grouped changes into a concise changelog report." + } + ] + }, + { + "planner": "gpt-5.6-sol", + "judge": "gpt-5.5", + "problem_index": 6, + "problem": "Decide which of three candidate caching strategies to adopt for a service, with ", + "b": 4, + "declared_ambiguous": 0, + "atomic_steps": 4, + "overclaimed": 1, + "declared_f": 0.0, + "corrected_f": 0.25, + "corrected_ambiguous": 1, + "plan": { + "id": "plan-881504", + "problem": "Decide which of three candidate caching strategies to adopt for a service, with evidence.", + "steps": [ + { + "id": "s1", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": "$input", + "instruction": "Extract the three caching strategies, service requirements, constraints, and all available quantitative or qualitative evidence. Normalize comparable metrics such as latency, hit rate, throughput, cost, consistency, availability, operational complexity, and implementation risk. Explicitly mark missing data and source each claim." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object containing candidates, criteria, constraints, normalized evidence, sources, and missing_data", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s2", + "kind": "capability", + "ref": "run_python@1", + "args": { + "input": "$s1", + "task": "Compare the three candidates using constraint filtering, normalized criterion scores, and transparent weights inferred from the stated service priorities. Calculate totals, rankings, sensitivity analysis across plausible weight changes, and evidence-confidence scores; do not invent missing measurements." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "object containing methodology, candidate scorecards, ranking, sensitivity_results, confidence, and evidence_gaps", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s3", + "kind": "capability", + "ref": "plot_chart@1", + "args": { + "data": "$s2.candidate_scorecards", + "chart_type": "grouped bar and weighted-score comparison", + "title": "Caching Strategy Evidence and Decision Scores" + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "chart artifact", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s4", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": { + "extracted_evidence": "$s1", + "analysis": "$s2", + "chart": "$s3" + }, + "instruction": "Write a decision report recommending one of the three caching strategies. Include an executive summary, requirements, evidence table with sources, comparison methodology and weights, trade-offs, sensitivity analysis, uncertainty and missing evidence, implementation risks, and a validation or rollback plan. Clearly distinguish measured facts from assumptions and explain why the recommendation wins." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [ + { + "id": "s1", + "atomic": true, + "why": "LLM can extract and structure evidence from provided input in one pass." + }, + { + "id": "s2", + "atomic": false, + "why": "Requires judgment to infer weights and score criteria beyond Python computation." + }, + { + "id": "s3", + "atomic": true, + "why": "Plotting supplied scorecard data as requested is a single charting operation." + }, + { + "id": "s4", + "atomic": true, + "why": "LLM can draft report from supplied extraction, analysis, and chart." + } + ] + }, + { + "planner": "gpt-5.6-sol", + "judge": "gpt-5.5", + "problem_index": 10, + "problem": "Estimate the annual carbon footprint of a company's cloud infrastructure from it", + "b": 6, + "declared_ambiguous": 0, + "atomic_steps": 6, + "overclaimed": 3, + "declared_f": 0.0, + "corrected_f": 0.5, + "corrected_ambiguous": 3, + "plan": { + "id": "plan-53733981", + "problem": "Estimate the annual carbon footprint of a company's cloud infrastructure from its billing exports.", + "steps": [ + { + "id": "s1", + "kind": "capability", + "ref": "run_python@1", + "args": { + "input": "$input", + "task": "Detect the billing-export format, load all supplied files or records, normalize line items into a common table with provider, account, service, SKU, region, usage quantity/unit, cost, currency, and billing period, and report missing or ambiguous fields." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "{records: table, coverage: object, warnings: string[]}", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s2", + "kind": "capability", + "ref": "web_search@1", + "args": { + "query": "cloud carbon footprint methodology AWS Azure Google Cloud regional carbon intensity energy usage emissions factors official documentation" + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "{results: [{title: string, url: string, snippet: string}]}", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s3", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": { + "sources": "$s2", + "billing_profile": "$s1.coverage" + }, + "instruction": "Produce a cited emissions-factor and estimation-method table applicable to the detected providers, services, regions, and usage units; distinguish operational electricity emissions, embodied emissions, provider-reported data, usage-based estimates, and spend-based fallback assumptions." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "{factors: table, methods: object, citations: [{claim: string, url: string}], limitations: string[]}", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s4", + "kind": "capability", + "ref": "run_python@1", + "args": { + "records": "$s1.records", + "factor_model": "$s3", + "task": "Map billing line items to factors, estimate monthly and annualized kgCO2e, avoid double counting, calculate provider/service/region breakdowns, quantify data coverage, and produce low/base/high scenarios for uncertain mappings and fallback assumptions." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "{annual_estimate: object, monthly: table, breakdowns: object, coverage: object, assumptions: string[], unmapped: table}", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s5", + "kind": "capability", + "ref": "plot_chart@1", + "args": { + "data": "$s4", + "charts": [ + "monthly emissions trend", + "annual emissions by provider and service", + "emissions by region", + "estimate coverage and uncertainty" + ] + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "{charts: chart[]}", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s6", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": { + "results": "$s4", + "charts": "$s5", + "sources": "$s3.citations", + "ingestion_warnings": "$s1.warnings" + }, + "instruction": "Write an executive-ready annual cloud carbon-footprint report stating the reporting period, total tCO2e and uncertainty range, methodology, operational-versus-embodied scope, key drivers, coverage, assumptions, exclusions, cited sources, limitations, and prioritized reduction recommendations. Clearly label annualized estimates when exports cover less than a full year." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [ + { + "id": "s1", + "atomic": false, + "why": "Open-ended multi-format billing detection and normalization requires extensive custom logic." + }, + { + "id": "s2", + "atomic": true, + "why": "A single web search can return titled URLs with snippets." + }, + { + "id": "s3", + "atomic": false, + "why": "Requires source evaluation and factors not accessible from search snippets alone." + }, + { + "id": "s4", + "atomic": false, + "why": "Mapping services to emissions factors and avoiding double counting requires complex judgment." + }, + { + "id": "s5", + "atomic": true, + "why": "Charting tool can plot requested charts from prepared result data." + }, + { + "id": "s6", + "atomic": true, + "why": "LLM transform can draft a report from supplied results, charts, and citations." + } + ] + }, + { + "planner": "gpt-5.6-sol", + "judge": "gpt-5.5", + "problem_index": 11, + "problem": "Given a corpus of 50,000 customer support tickets, identify the five product def", + "b": 5, + "declared_ambiguous": 0, + "atomic_steps": 5, + "overclaimed": 3, + "declared_f": 0.0, + "corrected_f": 0.6, + "corrected_ambiguous": 3, + "plan": { + "id": "plan-94922371", + "problem": "Given a corpus of 50,000 customer support tickets, identify the five product defects responsible for the most support load.", + "steps": [ + { + "id": "s1", + "kind": "capability", + "ref": "extract_text@1", + "args": { + "input": "$input", + "preserve_metadata": true, + "record_unit": "ticket" + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "array<{ticket_id:string,text:string,metadata:object}>", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s2", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": "$s1", + "mode": "batch_structured_extraction", + "instructions": "For each ticket, determine whether it concerns a genuine product defect rather than a usage question, feature request, account issue, or service inquiry. For defects, extract product/component, concise defect description, normalized candidate defect label, confidence, and any support-load fields present in metadata or text such as handling time, reopen count, contacts, or escalation. Preserve ticket_id." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "array<{ticket_id:string,is_defect:boolean,product:string|null,component:string|null,defect_description:string|null,candidate_label:string|null,confidence:number,load_signals:object}>", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s3", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": "$s2", + "instructions": "Consolidate semantically equivalent candidate defect labels into a canonical defect taxonomy, keeping distinct root causes separate when evidence supports doing so. Return an alias-to-canonical mapping with a clear canonical name and description for each defect." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "{defects:array<{canonical_id:string,name:string,description:string,aliases:array}>,alias_map:object}", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s4", + "kind": "capability", + "ref": "run_python@1", + "args": { + "inputs": { + "tickets": "$s2", + "taxonomy": "$s3" + }, + "task": "Map defect tickets to canonical defects, deduplicate by ticket_id, and aggregate support load. Use total handling time as the primary metric if it is populated reliably across the corpus; otherwise use unique ticket count. Also calculate ticket count, available secondary load indicators, share of total defect load, and confidence statistics. Rank defects and return the top five with representative anonymized ticket examples." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "{metric_used:string,coverage:object,total_defect_load:number,top_five:array<{rank:integer,canonical_id:string,name:string,description:string,load:number,load_share:number,ticket_count:integer,secondary_metrics:object,confidence_stats:object,examples:array}>}", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s5", + "kind": "capability", + "ref": "summarize@1", + "args": { + "input": "$s4", + "format": "report", + "instructions": "Produce a concise decision-ready report identifying the five product defects responsible for the most support load. Include the ranking, measured load and share, defect descriptions, supporting examples, methodology, metric coverage, and caveats about classification confidence or missing workload data." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [ + { + "id": "s1", + "atomic": true, + "why": "Text extraction can split tickets and preserve available metadata." + }, + { + "id": "s2", + "atomic": false, + "why": "Requires large-scale classification and defect judgment across 50,000 tickets." + }, + { + "id": "s3", + "atomic": false, + "why": "Semantic taxonomy consolidation over many labels requires clustering and judgment." + }, + { + "id": "s4", + "atomic": false, + "why": "Requires metric choice and mapping logic not fully specified." + }, + { + "id": "s5", + "atomic": true, + "why": "Summarization can turn aggregate results into a concise report." + } + ] + }, + { + "planner": "gpt-5.6-sol", + "judge": "gpt-5.5", + "problem_index": 13, + "problem": "Design and run an experiment that determines whether a new ranking model improve", + "b": 4, + "declared_ambiguous": 1, + "atomic_steps": 3, + "overclaimed": 0, + "declared_f": 0.25, + "corrected_f": 0.25, + "corrected_ambiguous": 1, + "plan": { + "id": "plan-32514979", + "problem": "Design and run an experiment that determines whether a new ranking model improves user outcomes.", + "steps": [ + { + "id": "s1", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": "", + "instruction": "Create a preregistered randomized controlled A/B test plan for the new ranking model versus the current model. Define eligibility, randomization unit, exposure logging, primary user-outcome metric, guardrails, hypotheses, minimum detectable effect, power and sample-size method, duration, contamination controls, data-quality checks, stopping rules, and analysis plan." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "experiment_plan", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s2", + "kind": "decompose", + "ref": null, + "args": {}, + "problem": "Implement the approved experiment plan in the product experimentation system, validate assignment and telemetry with an A/A or staged ramp, run the control and treatment arms until the prespecified stopping criterion is reached, and export de-identified assignment, exposure, outcome, and guardrail data.", + "inputs": [], + "outputs": [ + "experiment_data", + "execution_log" + ], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "{experiment_data: dataset, execution_log: log}", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s3", + "kind": "capability", + "ref": "run_python@1", + "args": { + "data": "", + "plan": "", + "task": "Validate sample ratios and telemetry; estimate intention-to-treat effects on the primary outcome and guardrails with confidence intervals and prespecified significance or multiplicity corrections; assess practical significance, heterogeneity, novelty or time trends, and sensitivity to missing data; return tables, diagnostics, and a ship/hold/reject conclusion constrained by the preregistered decision rule." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "experiment_analysis", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + }, + { + "id": "s4", + "kind": "capability", + "ref": "llm_transform@1", + "args": { + "input": { + "plan": "", + "execution_log": "", + "analysis": "" + }, + "instruction": "Produce a decision-ready experiment report covering design, execution, data-quality checks, primary and guardrail results with uncertainty, limitations, and a clear recommendation on whether the new ranking model improves user outcomes." + }, + "problem": null, + "inputs": [], + "outputs": [], + "guard": null, + "then": [], + "otherwise": [], + "branches": [], + "max_iterations": null, + "output_schema": "report", + "authority": { + "net": [], + "fs_write": [], + "subprocess": false, + "spend_usd": 0.0 + } + } + ], + "output_schema": "any", + "signature": "any->report" + }, + "verdicts": [ + { + "id": "s1", + "atomic": true, + "why": "LLM can draft a complete A/B test preregistration plan from the problem." + }, + { + "id": "s3", + "atomic": true, + "why": "Python can validate data and compute preregistered statistical estimates from provided inputs." + }, + { + "id": "s4", + "atomic": true, + "why": "LLM can synthesize provided plan, log, and analysis into a report." + } + ] + } +] \ No newline at end of file diff --git a/scripts/prototypes/minikernel/__init__.py b/scripts/prototypes/minikernel/__init__.py new file mode 100644 index 00000000..aa45ef5e --- /dev/null +++ b/scripts/prototypes/minikernel/__init__.py @@ -0,0 +1,25 @@ +"""A deliberately small, runnable kernel for the #485 redesign. + +Not a product. A schematic you can execute: every mechanism the issue proposes +is present in its cheapest honest form, so that its claims can be tested rather +than argued about. See scripts/prototypes/README.md. +""" + +from .capabilities import (BugReport, BugTracker, Capability, + CapabilityRegistry, SeparationOfDuty) +from .ir import Authority, Budget, Plan, Step, validate +from .library import SolvedProblemLibrary +from .planner import LLMPlanner, PlanDraft, StubPlanner, load_env_key +from .review import (ArtifactVersion, Criterion, Finding, InsightPool, + ReviewBoard) +from .runtime import Crash, Message, MessageBus, NodeResult, Runtime, RunStats +from .store import Store + +__all__ = [ + "Authority", "Budget", "Plan", "Step", "validate", "Store", + "Capability", "CapabilityRegistry", "BugReport", "BugTracker", + "SeparationOfDuty", "SolvedProblemLibrary", "StubPlanner", "LLMPlanner", + "PlanDraft", "load_env_key", "ReviewBoard", "Criterion", "Finding", + "ArtifactVersion", "InsightPool", "Runtime", "RunStats", "NodeResult", + "Message", "MessageBus", "Crash", +] diff --git a/scripts/prototypes/minikernel/capabilities.py b/scripts/prototypes/minikernel/capabilities.py new file mode 100644 index 00000000..84c6f8a4 --- /dev/null +++ b/scripts/prototypes/minikernel/capabilities.py @@ -0,0 +1,222 @@ +"""One lifecycle for tools, skills and reusable plans. + +#485 keeps tools, skills and pipelines apart, but they need the same things: +an immutable version, a declared contract, a declared authority envelope, a +test suite, an independent qualification, and a complete call history. So this +module gives them one type and one maturity ladder: + + draft -> candidate -> trusted (promotion, by an independent session) + -> quarantined -> revoked (demotion, by an independent session) + +Two rules that #485 leaves implicit and that the runtime enforces: + + R1 A capability is never mutated after review. A change publishes a NEW + version; old runs stay replayable against the version they used. + R2 The session that authored a capability may never be the session that + qualifies it -- and qualification means *running* it, not reading it. +""" + +from __future__ import annotations + +import time +import traceback +from dataclasses import dataclass, field +from typing import Any, Callable + +from .ir import Authority +from .store import Store, content_hash + +MATURITIES = ("draft", "candidate", "trusted", "quarantined", "revoked") + + +@dataclass +class Capability: + name: str + version: int + impl: Callable[[dict[str, Any]], Any] + input_schema: str + output_schema: str + side_effect: str = "pure" # pure | read | write | external + authority: Authority = field(default_factory=Authority) + tests: list[tuple[dict[str, Any], Any]] = field(default_factory=list) + maturity: str = "draft" + author_session: str = "unknown" + source: str = "" + + @property + def ref(self) -> str: + return f"{self.name}@{self.version}" + + @property + def content_hash(self) -> str: + return content_hash( + f"{self.name}|{self.version}|{self.input_schema}|{self.output_schema}" + f"|{self.source}" + ) + + +class SeparationOfDuty(Exception): + pass + + +class CapabilityRegistry: + def __init__(self, store: Store): + self.store = store + self._caps: dict[str, Capability] = {} + + # -------------------------------------------------------------- lifecycle + + def register(self, cap: Capability) -> str: + if cap.ref in self._caps: + raise ValueError(f"{cap.ref} already registered; publish a new version") + self._caps[cap.ref] = cap + self.store.append_event( + "registry", cap.ref, "capability_registered", + {"ref": cap.ref, "hash": cap.content_hash, "maturity": cap.maturity, + "side_effect": cap.side_effect, "author": cap.author_session}, + session_id=cap.author_session, + ) + return cap.ref + + def seed(self, cap: Capability) -> str: + """Register a capability that ships with the kernel as already trusted.""" + cap.maturity = "trusted" + cap.author_session = "kernel" + return self.register(cap) + + def qualify(self, ref: str, reviewer_session: str) -> tuple[bool, list[str]]: + """Run the capability's tests in a session that did not author it (R2).""" + cap = self._caps[ref] + if reviewer_session == cap.author_session: + raise SeparationOfDuty( + f"session {reviewer_session!r} authored {ref} and may not qualify it" + ) + if not cap.tests: + self._set_maturity(ref, "quarantined", reviewer_session, + "no tests supplied") + return False, ["capability ships no tests; cannot be qualified"] + failures: list[str] = [] + for i, (args, expected) in enumerate(cap.tests): + try: + got = cap.impl(dict(args)) + except Exception: + failures.append(f"test {i}: raised\n{traceback.format_exc(limit=2)}") + continue + if got != expected: + failures.append(f"test {i}: expected {expected!r}, observed {got!r}") + ok = not failures + self._set_maturity(ref, "trusted" if ok else "quarantined", + reviewer_session, "; ".join(failures)[:400]) + return ok, failures + + def revoke(self, ref: str, reviewer_session: str, reason: str) -> None: + self._set_maturity(ref, "revoked", reviewer_session, reason) + + def _set_maturity(self, ref: str, maturity: str, session: str, reason: str) -> None: + assert maturity in MATURITIES + self._caps[ref].maturity = maturity + self.store.append_event( + "registry", ref, "capability_maturity_changed", + {"ref": ref, "maturity": maturity, "reason": reason}, + session_id=session, + ) + + # ------------------------------------------------------------ invocation + + def maturities(self) -> dict[str, str]: + return {ref: c.maturity for ref, c in self._caps.items()} + + def get(self, ref: str) -> Capability: + return self._caps[ref] + + def invoke( + self, + ref: str, + args: dict[str, Any], + run_id: str, + node_id: str, + session_id: str, + authority: Authority, + ) -> Any: + cap = self._caps.get(ref) + if cap is None: + raise KeyError(f"no such capability {ref!r}") + if cap.maturity not in ("trusted", "candidate"): + raise PermissionError(f"{ref} is {cap.maturity}; refusing to invoke") + if not authority.covers(cap.authority): + raise PermissionError( + f"{ref} needs authority the caller does not hold: " + f"net={sorted(cap.authority.net)} subprocess={cap.authority.subprocess}" + ) + t0 = time.time() + try: + result = cap.impl(dict(args)) + error = None + except Exception as exc: # recorded, then re-raised: never swallowed + result, error = None, f"{type(exc).__name__}: {exc}" + self.store.append_event( + run_id, node_id, "capability_invoked", + {"capability": ref, "hash": cap.content_hash, "args": args, + "error": error, "seconds": round(time.time() - t0, 4), + "side_effect": cap.side_effect}, + session_id=session_id, + ) + if error: + raise RuntimeError(f"{ref} failed: {error}") + return result + + +# ------------------------------------------------------- bug-report workflow + + +@dataclass +class BugReport: + id: str + capability_ref: str + use_case: str + expected: str + observed: str + reporter_session: str + status: str = "open" + + +class BugTracker: + def __init__(self, store: Store, registry: CapabilityRegistry): + self.store = store + self.registry = registry + self.reports: dict[str, BugReport] = {} + + def file(self, report: BugReport) -> str: + self.reports[report.id] = report + self.store.append_event( + "registry", report.capability_ref, "bug_reported", + {"id": report.id, "expected": report.expected, + "observed": report.observed, "use_case": report.use_case}, + session_id=report.reporter_session, + ) + return report.id + + def triage(self, report_id: str, reviewer_session: str) -> str: + """A DIFFERENT agent decides, using the call history as evidence.""" + rep = self.reports[report_id] + if reviewer_session == rep.reporter_session: + raise SeparationOfDuty("bug reports are triaged by a different session") + history = self.store.view_tool_history(rep.capability_ref) + failures = [h for h in history if '"error": null' not in h["payload"]] + rate = len(failures) / len(history) if history else 0.0 + if rate >= 0.5 and len(history) >= 4: + self.registry.revoke(rep.capability_ref, reviewer_session, + f"{len(failures)}/{len(history)} invocations failed") + decision = "revoked" + elif failures: + decision = "fix" + else: + decision = "clarify_usage" + rep.status = decision + self.store.append_event( + "registry", rep.capability_ref, "bug_triaged", + {"id": report_id, "decision": decision, "failure_rate": round(rate, 3), + "invocations": len(history)}, + session_id=reviewer_session, + ) + return decision diff --git a/scripts/prototypes/minikernel/ir.py b/scripts/prototypes/minikernel/ir.py new file mode 100644 index 00000000..7726760d --- /dev/null +++ b/scripts/prototypes/minikernel/ir.py @@ -0,0 +1,286 @@ +"""Typed plan IR, its validator, and the budget ledger. + +#485 asks for "loops, conditionals, and goto ... a complete (albeit simple) +language". This IR is deliberately structured: sequence + branch + bounded loop ++ call is already Turing-complete, and every one of those forms can be +statically bounded, resumed, diffed and drawn. Unstructured `goto` buys nothing +and costs all four. The repo's five open control-flow bugs (#474-#478) are the +empirical argument: they are all in a hand-written YAML dialect with a +hand-written interpreter. + +The validator is where the design gets its teeth. It rejects a plan that: + - loops without a static iteration bound, + - calls a capability that is not registered, or not promoted, + - requests authority its parent does not hold (authority narrows downward, + never widens), + - exceeds the fan-out cap (#485's "on the order of 10 or fewer" steps), + - declares no output contract. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field, replace +from typing import Any, Literal + +Kind = Literal[ + "capability", "plan", "decompose", "branch", "loop", "parallel", "return", "fail" +] + +MAX_STEPS_PER_PLAN = 12 +MAX_LOOP_ITERATIONS = 64 + + +class PlanInvalid(Exception): + def __init__(self, errors: list[str]): + super().__init__("; ".join(errors)) + self.errors = errors + + +@dataclass(frozen=True) +class Authority: + """What a node is allowed to do. Inherited downward, never broadened.""" + + net: frozenset[str] = frozenset() + fs_write: frozenset[str] = frozenset() + subprocess: bool = False + spend_usd: float = 0.0 + + def covers(self, other: "Authority") -> bool: + return ( + other.net <= self.net + and other.fs_write <= self.fs_write + and (self.subprocess or not other.subprocess) + and other.spend_usd <= self.spend_usd + 1e-9 + ) + + def narrow(self, other: "Authority") -> "Authority": + return Authority( + net=self.net & other.net, + fs_write=self.fs_write & other.fs_write, + subprocess=self.subprocess and other.subprocess, + spend_usd=min(self.spend_usd, other.spend_usd), + ) + + +@dataclass(frozen=True) +class Step: + id: str + kind: Kind + # capability / plan + ref: str | None = None + args: dict[str, Any] = field(default_factory=dict) + # decompose + problem: str | None = None + inputs: tuple[str, ...] = () + outputs: tuple[str, ...] = () + # branch / loop / parallel + guard: str | None = None + then: tuple["Step", ...] = () + otherwise: tuple["Step", ...] = () + branches: tuple[tuple["Step", ...], ...] = () + max_iterations: int | None = None + # everything + output_schema: str = "any" + authority: Authority = Authority() + + def children(self) -> list["Step"]: + out = list(self.then) + list(self.otherwise) + for b in self.branches: + out += list(b) + return out + + +@dataclass(frozen=True) +class Plan: + id: str + problem: str + steps: tuple[Step, ...] + output_schema: str = "any" + signature: str = "any->any" + + def hash(self) -> str: + from .store import content_hash + + return content_hash(json.dumps(as_jsonable(self), sort_keys=True)) + + def walk(self) -> list[Step]: + out: list[Step] = [] + stack = list(self.steps) + while stack: + s = stack.pop(0) + out.append(s) + stack = s.children() + stack + return out + + +def as_jsonable(obj: Any) -> Any: + if isinstance(obj, (Plan, Step, Authority)): + d = {} + for k, v in obj.__dict__.items(): + d[k] = as_jsonable(v) + return d + if isinstance(obj, (list, tuple)): + return [as_jsonable(v) for v in obj] + if isinstance(obj, (set, frozenset)): + return sorted(as_jsonable(v) for v in obj) + if isinstance(obj, dict): + return {k: as_jsonable(v) for k, v in obj.items()} + return obj + + +def validate( + plan: Plan, + known_capabilities: dict[str, str], + parent_authority: Authority, + max_steps: int = MAX_STEPS_PER_PLAN, +) -> list[str]: + """Return a list of errors. Empty list == valid.""" + errors: list[str] = [] + seen_ids: set[str] = set() + + if not plan.steps: + errors.append("plan has no steps") + if len(plan.steps) > max_steps: + errors.append( + f"plan has {len(plan.steps)} top-level steps (cap {max_steps})" + ) + + def check(step: Step, depth: int) -> None: + if step.id in seen_ids: + errors.append(f"duplicate step id {step.id!r}") + seen_ids.add(step.id) + if not step.output_schema: + errors.append(f"{step.id}: no output contract") + if not parent_authority.covers(step.authority): + errors.append( + f"{step.id}: requests authority beyond its parent " + f"(net={sorted(step.authority.net)}, subprocess={step.authority.subprocess})" + ) + if step.kind == "capability": + if not step.ref: + errors.append(f"{step.id}: capability step with no ref") + elif step.ref not in known_capabilities: + errors.append(f"{step.id}: unknown capability {step.ref!r}") + elif known_capabilities[step.ref] not in ("trusted", "candidate"): + errors.append( + f"{step.id}: capability {step.ref!r} is " + f"{known_capabilities[step.ref]}, not promoted" + ) + elif step.kind == "decompose": + if not step.problem: + errors.append(f"{step.id}: decompose step with no problem statement") + if not step.outputs: + errors.append(f"{step.id}: decompose step declares no outputs") + elif step.kind == "loop": + if step.max_iterations is None: + errors.append(f"{step.id}: loop has no static iteration bound") + elif not (1 <= step.max_iterations <= MAX_LOOP_ITERATIONS): + errors.append( + f"{step.id}: loop bound {step.max_iterations} outside " + f"1..{MAX_LOOP_ITERATIONS}" + ) + if not step.guard: + errors.append(f"{step.id}: loop has no guard") + if not step.then: + errors.append(f"{step.id}: loop has an empty body") + elif step.kind == "branch": + if not step.guard: + errors.append(f"{step.id}: branch has no guard") + if not step.then and not step.otherwise: + errors.append(f"{step.id}: branch has no arms") + elif step.kind == "parallel": + if len(step.branches) < 2: + errors.append(f"{step.id}: parallel with fewer than 2 branches") + elif step.kind == "plan": + if not step.ref: + errors.append(f"{step.id}: plan call with no ref") + for c in step.children(): + check(c, depth + 1) + + for s in plan.steps: + check(s, 0) + if not any(s.kind in ("capability", "decompose", "plan") for s in plan.walk()): + errors.append("plan does no work: no capability, plan or decompose step") + return errors + + +# ------------------------------------------------------------------ budgets + + +class BudgetExhausted(Exception): + pass + + +@dataclass +class Budget: + """A credit line, not a depth cap. + + A depth cap silently truncates and returns a plausible-looking answer. A + budget that is *spent* forces the node to say so, and hands the decision to + re-allocate, re-plan or escalate to somebody with a wider view. + """ + + tokens: int + usd: float + nodes: int + seconds: float + spent_tokens: int = 0 + spent_usd: float = 0.0 + spent_nodes: int = 0 + spent_seconds: float = 0.0 + reserve_frac: float = 0.30 + + def remaining(self) -> "Budget": + return Budget( + max(0, self.tokens - self.spent_tokens), + max(0.0, self.usd - self.spent_usd), + max(0, self.nodes - self.spent_nodes), + max(0.0, self.seconds - self.spent_seconds), + reserve_frac=self.reserve_frac, + ) + + def exhausted(self) -> bool: + r = self.remaining() + return r.tokens <= 0 or r.usd <= 0 or r.nodes <= 0 or r.seconds <= 0 + + def spend(self, tokens: int = 0, usd: float = 0.0, nodes: int = 0, + seconds: float = 0.0) -> None: + self.spent_tokens += tokens + self.spent_usd += usd + self.spent_nodes += nodes + self.spent_seconds += seconds + + def child_share(self, n_children: int) -> "Budget": + """Split, holding a reserve back for reallocation and escalation.""" + n = max(1, n_children) + r = self.remaining() + keep = 1.0 - self.reserve_frac + return Budget( + int(r.tokens * keep / n), + r.usd * keep / n, + max(1, int(r.nodes * keep / n)), + r.seconds * keep / n, + reserve_frac=self.reserve_frac, + ) + + def absorb(self, child: "Budget") -> None: + self.spend(child.spent_tokens, child.spent_usd, child.spent_nodes, + child.spent_seconds) + + def top_up(self, other: "Budget", tokens: int) -> int: + """Move credit from a parent's reserve into this budget. Recorded.""" + avail = other.remaining().tokens + moved = min(tokens, avail) + other.spend(tokens=moved) + self.tokens += moved + return moved + + def as_dict(self) -> dict[str, Any]: + r = self.remaining() + return { + "tokens": self.tokens, "spent_tokens": self.spent_tokens, + "remaining_tokens": r.tokens, "nodes": self.nodes, + "spent_nodes": self.spent_nodes, "usd": round(self.usd, 4), + "spent_usd": round(self.spent_usd, 4), + } diff --git a/scripts/prototypes/minikernel/library.py b/scripts/prototypes/minikernel/library.py new file mode 100644 index 00000000..096cd7c3 --- /dev/null +++ b/scripts/prototypes/minikernel/library.py @@ -0,0 +1,234 @@ +"""The solved-problem library -- the termination mechanism, not a cache. + +#485 files "compound engineering" under component 3, as an efficiency story. +The branching analysis says it is actually the control system for component 1: +recursion terminates in expectation iff `m = b*f < 1`, and every solved subtree +that lands in the library converts a future ambiguous step into an atomic one, +which is the only mechanism in the design that drives `f` down. A system that +starts supercritical becomes subcritical by remembering. + +Retrieval keys on TWO things, because semantic similarity alone produces the +"close enough, wrong shape" failure: a normalised statement similarity AND a +compatible typed I/O signature. Both must pass. +""" + +from __future__ import annotations + +import json +import math +import re +from dataclasses import dataclass, field +from typing import Any + +from .ir import Plan +from .store import Store, content_hash + +_STOP = { + "the", "a", "an", "and", "or", "of", "to", "for", "in", "on", "with", "by", + "from", "that", "this", "it", "is", "are", "be", "as", "at", "into", "then", +} + + +def tokens_of(statement: str) -> set[str]: + return { + w for w in re.findall(r"[a-z0-9]+", statement.lower()) + if len(w) > 2 and w not in _STOP + } + + +def similarity(a: str, b: str) -> float: + ta, tb = tokens_of(a), tokens_of(b) + if not ta or not tb: + return 0.0 + return len(ta & tb) / len(ta | tb) + + +def signature_compatible(want: str, have: str) -> bool: + """`in1,in2->out`. `any` is a wildcard on either side of the arrow.""" + def parse(sig: str) -> tuple[list[str], str]: + lhs, _, rhs = sig.partition("->") + return [p.strip() for p in lhs.split(",") if p.strip()], rhs.strip() or "any" + + wi, wo = parse(want) + hi, ho = parse(have) + if wo != "any" and ho != "any" and wo != ho: + return False + if len(wi) != len(hi): + return False + return all(x == "any" or y == "any" or x == y for x, y in zip(wi, hi)) + + +@dataclass +class Intractable: + """A recorded NEGATIVE result. + + Found by scenario S7: a mission that ends at the depth cap teaches the + system nothing, so re-running it costs exactly as much as the first time, + forever. The library must remember what it could NOT solve, and under what + allowance -- otherwise the only regime where learning matters is the one + regime where learning cannot start. + """ + + key: str + statement: str + signature: str + reason: str + depth_allowance: int + attempts: int = 1 + + +@dataclass +class Solution: + key: str + statement: str + signature: str + plan_json: str + evidence: dict[str, Any] = field(default_factory=dict) + uses: int = 0 + wins: int = 0 + provenance: str = "" + + @property + def reliability(self) -> float: + """Laplace-smoothed success rate: an unproven entry is not a sure thing.""" + return (self.wins + 1) / (self.uses + 2) + + +class SolvedProblemLibrary: + def __init__(self, store: Store, sim_threshold: float = 0.55, + min_reliability: float = 0.4): + self.store = store + self.sim_threshold = sim_threshold + self.min_reliability = min_reliability + self.entries: dict[str, Solution] = {} + self.dead_ends: dict[str, Intractable] = {} + + def __len__(self) -> int: + return len(self.entries) + + def publish(self, statement: str, signature: str, plan: Plan, + evidence: dict[str, Any], provenance: str = "") -> str | None: + """Publish a solved problem -- if its shape is concrete enough to reuse. + + FIX (found by scenario S7): the two-key match is only as strong as the + weaker key. A solution stored with signature `any->any` matches every + later query, so text similarity silently becomes the ONLY key and the + "close enough, wrong shape" failure comes straight back. An untyped + solution is therefore not published at all. + """ + from .ir import as_jsonable + + _, _, out = signature.partition("->") + if out.strip() in ("", "any"): + self.store.append_event( + "library", content_hash(statement), "solution_not_published", + {"reason": "wildcard output signature; not safely reusable", + "signature": signature, "statement": statement[:200]}, + ) + return None + key = content_hash(f"{sorted(tokens_of(statement))}|{signature}") + if key in self.entries: + return key + sol = Solution(key, statement, signature, + json.dumps(as_jsonable(plan), sort_keys=True), + evidence, provenance=provenance) + self.entries[key] = sol + self.store.append_event( + "library", key, "solution_published", + {"statement": statement[:200], "signature": signature, + "evidence": evidence, "provenance": provenance}, + ) + return key + + def lookup(self, statement: str, signature: str) -> Solution | None: + best, best_sim = None, 0.0 + for sol in self.entries.values(): + if not signature_compatible(signature, sol.signature): + continue + if sol.reliability < self.min_reliability: + continue + s = similarity(statement, sol.statement) + if s >= self.sim_threshold and s > best_sim: + best, best_sim = sol, s + if best is not None: + self.store.append_event( + "library", best.key, "solution_hit", + {"query": statement[:200], "similarity": round(best_sim, 3), + "reliability": round(best.reliability, 3)}, + ) + return best + + # ------------------------------------------------------ negative results + + @staticmethod + def _neg_key(statement: str, signature: str) -> str: + return content_hash(f"NEG|{sorted(tokens_of(statement))}|{signature}") + + def record_intractable(self, statement: str, signature: str, reason: str, + depth_allowance: int) -> str: + key = self._neg_key(statement, signature) + rec = self.dead_ends.get(key) + if rec is None: + rec = Intractable(key, statement, signature, reason, depth_allowance) + self.dead_ends[key] = rec + else: + rec.attempts += 1 + rec.depth_allowance = max(rec.depth_allowance, depth_allowance) + self.store.append_event( + "library", key, "intractable_recorded", + {"statement": statement[:200], "signature": signature, + "reason": reason, "depth_allowance": depth_allowance, + "attempts": rec.attempts}, + ) + return key + + def lookup_intractable(self, statement: str, signature: str, + depth_allowance: int) -> Intractable | None: + """A dead end only counts if the earlier attempt had AT LEAST as much + room as this one. More budget is a legitimate reason to try again.""" + for rec in self.dead_ends.values(): + if not signature_compatible(signature, rec.signature): + continue + if rec.depth_allowance < depth_allowance: + continue + if similarity(statement, rec.statement) >= self.sim_threshold: + self.store.append_event("library", rec.key, "intractable_hit", + {"query": statement[:200], + "attempts": rec.attempts}) + return rec + return None + + def record_use(self, key: str, outcome: str) -> None: + """Record evidence about a reused plan. + + FIX (found by scenario S7): reliability must only move on evidence + ABOUT THE PLAN. A run cut short by budget or by the depth cap says + nothing about whether the cached plan was right, but counting it as a + failure drops reliability below the retrieval floor after a single + unlucky mission -- so budget pressure silently un-learns the library. + """ + sol = self.entries.get(key) + if sol is None: + return + if outcome in ("budget_exhausted", "escalated"): + self.store.append_event("library", key, "solution_use_uncounted", + {"outcome": outcome}) + return + sol.uses += 1 + sol.wins += int(outcome == "completed") + self.store.append_event("library", key, "solution_used", + {"outcome": outcome, "uses": sol.uses, + "reliability": round(sol.reliability, 3)}) + + def stats(self) -> dict[str, Any]: + if not self.entries: + return {"size": 0, "dead_ends": len(self.dead_ends), + "mean_reliability": 0.0, "total_uses": 0} + return { + "size": len(self.entries), + "dead_ends": len(self.dead_ends), + "mean_reliability": round( + sum(s.reliability for s in self.entries.values()) / len(self.entries), 3 + ), + "total_uses": sum(s.uses for s in self.entries.values()), + } diff --git a/scripts/prototypes/minikernel/planner.py b/scripts/prototypes/minikernel/planner.py new file mode 100644 index 00000000..99fa668a --- /dev/null +++ b/scripts/prototypes/minikernel/planner.py @@ -0,0 +1,272 @@ +"""Planners: the thing that decides `f`. + +`f` -- the fraction of emitted steps a planner calls AMBIGUOUS rather than +atomic -- is the load-bearing parameter of the whole design. With mean fan-out +`b`, recursion is finite in expectation iff `m = b*f < 1`. Every review of #485 +so far has had to GUESS `f`. This module exists so it can be measured. + +Two implementations behind one interface: + + StubPlanner deterministic, seeded, no network. Used by the tests and by the + ablation sweeps, where we want to *set* f and observe the system. + + LLMPlanner a real model, asked to decompose a real problem against the real + capability library, emitting strict JSON. Used to *measure* f -- + and, critically, to measure f as a function of library size, + which is the claim that the library is the termination mechanism. + +Note what `f` is NOT: a property of the model alone. It is a property of +(model, problem distribution, library contents). Any measured value must be +quoted with all three. +""" + +from __future__ import annotations + +import json +import os +import random +import re +import time +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from typing import Any, Protocol + +from .ir import Authority, Plan, Step + + +@dataclass +class PlanDraft: + plan: Plan + rationale: str = "" + tokens_in: int = 0 + tokens_out: int = 0 + raw: str = "" + model: str = "stub" + + @property + def fan_out(self) -> int: + return len(self.plan.steps) + + @property + def n_ambiguous(self) -> int: + return sum(1 for s in self.plan.steps if s.kind == "decompose") + + @property + def f(self) -> float: + return self.n_ambiguous / self.fan_out if self.fan_out else 0.0 + + +class Planner(Protocol): + def decompose(self, problem: str, signature: str, + capabilities: dict[str, str], depth: int) -> PlanDraft: ... + + +# --------------------------------------------------------------- stub planner + + +@dataclass +class StubPlanner: + """Deterministic planner with a *settable* ambiguity rate. + + Used to drive the system through regimes we cannot afford to reach with a + real model (m > 1 with thousands of nodes). + """ + + seed: int = 0 + b: int = 5 + f: float = 0.2 + capability_pool: tuple[str, ...] = ("echo@1",) + + def _rng_for(self, problem: str, depth: int) -> random.Random: + """Deterministic in (problem, depth). + + FIX (found by scenario S7): the first version carried one RNG stream + across calls, so asking the same question twice produced different + plans. No real planner is memoryless like that, and it made the + library's benefit unmeasurable -- mission 5 drew a worse plan than + mission 1 for reasons that had nothing to do with the library. A + planner must be a function of its inputs for a cache over it to mean + anything, which is also why the runtime content-addresses plan inputs. + """ + import hashlib + + h = hashlib.sha256(f"{self.seed}|{problem}|{depth}".encode()).digest() + return random.Random(int.from_bytes(h[:8], "big")) + + def decompose(self, problem: str, signature: str, + capabilities: dict[str, str], depth: int) -> PlanDraft: + rng = self._rng_for(problem, depth) + usable = [r for r, m in capabilities.items() if m in ("trusted", "candidate")] + usable = usable or list(self.capability_pool) + n = max(2, min(self.b, 12)) + steps: list[Step] = [] + for i in range(n): + if rng.random() < self.f: + steps.append( + Step(id=f"s{i}", kind="decompose", + problem=f"{problem} :: part {i} (depth {depth})", + outputs=("y",), output_schema="str") + ) + else: + steps.append( + Step(id=f"s{i}", kind="capability", + ref=rng.choice(usable), + args={"text": f"{problem}#{i}"}, output_schema="str") + ) + return PlanDraft(Plan(id=f"plan-{abs(hash(problem)) % 10**8}", + problem=problem, steps=tuple(steps), + signature=signature), + rationale="stub", tokens_in=0, tokens_out=0) + + +# ---------------------------------------------------------------- LLM planner + +SYSTEM = """You are the decomposition planner of a recursive problem-solving \ +runtime. Break the given problem into a SHORT pipeline of concrete steps \ +(aim for 3-7, hard maximum 10). + +Each step is exactly one of: + {"id": "s1", "kind": "capability", "ref": "", + "args": {...}, "output_schema": ""} + -- use this when the step can be done RIGHT NOW by that capability, with + at most trivial adaptation of its output. + {"id": "s2", "kind": "decompose", "problem": "", + "outputs": ["name"], "output_schema": ""} + -- use this ONLY when you do not know how to do the step with the + available capabilities and it must itself be broken down further. + +Be honest and be economical: marking a step "decompose" costs a whole recursive \ +subtree, so only do it when the step genuinely needs one. If an available \ +capability does the job, use it. + +Reply with ONLY a JSON object: {"steps": [...], "rationale": ""}""" + + +class LLMPlanner: + """OpenAI-compatible chat completions over stdlib urllib. No SDK, no mocks.""" + + def __init__(self, model: str, api_key: str, + base_url: str = "https://api.openai.com/v1", + max_retries: int = 3, temperature: float | None = None): + self.model = model + self.api_key = api_key + self.base_url = base_url.rstrip("/") + self.max_retries = max_retries + self.temperature = temperature + self.calls = 0 + self.tokens_in = 0 + self.tokens_out = 0 + + def _chat(self, messages: list[dict[str, str]], + max_tokens: int = 1600) -> tuple[str, int, int]: + payload: dict[str, Any] = {"model": self.model, "messages": messages, + "max_completion_tokens": max_tokens} + if self.temperature is not None: + payload["temperature"] = self.temperature + last: Exception | None = None + for attempt in range(self.max_retries): + req = urllib.request.Request( + f"{self.base_url}/chat/completions", + data=json.dumps(payload).encode(), + headers={"Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout=180) as resp: + body = json.load(resp) + self.calls += 1 + usage = body.get("usage", {}) + ti = int(usage.get("prompt_tokens", 0)) + to = int(usage.get("completion_tokens", 0)) + self.tokens_in += ti + self.tokens_out += to + return body["choices"][0]["message"]["content"] or "", ti, to + except urllib.error.HTTPError as exc: + detail = exc.read()[:300].decode("utf8", "ignore") + last = RuntimeError(f"HTTP {exc.code}: {detail}") + if exc.code in (429, 500, 502, 503, 529): + time.sleep(2 ** attempt) + continue + raise last + except Exception as exc: + last = exc + time.sleep(2 ** attempt) + raise RuntimeError(f"chat failed after {self.max_retries} attempts: {last}") + + @staticmethod + def _extract_json(text: str) -> dict[str, Any]: + text = text.strip() + fence = re.search(r"```(?:json)?\s*(.*?)```", text, re.S) + if fence: + text = fence.group(1).strip() + start = text.find("{") + if start < 0: + raise ValueError(f"no JSON object in reply: {text[:200]!r}") + depth, end = 0, None + for i, ch in enumerate(text[start:], start): + depth += (ch == "{") - (ch == "}") + if depth == 0: + end = i + 1 + break + if end is None: + raise ValueError(f"unterminated JSON in reply: {text[:200]!r}") + return json.loads(text[start:end]) + + def decompose(self, problem: str, signature: str, + capabilities: dict[str, str], depth: int) -> PlanDraft: + usable = sorted(r for r, m in capabilities.items() + if m in ("trusted", "candidate")) + user = ( + f"PROBLEM: {problem}\n" + f"REQUIRED SIGNATURE: {signature}\n" + f"RECURSION DEPTH: {depth}\n" + f"AVAILABLE CAPABILITIES ({len(usable)}):\n" + + ("\n".join(f" - {r}" for r in usable) if usable else " (none)") + ) + content, ti, to = self._chat( + [{"role": "system", "content": SYSTEM}, {"role": "user", "content": user}] + ) + data = self._extract_json(content) + steps: list[Step] = [] + for i, raw in enumerate(data.get("steps", [])): + kind = raw.get("kind") + sid = str(raw.get("id") or f"s{i}") + if kind == "capability": + steps.append(Step(id=sid, kind="capability", ref=raw.get("ref"), + args=raw.get("args") or {}, + output_schema=str(raw.get("output_schema") or "any"))) + elif kind == "decompose": + outs = tuple(raw.get("outputs") or ("y",)) + steps.append(Step(id=sid, kind="decompose", + problem=str(raw.get("problem") or problem), + outputs=outs, + output_schema=str(raw.get("output_schema") or "any"))) + else: + # An unrecognised kind is NOT silently coerced -- it is recorded + # as a validator-visible error by emitting an invalid step. + steps.append(Step(id=sid, kind="capability", + ref=str(raw.get("ref") or f""), + output_schema="")) + return PlanDraft( + Plan(id=f"plan-{abs(hash((problem, self.model))) % 10**8}", + problem=problem, steps=tuple(steps), signature=signature), + rationale=str(data.get("rationale", "")), + tokens_in=ti, tokens_out=to, raw=content, model=self.model, + ) + + +def load_env_key(name: str = "OPENAI_API_KEY") -> str | None: + if os.environ.get(name): + return os.environ[name] + path = os.path.expanduser("~/.orchestrator/.env") + if not os.path.exists(path): + return None + for line in open(path): + line = line.strip() + if line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + if k.strip() == name: + return v.strip().strip('"').strip("'") + return None diff --git a/scripts/prototypes/minikernel/review.py b/scripts/prototypes/minikernel/review.py new file mode 100644 index 00000000..91708d1e --- /dev/null +++ b/scripts/prototypes/minikernel/review.py @@ -0,0 +1,308 @@ +"""The review protocol: evidential gate + frozen concern ledger. + +Three mechanisms, each answering a measured failure mode: + + M1 SEPARATION OF DUTY. A session may not review an artifact version it + authored. Enforced on session identity, not on politeness. + + M2 FROZEN CRITERIA + CONCERN LEDGER. Acceptance criteria are frozen before + authoring. Every finding must cite a frozen criterion; one that cites + none is auto-filed `deferred`, never `open`. This is what stops scope + drift from turning a 3.6-round loop into an 8.6-round loop with a + non-terminating tail (round-1 simulation, red_team_gate.py). + + M3 EVIDENTIAL GATE. A finding may only BLOCK if it ships a reproducible + artifact -- a failing check with an observed value. A prose worry is + recorded as a `risk`: visible, attached to the artifact forever, but + non-blocking. This is what makes the loop terminate on evidence rather + than on the reviewer's imagination running out, and it is what makes + reviewer detection rate measurable (you can count artifacts). + +What the gate does NOT do is certify correctness. `P(truly clean | passed)` is +bounded by the reviewer's detection rate, and no gate policy raises it. The +kernel therefore records `residual_risk` on every pass rather than reporting +"clean". +""" + +from __future__ import annotations + +import itertools +from dataclasses import dataclass, field +from typing import Any, Callable + +from .capabilities import SeparationOfDuty +from .store import Store, content_hash + +_ids = itertools.count(1) + + +@dataclass(frozen=True) +class Criterion: + id: str + text: str + check: Callable[[Any], tuple[bool, str]] | None = None + + +@dataclass +class Finding: + id: str + criterion_id: str | None + severity: str # blocker | major | minor + summary: str + evidence: str | None # None => prose worry => risk, non-blocking + status: str = "open" # open|fixed|accepted_risk|invalid|deferred|risk + round: int = 0 + + @property + def blocking(self) -> bool: + return ( + self.status == "open" + and self.evidence is not None + and self.criterion_id is not None + and self.severity in ("blocker", "major") + ) + + +@dataclass +class ArtifactVersion: + artifact_id: str + version: int + body: str + author_session: str + + @property + def ref(self) -> str: + return f"{self.artifact_id}@v{self.version}" + + @property + def hash(self) -> str: + return content_hash(self.body) + + +@dataclass +class ReviewOutcome: + passed: bool + rounds: int + findings: list[Finding] + escalated: bool = False + residual_risk: float = 0.0 + reviewer_tokens: int = 0 + + def open_blockers(self) -> list[Finding]: + return [f for f in self.findings if f.blocking] + + def risks(self) -> list[Finding]: + return [f for f in self.findings if f.status in ("risk", "deferred")] + + +class ReviewBoard: + def __init__(self, store: Store, max_rounds: int = 6, + detection_rate_prior: float = 0.6): + self.store = store + self.max_rounds = max_rounds + self.detection_rate_prior = detection_rate_prior + self.ledger: dict[str, list[Finding]] = {} + + def freeze(self, artifact_id: str, criteria: list[Criterion]) -> None: + self.store.append_event( + "review", artifact_id, "criteria_frozen", + {"criteria": [{"id": c.id, "text": c.text} for c in criteria]}, + ) + self._frozen = getattr(self, "_frozen", {}) + self._frozen[artifact_id] = {c.id: c for c in criteria} + + def frozen(self, artifact_id: str) -> dict[str, Criterion]: + return getattr(self, "_frozen", {}).get(artifact_id, {}) + + def file(self, artifact: ArtifactVersion, reviewer_session: str, + finding: Finding) -> Finding: + """Classify a submitted finding against the frozen criteria (M2, M3).""" + frozen = self.frozen(artifact.artifact_id) + if finding.criterion_id not in frozen: + finding.status = "deferred" # out of frozen scope: cannot block + finding.criterion_id = None + elif finding.evidence is None: + finding.status = "risk" # prose worry: visible, non-blocking + self.ledger.setdefault(artifact.artifact_id, []).append(finding) + self.store.append_event( + "review", artifact.ref, "finding_filed", + {"id": finding.id, "criterion": finding.criterion_id, + "severity": finding.severity, "status": finding.status, + "has_evidence": finding.evidence is not None, + "summary": finding.summary[:200]}, + session_id=reviewer_session, + ) + return finding + + def run( + self, + artifact: ArtifactVersion, + criteria: list[Criterion], + reviewer_session: str, + revise: Callable[[ArtifactVersion, list[Finding]], ArtifactVersion], + reviewer: Callable[[ArtifactVersion, list[Criterion], int], list[Finding]] | None = None, + ) -> tuple[ArtifactVersion, ReviewOutcome]: + """Author <-> reviewer loop, bounded, over immutable artifact versions.""" + if reviewer_session == artifact.author_session: + raise SeparationOfDuty( + f"session {reviewer_session!r} authored {artifact.ref}" + ) + self.freeze(artifact.artifact_id, criteria) + reviewer = reviewer or self._default_reviewer + rounds = 0 + all_findings: list[Finding] = [] + tokens = 0 + current = artifact + while rounds < self.max_rounds: + rounds += 1 + submitted = reviewer(current, criteria, rounds) + tokens += 1200 + 400 * len(submitted) + classified = [self.file(current, reviewer_session, f) for f in submitted] + all_findings += classified + blockers = [f for f in classified if f.blocking] + if not blockers: + residual = self.detection_rate_prior + outcome = ReviewOutcome( + True, rounds, all_findings, False, + residual_risk=round(1.0 - residual, 3), reviewer_tokens=tokens, + ) + self.store.append_event( + "review", current.ref, "review_passed", + {"rounds": rounds, "risks": len(outcome.risks()), + "residual_risk": outcome.residual_risk}, + session_id=reviewer_session, + ) + return current, outcome + current = revise(current, blockers) + for f in blockers: + f.status = "fixed" + self.store.append_event( + "review", current.ref, "artifact_revised", + {"fixed": [f.id for f in blockers], "version": current.version}, + session_id=current.author_session, + ) + outcome = ReviewOutcome( + False, rounds, all_findings, escalated=True, + residual_risk=1.0, reviewer_tokens=tokens, + ) + self.store.append_event( + "review", current.ref, "review_escalated", + {"rounds": rounds, "open": len(outcome.open_blockers())}, + session_id=reviewer_session, + ) + return current, outcome + + def _default_reviewer( + self, artifact: ArtifactVersion, criteria: list[Criterion], round_: int + ) -> list[Finding]: + """Executable-criteria reviewer: run every check, report what fails. + + This is the evidential gate in its purest form -- the reviewer cannot + express a concern it cannot demonstrate. + """ + out: list[Finding] = [] + for c in criteria: + if c.check is None: + continue + ok, observed = c.check(artifact.body) + if not ok: + out.append( + Finding( + id=f"F{next(_ids)}", criterion_id=c.id, severity="blocker", + summary=f"criterion {c.id} not satisfied", + evidence=observed, round=round_, + ) + ) + return out + + +# ------------------------------------------------------------ insight pool + + +class InsightPool: + """Insights are notes with `kind='insight'`; the pool is the gate around them. + + #485 requires an independent red-team pass on both insertion and removal. + Added here: contradiction detection at insertion (two accepted insights that + disagree will otherwise both be retrieved and silently degrade every + downstream agent) and a scope tag so a run-local insight cannot leak. + """ + + def __init__(self, store: Store, board: ReviewBoard): + self.store = store + self.board = board + + def propose(self, run_id: str, node_id: str, text: str, author_session: str, + reviewer_session: str, scope: str = "run") -> tuple[bool, str]: + if reviewer_session == author_session: + raise SeparationOfDuty("insights are reviewed by a different session") + contradiction = self._contradicts(text) + if contradiction is not None: + self.store.append_event( + run_id, node_id, "insight_rejected", + {"reason": "contradicts", "conflicts_with": contradiction}, + session_id=reviewer_session, + ) + return False, f"contradicts accepted insight {contradiction}" + seq = self.store.append_note(run_id, node_id, "insight", text, + session_id=author_session, scope=scope) + self.store.append_event(run_id, node_id, "insight_accepted", + {"seq": seq, "scope": scope}, + session_id=reviewer_session) + return True, str(seq) + + _NEGATIONS = (" not ", " never ", " no ", " cannot ", " isn't ", " doesn't ") + _AUX = {"does", "did", "will", "would", "shall", "should", "must", + "have", "has", "had", "been", "being", "were", "was"} + + @staticmethod + def _stem(w: str) -> str: + for suf in ("ing", "ies", "ed", "es", "s"): + if w.endswith(suf) and len(w) - len(suf) >= 4: + return w[: -len(suf)] + return w + + def _contradicts(self, text: str) -> str | None: + """Cheap polarity check over shared content words. + + Deliberately crude: the point of the experiment is whether the pool + NEEDS contradiction detection, not whether this detector is good. + + FIX (found by scenario S9): the first version compared raw surface + forms, so "drifts" and "does not drift" shared only half their tokens + and the contradiction sailed through. Two accepted insights that + disagree are worse than no insight at all -- both get retrieved, and + every downstream agent quietly averages them. + """ + def core(s: str) -> set[str]: + low = f" {s.lower()} " + for n in self._NEGATIONS: + low = low.replace(n, " ") + return {self._stem(w) for w in low.split() + if len(w) > 3 and w not in self._AUX} + + def polarity(s: str) -> bool: + return any(n in f" {s.lower()} " for n in self._NEGATIONS) + + mine, mypol = core(text), polarity(text) + for row in self.store.view_insights(): + body = self.store.get_blob(row["body_hash"]) or "" + theirs = core(body) + if not theirs: + continue + overlap = len(mine & theirs) / max(1, len(mine | theirs)) + if overlap > 0.6 and polarity(body) != mypol: + return f"note:{row['seq']}" + return None + + def retract(self, seq: int, author_session: str, reviewer_session: str, + reason: str) -> bool: + if reviewer_session == author_session: + raise SeparationOfDuty("retractions are reviewed by a different session") + self.store._db.execute( + "UPDATE notes SET status='retracted' WHERE seq=?", (seq,) + ) + self.store._db.commit() + self.store.append_event("registry", f"note:{seq}", "insight_retracted", + {"reason": reason}, session_id=reviewer_session) + return True diff --git a/scripts/prototypes/minikernel/runtime.py b/scripts/prototypes/minikernel/runtime.py new file mode 100644 index 00000000..9356773b --- /dev/null +++ b/scripts/prototypes/minikernel/runtime.py @@ -0,0 +1,561 @@ +"""The durable executor. + +Everything the runtime knows is derived from the event log. There is no +in-memory tree of live agent objects: a logical node may be executed by many +sessions over its lifetime (retry, resume, model fallback, context rollover), +and one session may execute many nodes. `node == agent instance`, as #485 +phrases it, is too rigid to survive a crash. + +The four properties this module exists to demonstrate: + + P1 Durable nested runs. A child plan gets its own run_id with a recorded + parent_run_id -- not an Orchestrator constructed inside a tool. + P2 Resume. Kill the process mid-run; replay the log; completed work is + skipped and the projection of the resumed run equals the projection of an + uninterrupted one. + P3 Loud exhaustion. A node that runs out of budget escalates to its parent + with partial results and an explicit status. It never returns a green + checkmark over truncated work. + P4 Messages at deterministic boundaries. A message addressed to a pending + node is delivered when that node starts, as a recorded event that mutates + its inputs -- not as tokens injected into a live prompt. +""" + +from __future__ import annotations + +import json +import os +import time +from dataclasses import dataclass, field +from typing import Any, Callable + +from .capabilities import CapabilityRegistry +from .ir import (Authority, Budget, Plan, PlanInvalid, Step, as_jsonable, + validate) +from .library import SolvedProblemLibrary +from .planner import PlanDraft, Planner +from .review import ArtifactVersion, Criterion, ReviewBoard +from .store import Store, content_hash + +TERMINAL = ("completed", "failed", "budget_exhausted", "escalated", "blocked") + + +@dataclass +class Message: + id: str + to_node: str + kind: str # instruction|observation|question|cancel|scope_change + payload: dict[str, Any] + hops: int = 0 + visited: tuple[str, ...] = () + max_hops: int = 8 + + +class MessageBus: + """Addressed transport; the org tree carries AUTHORITY, not packets. + + Tree-only routing does deliver (simulation: >94% even at 1M nodes) but it + burns a routing decision per hop -- 7.6 model calls instead of 1 at b=10, + d=4 -- and every relay hop is a scope-leak opportunity. So the tree decides + who may commit what; the bus decides who receives what. + """ + + def __init__(self, store: Store): + self.store = store + self.inbox: dict[str, list[Message]] = {} + self.dead_letters: list[Message] = [] + + def send(self, run_id: str, sender: str, msg: Message) -> None: + if msg.hops >= msg.max_hops or msg.to_node in msg.visited: + self.dead_letters.append(msg) + self.store.append_event(run_id, sender, "message_dead_lettered", + {"id": msg.id, "to": msg.to_node, + "hops": msg.hops}) + return + self.inbox.setdefault(msg.to_node, []).append(msg) + self.store.append_event(run_id, sender, "message_sent", + {"id": msg.id, "to": msg.to_node, "kind": msg.kind}) + + def take(self, node_key: str) -> list[Message]: + return self.inbox.pop(node_key, []) + + +@dataclass +class NodeResult: + status: str + value: Any = None + partial: bool = False + reason: str = "" + tokens: int = 0 + + +@dataclass +class RunStats: + nodes: int = 0 + capability_calls: int = 0 + planner_calls: int = 0 + library_hits: int = 0 + library_misses: int = 0 + reviews: int = 0 + review_rounds: int = 0 + escalations: int = 0 + validator_rejections: int = 0 + dead_end_hits: int = 0 + demoted: int = 0 + declared_ambiguous: int = 0 + replans: int = 0 + tokens: int = 0 + max_depth: int = 0 + fan_outs: list[int] = field(default_factory=list) + ambiguous: int = 0 + emitted: int = 0 + + @property + def f_measured(self) -> float: + return self.ambiguous / self.emitted if self.emitted else 0.0 + + @property + def b_measured(self) -> float: + return sum(self.fan_outs) / len(self.fan_outs) if self.fan_outs else 0.0 + + @property + def m_measured(self) -> float: + """Offspring mean = E[ambiguous children per decomposition]. + + NOT mean(b) * mean(f): b and f are correlated across problems, and the + product of the means gave the wrong regime for one of the real + planners measured (subcritical 1.02 vs the correct 0.88). + """ + return self.ambiguous / len(self.fan_outs) if self.fan_outs else 0.0 + + @property + def m_declared(self) -> float: + """What the planner CLAIMED, before admission control demoted anything.""" + return (self.declared_ambiguous / len(self.fan_outs) + if self.fan_outs else 0.0) + + +class Crash(BaseException): + """Raised by the crash hook. Deliberately not an Exception subclass so no + `except Exception` in the executor can accidentally swallow it.""" + + +class Runtime: + def __init__( + self, + store: Store, + registry: CapabilityRegistry, + library: SolvedProblemLibrary, + planner: Planner, + board: ReviewBoard | None = None, + max_depth: int = 6, + review_plans: bool = True, + session_prefix: str = "sess", + admission: "Callable[[Step, str], tuple[bool, str]] | None" = None, + ): + self.store = store + self.registry = registry + self.library = library + self.planner = planner + self.board = board or ReviewBoard(store) + self.max_depth = max_depth + self.review_plans = review_plans + self.bus = MessageBus(store) + self.session_prefix = session_prefix + self._session_n = 0 + self.stats = RunStats() + self.crash_at: str | None = None + # ADMISSION CONTROL. Measured on real planners (probe_optimism.py): + # 35-44% of steps a planner marks "atomic" cannot in fact be done by + # the capability it names. Declared `f` therefore understates the + # branching mean by more than 10x on the best planner tested -- its + # apparent m of 0.12 was really 1.62. "Atomic" has to be a claim that + # is checked, not a label the author gets to assign. A step that fails + # admission is demoted to `decompose` rather than executed. + self.admission = admission + + # ---------------------------------------------------------------- helpers + + def new_session(self, role: str) -> str: + self._session_n += 1 + return f"{self.session_prefix}-{role}-{self._session_n}" + + def completed_nodes(self, root_run_id: str) -> dict[str, Any]: + """Projection: which (run, node) pairs already finished, and with what.""" + done: dict[str, Any] = {} + for ev in self.store.events(root_run_id): + if ev["type"] == "node_completed": + done[f"{ev['run_id']}/{ev['node_id']}"] = json.loads(ev["payload"]) + return done + + def projection(self, root_run_id: str) -> dict[str, Any]: + """The whole run, rebuilt from events. Two runs are equal iff these are.""" + tree: dict[str, Any] = {} + results: dict[str, Any] = {} + statuses: dict[str, str] = {} + for ev in self.store.events(root_run_id): + t, payload = ev["type"], json.loads(ev["payload"]) + key = f"{ev['run_id']}/{ev['node_id']}" + if t == "run_created": + tree[ev["run_id"]] = {"parent": ev["parent_run_id"], + "problem": payload.get("problem")} + elif t == "node_completed": + results[key] = payload.get("value") + statuses[key] = payload.get("status", "completed") + elif t in ("node_budget_exhausted", "node_failed", "node_escalated"): + statuses[key] = t.replace("node_", "") + return {"tree": tree, "results": results, "statuses": statuses} + + # --------------------------------------------------------------- execution + + def run( + self, + problem: str, + budget: Budget, + authority: Authority, + run_id: str = "root", + signature: str = "any->any", + resume: bool = False, + ) -> NodeResult: + skip = self.completed_nodes(run_id) if resume else {} + if resume: + self.store.append_event(run_id, run_id, "run_resumed", + {"completed": len(skip)}) + return self._solve(problem, signature, budget, authority, run_id, + parent_run_id=None, depth=0, skip=skip) + + def _solve(self, problem: str, signature: str, budget: Budget, + authority: Authority, run_id: str, parent_run_id: str | None, + depth: int, skip: dict[str, Any]) -> NodeResult: + self.stats.max_depth = max(self.stats.max_depth, depth) + self.store.append_event(run_id, run_id, "run_created", + {"problem": problem, "depth": depth, + "budget": budget.as_dict()}, + parent_run_id=parent_run_id) + allowance = self.max_depth - depth + if depth > self.max_depth: + return self._escalate(run_id, run_id, "depth cap reached", budget) + if budget.exhausted(): + return self._exhausted(run_id, run_id, budget, None) + + # 0. Has this already been shown to be out of reach at this allowance? + # Re-deriving a known dead end is the most expensive way to learn + # nothing (scenario S7). + dead = self.library.lookup_intractable(problem, signature, allowance) + if dead is not None: + self.stats.dead_end_hits += 1 + return self._escalate( + run_id, run_id, + f"known dead end after {dead.attempts} attempt(s): {dead.reason}", + budget) + + # 1. Has this been solved before? (the termination mechanism) + hit = self.library.lookup(problem, signature) + if hit is not None: + self.stats.library_hits += 1 + plan = _plan_from_json(hit.plan_json) + self.store.append_event(run_id, run_id, "plan_reused", + {"key": hit.key, + "reliability": round(hit.reliability, 3)}) + else: + self.stats.library_misses += 1 + plan = self._author_plan(problem, signature, authority, run_id, + budget, depth) + if plan is None: + return self._escalate(run_id, run_id, + "no valid plan after replanning", budget) + + result = self._execute_plan(plan, budget, authority, run_id, depth, skip) + + if result.status in ("escalated", "budget_exhausted") and depth > 0: + self.library.record_intractable(problem, signature, result.reason, + allowance) + if hit is not None: + self.library.record_use(hit.key, result.status) + elif result.status == "completed": + self.library.publish(problem, signature, plan, + {"nodes": self.stats.nodes, + "tokens": budget.spent_tokens}, + provenance=run_id) + return result + + def _author_plan(self, problem: str, signature: str, authority: Authority, + run_id: str, budget: Budget, depth: int) -> Plan | None: + """Plan -> admit -> validate -> review. In that order. + + Admission runs BEFORE the branching statistics are recorded, because + the whole point of the measurement is that the planner's own count of + ambiguous steps is not the quantity that governs termination. + """ + author = self.new_session("planner") + for attempt in range(3): + draft = self.planner.decompose(problem, signature, + self.registry.maturities(), depth) + self.stats.planner_calls += 1 + self.stats.tokens += draft.tokens_in + draft.tokens_out + budget.spend(tokens=draft.tokens_in + draft.tokens_out) + declared_ambiguous = draft.n_ambiguous + + if self.admission is not None: + draft = PlanDraft( + Plan(draft.plan.id, draft.plan.problem, + tuple(self._admit(st, run_id) for st in draft.plan.steps), + draft.plan.output_schema, draft.plan.signature), + draft.rationale, draft.tokens_in, draft.tokens_out, + draft.raw, draft.model) + + self.stats.fan_outs.append(draft.fan_out) + self.stats.emitted += draft.fan_out + self.stats.ambiguous += draft.n_ambiguous + self.stats.declared_ambiguous += declared_ambiguous + self.store.append_event( + run_id, run_id, "plan_drafted", + {"attempt": attempt, "steps": draft.fan_out, + "ambiguous": draft.n_ambiguous, + "declared_ambiguous": declared_ambiguous, + "demoted": draft.n_ambiguous - declared_ambiguous, + "f": round(draft.f, 3), "model": draft.model, + "rationale": draft.rationale[:300], + "plan": as_jsonable(draft.plan)}, + session_id=author, + ) + errors = validate(draft.plan, self.registry.maturities(), authority) + if not errors: + if self.review_plans: + self._review_plan(draft, run_id, author) + return draft.plan + self.stats.validator_rejections += 1 + self.stats.replans += 1 + self.store.append_event(run_id, run_id, "plan_rejected", + {"attempt": attempt, "errors": errors}, + session_id=author) + return None + + def _admit(self, step: Step, run_id: str) -> Step: + """Verify an `atomic` claim before the runtime acts on it.""" + if step.kind != "capability" or self.admission is None: + return step + ok, why = self.admission(step, run_id) + if ok: + return step + self.stats.demoted += 1 + self.store.append_event( + run_id, step.id, "atomicity_rejected", + {"capability": step.ref, "reason": why, + "declared_output": step.output_schema}) + return Step( + id=step.id, kind="decompose", + problem=f"{why} (was claimed atomic via {step.ref})", + outputs=("y",), output_schema=step.output_schema or "any", + authority=step.authority, + ) + + def _review_plan(self, draft: PlanDraft, run_id: str, author: str) -> None: + """Evidential review of the PLAN, before a single token is spent on it.""" + reviewer = self.new_session("reviewer") + artifact = ArtifactVersion(f"plan:{run_id}", 1, + json.dumps(as_jsonable(draft.plan), + sort_keys=True), author) + maturities = self.registry.maturities() + + def bounded(body: str) -> tuple[bool, str]: + p = json.loads(body) + bad = [s["id"] for s in p["steps"] + if s["kind"] == "loop" and not s.get("max_iterations")] + return (not bad, f"unbounded loops: {bad}") + + def known(body: str) -> tuple[bool, str]: + p = json.loads(body) + bad = [s["ref"] for s in p["steps"] + if s["kind"] == "capability" and s["ref"] not in maturities] + return (not bad, f"unknown capabilities: {bad}") + + def sized(body: str) -> tuple[bool, str]: + p = json.loads(body) + return (len(p["steps"]) <= 10, f"{len(p['steps'])} steps") + + criteria = [ + Criterion("C1", "every loop has a static bound", bounded), + Criterion("C2", "every capability referenced exists", known), + Criterion("C3", "fan-out is at most 10 steps", sized), + ] + _, outcome = self.board.run( + artifact, criteria, reviewer, + revise=lambda a, fs: ArtifactVersion(a.artifact_id, a.version + 1, + a.body, a.author_session), + ) + self.stats.reviews += 1 + self.stats.review_rounds += outcome.rounds + self.stats.tokens += outcome.reviewer_tokens + + def _execute_plan(self, plan: Plan, budget: Budget, authority: Authority, + run_id: str, depth: int, skip: dict[str, Any]) -> NodeResult: + last: NodeResult = NodeResult("completed", None) + partial = False + unresolved: list[str] = [] + for step in plan.steps: + key = f"{run_id}/{step.id}" + if key in skip: + last = NodeResult(skip[key].get("status", "completed"), + skip[key].get("value")) + continue + for msg in self.bus.take(key): + self.store.append_event(run_id, step.id, "message_delivered", + {"id": msg.id, "kind": msg.kind, + "payload": msg.payload}) + if msg.kind == "cancel": + return self._escalate(run_id, step.id, "cancelled by message", + budget) + if msg.kind == "scope_change" and "args" in msg.payload: + step = Step(**{**step.__dict__, + "args": {**step.args, **msg.payload["args"]}}) + if self.crash_at == key: + self.store.append_event(run_id, step.id, "node_started", + {"crash": True}) + raise Crash(key) + if budget.exhausted(): + self._exhausted(run_id, step.id, budget, last.value) + return NodeResult("budget_exhausted", last.value, partial=True, + reason=f"budget spent at step {step.id}") + self.stats.nodes += 1 + budget.spend(nodes=1) + self.store.append_event(run_id, step.id, "node_started", + {"kind": step.kind}) + res = self._execute_step(step, budget, authority, run_id, depth, skip) + partial = partial or res.partial + self.store.append_event( + run_id, step.id, + "node_completed" if res.status == "completed" else f"node_{res.status}", + {"status": res.status, "value": res.value, "reason": res.reason}, + ) + if res.status == "failed": + return res + if res.status == "budget_exhausted": + return NodeResult("budget_exhausted", last.value, partial=True, + reason=res.reason) + if res.status == "escalated": + # FIX (scenario S7): the first version returned here, so one + # unreachable subtree cancelled every sibling that could still + # have been solved -- and with them everything the run would + # have learned. Harvest what is reachable, then escalate once, + # naming what is unresolved. + unresolved.append(step.id) + partial = True + continue + last = res + if unresolved: + return NodeResult("escalated", last.value, partial=True, + reason=f"unresolved steps: {unresolved}") + return NodeResult("completed", last.value, partial=partial) + + def _execute_step(self, step: Step, budget: Budget, authority: Authority, + run_id: str, depth: int, skip: dict[str, Any]) -> NodeResult: + node_authority = authority.narrow(step.authority) if ( + step.authority != Authority()) else authority + if step.kind == "capability": + self.stats.capability_calls += 1 + try: + value = self.registry.invoke(step.ref, step.args, run_id, step.id, + self.new_session("worker"), + node_authority) + except (PermissionError, KeyError, RuntimeError) as exc: + return NodeResult("failed", None, reason=str(exc)) + budget.spend(tokens=200, usd=0.0002) + return NodeResult("completed", value) + if step.kind == "decompose": + child_budget = budget.child_share(1) + child_run = f"{run_id}/{step.id}" + res = self._solve(step.problem or "", f"any->{step.output_schema}", + child_budget, node_authority, child_run, run_id, + depth + 1, skip) + budget.absorb(child_budget) + if res.status == "budget_exhausted": + # P3: the child ran out. The PARENT decides -- top up from the + # reserve once, then escalate. Never silently accept partial. + moved = child_budget.top_up(budget, int(budget.remaining().tokens * 0.5)) + self.store.append_event(run_id, step.id, "budget_reallocated", + {"moved_tokens": moved, + "child": child_run}) + self.stats.escalations += 1 + if moved <= 0: + return NodeResult("budget_exhausted", res.value, partial=True, + reason="child exhausted, no reserve left") + res = self._solve(step.problem or "", f"any->{step.output_schema}", + child_budget, node_authority, child_run, run_id, + depth + 1, self.completed_nodes(child_run)) + budget.absorb(child_budget) + return res + if step.kind == "branch": + arm = step.then if _truthy(step.guard) else step.otherwise + return self._execute_plan(Plan(f"{step.id}-arm", "", tuple(arm)), + budget, node_authority, run_id, depth, skip) + if step.kind == "loop": + out: Any = None + for i in range(step.max_iterations or 0): + if budget.exhausted(): + return NodeResult("budget_exhausted", out, partial=True, + reason=f"loop {step.id} at iteration {i}") + r = self._execute_plan(Plan(f"{step.id}-body-{i}", "", + tuple(step.then)), budget, + node_authority, run_id, depth, skip) + out = r.value + if r.status != "completed": + return r + return NodeResult("completed", out) + if step.kind == "parallel": + values = [] + for i, branch in enumerate(step.branches): + r = self._execute_plan(Plan(f"{step.id}-b{i}", "", tuple(branch)), + budget, node_authority, run_id, depth, skip) + if r.status != "completed": + return r + values.append(r.value) + return NodeResult("completed", values) + if step.kind == "return": + return NodeResult("completed", step.args.get("value")) + if step.kind == "fail": + return NodeResult("failed", None, reason=str(step.args.get("reason", ""))) + return NodeResult("failed", None, reason=f"unknown step kind {step.kind!r}") + + # ------------------------------------------------------------- terminals + + def _exhausted(self, run_id: str, node_id: str, budget: Budget, + partial: Any) -> NodeResult: + self.store.append_event(run_id, node_id, "node_budget_exhausted", + {"budget": budget.as_dict(), "partial": partial}) + return NodeResult("budget_exhausted", partial, partial=True, + reason="budget exhausted") + + def _escalate(self, run_id: str, node_id: str, reason: str, + budget: Budget) -> NodeResult: + self.stats.escalations += 1 + self.store.append_event(run_id, node_id, "node_escalated", + {"reason": reason, "budget": budget.as_dict()}) + return NodeResult("escalated", None, partial=True, reason=reason) + + +def _truthy(guard: str | None) -> bool: + """Guards are evaluated fail-closed: anything not understood is False.""" + if not guard: + return False + return guard.strip().lower() in ("true", "1", "yes") + + +def _plan_from_json(text: str) -> Plan: + d = json.loads(text) + + def step(s: dict[str, Any]) -> Step: + return Step( + id=s["id"], kind=s["kind"], ref=s.get("ref"), args=s.get("args") or {}, + problem=s.get("problem"), inputs=tuple(s.get("inputs") or ()), + outputs=tuple(s.get("outputs") or ()), guard=s.get("guard"), + then=tuple(step(x) for x in s.get("then") or ()), + otherwise=tuple(step(x) for x in s.get("otherwise") or ()), + branches=tuple(tuple(step(x) for x in b) + for b in s.get("branches") or ()), + max_iterations=s.get("max_iterations"), + output_schema=s.get("output_schema", "any"), + ) + + return Plan(d["id"], d["problem"], tuple(step(s) for s in d["steps"]), + d.get("output_schema", "any"), d.get("signature", "any->any")) diff --git a/scripts/prototypes/minikernel/store.py b/scripts/prototypes/minikernel/store.py new file mode 100644 index 00000000..fff99ae7 --- /dev/null +++ b/scripts/prototypes/minikernel/store.py @@ -0,0 +1,514 @@ +"""One memory substrate, four views. + +#485 describes four stores: the shared scratchpad, the insight pool, the +context/"inode" tables, and the tool-call history. They need identical +primitives -- content-addressed immutable blobs, an append-only log with +monotonic sequence numbers, a summary tree, and full-text retrieval -- so this +module builds ONE substrate and exposes the four as queries over it. + +Design claims under test here: + +* C1 Appends never block reads. The #485 protocol ("grab the lock, read the + tail, *consider whether it alters your plans*, write, release") puts an + LLM call inside a critical section. Here the lock guards an INSERT only. +* C2 Nothing is ever re-summarised. Segments are sealed at a fixed token + boundary and summarised exactly once, so summary cost is linear in log + length rather than quadratic. +* C3 A summary is never the only copy. Every summary records the sequence + range and content hashes it covers, so it can always be exchanged for + its source. +* C4 Every context window an agent receives is accompanied by a manifest + naming every artifact and range that went into it. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import sqlite3 +import threading +import time +from dataclasses import dataclass, field +from typing import Any, Callable, Iterable, Sequence + +# Token accounting. Nothing here needs a real tokeniser to be useful, but the +# estimate must be conservative (over- rather than under-count) so that a +# compiled window never overflows the model it was compiled for. +CHARS_PER_TOKEN = 3.5 + + +def n_tokens(text: str) -> int: + return max(1, int(len(text) / CHARS_PER_TOKEN) + 1) + + +def content_hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest()[:32] + + +SCHEMA = """ +PRAGMA journal_mode=WAL; + +CREATE TABLE IF NOT EXISTS blobs( + hash TEXT PRIMARY KEY, + media_type TEXT NOT NULL, + n_tokens INTEGER NOT NULL, + body TEXT NOT NULL, + created_at REAL NOT NULL +); + +-- Append-only. The source of truth. Everything else in the system is a +-- projection of this table. +CREATE TABLE IF NOT EXISTS events( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL, + parent_run_id TEXT, + node_id TEXT NOT NULL, + seq INTEGER NOT NULL, + type TEXT NOT NULL, + session_id TEXT, + payload TEXT NOT NULL, + created_at REAL NOT NULL, + UNIQUE(run_id, seq) +); +CREATE INDEX IF NOT EXISTS idx_events_run ON events(run_id, seq); +CREATE INDEX IF NOT EXISTS idx_events_parent ON events(parent_run_id); +CREATE INDEX IF NOT EXISTS idx_events_type ON events(type); + +-- The scratchpad, as an append-only journal. `kind` is the externalised +-- operational record (#485 wants "all thinking"; this stores the parts another +-- agent can act on, not raw chain-of-thought). +CREATE TABLE IF NOT EXISTS notes( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL, + node_id TEXT NOT NULL, + session_id TEXT, + kind TEXT NOT NULL, + body_hash TEXT NOT NULL, + n_tokens INTEGER NOT NULL, + scope TEXT NOT NULL DEFAULT 'run', + status TEXT NOT NULL DEFAULT 'accepted', + created_at REAL NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_notes_kind ON notes(kind, status); + +-- Sealed segments and the summary DAG above them. level 0 segments cover raw +-- note ranges; level n>0 segments cover level n-1 segments. +CREATE TABLE IF NOT EXISTS segments( + id INTEGER PRIMARY KEY AUTOINCREMENT, + level INTEGER NOT NULL, + lo INTEGER NOT NULL, + hi INTEGER NOT NULL, + covers TEXT NOT NULL, + summary_hash TEXT NOT NULL, + n_tokens INTEGER NOT NULL, + created_at REAL NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_segments_level ON segments(level, lo); + +CREATE VIRTUAL TABLE IF NOT EXISTS note_fts USING fts5( + body, kind UNINDEXED, seq UNINDEXED, tokenize='porter' +); +CREATE VIRTUAL TABLE IF NOT EXISTS blob_fts USING fts5( + body, hash UNINDEXED, tokenize='porter' +); +""" + + +@dataclass +class WindowEntry: + """One line of a context manifest -- what was supplied, and where from.""" + + lane: str + ref: str + tokens: int + body: str + + +@dataclass +class ContextWindow: + entries: list[WindowEntry] = field(default_factory=list) + dropped: list[str] = field(default_factory=list) + + @property + def tokens(self) -> int: + return sum(e.tokens for e in self.entries) + + def text(self) -> str: + return "\n\n".join(f"[{e.lane}:{e.ref}]\n{e.body}" for e in self.entries) + + def manifest(self) -> list[dict[str, Any]]: + return [ + {"lane": e.lane, "ref": e.ref, "tokens": e.tokens} for e in self.entries + ] + + +class Store: + def __init__(self, path: str, segment_tokens: int = 2000, fanout: int = 8): + self.path = path + self.segment_tokens = segment_tokens + self.fanout = fanout + self._db = sqlite3.connect(path, check_same_thread=False, timeout=30.0) + self._db.row_factory = sqlite3.Row + self._db.executescript(SCHEMA) + self._db.commit() + # Guards writes only. Readers never take it -- claim C1. + self._wlock = threading.Lock() + + def close(self) -> None: + self._db.close() + + # ---------------------------------------------------------------- blobs + + def put_blob(self, body: str, media_type: str = "text/plain") -> str: + h = content_hash(body) + with self._wlock: + self._db.execute( + "INSERT OR IGNORE INTO blobs(hash, media_type, n_tokens, body," + " created_at) VALUES(?,?,?,?,?)", + (h, media_type, n_tokens(body), body, time.time()), + ) + self._db.execute( + "INSERT INTO blob_fts(body, hash) VALUES(?,?)", (body, h) + ) + self._db.commit() + return h + + def get_blob(self, h: str) -> str | None: + row = self._db.execute("SELECT body FROM blobs WHERE hash=?", (h,)).fetchone() + return row["body"] if row else None + + # --------------------------------------------------------------- events + + def append_event( + self, + run_id: str, + node_id: str, + type_: str, + payload: dict[str, Any] | None = None, + parent_run_id: str | None = None, + session_id: str | None = None, + ) -> int: + body = json.dumps(payload or {}, sort_keys=True, default=str) + with self._wlock: + seq = self._db.execute( + "SELECT COALESCE(MAX(seq), 0) + 1 FROM events WHERE run_id=?", + (run_id,), + ).fetchone()[0] + self._db.execute( + "INSERT INTO events(run_id, parent_run_id, node_id, seq, type," + " session_id, payload, created_at) VALUES(?,?,?,?,?,?,?,?)", + (run_id, parent_run_id, node_id, seq, type_, session_id, body, + time.time()), + ) + self._db.commit() + return seq + + def events(self, root_run_id: str, descendants: bool = True) -> list[sqlite3.Row]: + if not descendants: + return list( + self._db.execute( + "SELECT * FROM events WHERE run_id=? ORDER BY seq", (root_run_id,) + ) + ) + return list( + self._db.execute( + "WITH RECURSIVE sub(r) AS (" + " SELECT ?" + " UNION" + " SELECT e.run_id FROM events e JOIN sub s ON e.parent_run_id = s.r" + ") SELECT e.* FROM events e JOIN sub ON e.run_id = sub.r" + " ORDER BY e.run_id, e.seq", + (root_run_id,), + ) + ) + + # ---------------------------------------------------- journal (scratchpad) + + def append_note( + self, + run_id: str, + node_id: str, + kind: str, + body: str, + session_id: str | None = None, + scope: str = "run", + status: str = "accepted", + ) -> int: + """Append to the shared journal. O(1), no deliberation under lock.""" + h = self.put_blob(body, "text/note") + with self._wlock: + cur = self._db.execute( + "INSERT INTO notes(run_id, node_id, session_id, kind, body_hash," + " n_tokens, scope, status, created_at) VALUES(?,?,?,?,?,?,?,?,?)", + (run_id, node_id, session_id, kind, h, n_tokens(body), scope, + status, time.time()), + ) + seq = cur.lastrowid + self._db.execute( + "INSERT INTO note_fts(body, kind, seq) VALUES(?,?,?)", (body, kind, seq) + ) + self._db.commit() + return int(seq) + + def note(self, seq: int) -> sqlite3.Row | None: + return self._db.execute("SELECT * FROM notes WHERE seq=?", (seq,)).fetchone() + + def note_body(self, seq: int) -> str: + row = self.note(seq) + return self.get_blob(row["body_hash"]) or "" if row else "" + + def notes_tail(self, budget_tokens: int) -> list[sqlite3.Row]: + """Most recent notes that fit, newest-first walk, returned oldest-first.""" + out: list[sqlite3.Row] = [] + used = 0 + for row in self._db.execute("SELECT * FROM notes ORDER BY seq DESC"): + if used + row["n_tokens"] > budget_tokens: + break + out.append(row) + used += row["n_tokens"] + return list(reversed(out)) + + # ----------------------------------------------------------- summary DAG + + def unsealed_span(self) -> tuple[int, int, int]: + """(lo, hi, tokens) of notes not yet covered by a level-0 segment.""" + row = self._db.execute( + "SELECT COALESCE(MAX(hi), 0) FROM segments WHERE level=0" + ).fetchone() + lo = int(row[0]) + 1 + agg = self._db.execute( + "SELECT COALESCE(MAX(seq),0), COALESCE(SUM(n_tokens),0) FROM notes" + " WHERE seq >= ?", + (lo,), + ).fetchone() + return lo, int(agg[0]), int(agg[1]) + + def seal(self, summarize: Callable[[str, int], str]) -> list[int]: + """Seal every complete segment. Each range is summarised exactly once. + + `summarize(text, level)` returns the summary. Sealing is idempotent: + calling it twice with no new notes does nothing (claim C2). + """ + created: list[int] = [] + # level 0: raw notes -> segment summaries + while True: + lo, hi, tokens = self.unsealed_span() + if tokens < self.segment_tokens or hi < lo: + break + span_lo, acc, span_hi = lo, 0, lo - 1 + for row in self._db.execute( + "SELECT seq, n_tokens FROM notes WHERE seq >= ? ORDER BY seq", (lo,) + ): + acc += row["n_tokens"] + span_hi = row["seq"] + if acc >= self.segment_tokens: + break + bodies, hashes = [], [] + for row in self._db.execute( + "SELECT seq, kind, body_hash FROM notes WHERE seq BETWEEN ? AND ?" + " ORDER BY seq", + (span_lo, span_hi), + ): + hashes.append(row["body_hash"]) + bodies.append(f"({row['seq']}/{row['kind']}) {self.get_blob(row['body_hash'])}") + created.append( + self._insert_segment(0, span_lo, span_hi, hashes, + summarize("\n".join(bodies), 0)) + ) + # levels 1..n: segments -> higher summaries + level = 0 + while True: + rows = list( + self._db.execute( + "SELECT * FROM segments WHERE level=? ORDER BY lo", (level,) + ) + ) + higher = { + c + for r in self._db.execute( + "SELECT covers FROM segments WHERE level=?", (level + 1,) + ) + for c in json.loads(r["covers"]) + } + pending = [r for r in rows if str(r["id"]) not in higher] + if len(pending) < self.fanout: + break + for i in range(0, len(pending) - self.fanout + 1, self.fanout): + group = pending[i : i + self.fanout] + text = "\n".join(self.get_blob(g["summary_hash"]) or "" for g in group) + created.append( + self._insert_segment( + level + 1, + group[0]["lo"], + group[-1]["hi"], + [str(g["id"]) for g in group], + summarize(text, level + 1), + ) + ) + level += 1 + return created + + def _insert_segment( + self, level: int, lo: int, hi: int, covers: Sequence[str], summary: str + ) -> int: + h = self.put_blob(summary, "text/summary") + with self._wlock: + cur = self._db.execute( + "INSERT INTO segments(level, lo, hi, covers, summary_hash, n_tokens," + " created_at) VALUES(?,?,?,?,?,?,?)", + (level, lo, hi, json.dumps(list(covers)), h, n_tokens(summary), + time.time()), + ) + self._db.commit() + return int(cur.lastrowid) + + def top_segments(self) -> list[sqlite3.Row]: + """Highest-level segments plus any lower ones they do not cover.""" + rows = list(self._db.execute("SELECT * FROM segments")) + covered: set[str] = set() + for r in rows: + if r["level"] > 0: + covered |= set(json.loads(r["covers"])) + return sorted( + (r for r in rows if str(r["id"]) not in covered), key=lambda r: (r["lo"],) + ) + + def expand(self, segment_id: int) -> list[str]: + """Exchange a summary for what it covers -- claim C3.""" + row = self._db.execute( + "SELECT * FROM segments WHERE id=?", (segment_id,) + ).fetchone() + if row is None: + return [] + covers = json.loads(row["covers"]) + if row["level"] == 0: + return [self.get_blob(h) or "" for h in covers] + out = [] + for cid in covers: + child = self._db.execute( + "SELECT summary_hash FROM segments WHERE id=?", (int(cid),) + ).fetchone() + if child: + out.append(self.get_blob(child["summary_hash"]) or "") + return out + + # ------------------------------------------------------------ retrieval + + @staticmethod + def _fts_query(query: str) -> str: + terms = [t for t in re.findall(r"[A-Za-z0-9_]+", query) if len(t) > 2] + return " OR ".join(f'"{t}"' for t in terms[:24]) + + def search_notes(self, query: str, k: int = 8, kind: str | None = None): + q = self._fts_query(query) + if not q: + return [] + sql = ( + "SELECT n.*, bm25(note_fts) AS score FROM note_fts" + " JOIN notes n ON n.seq = note_fts.seq" + " WHERE note_fts MATCH ?" + ) + args: list[Any] = [q] + if kind: + sql += " AND n.kind = ?" + args.append(kind) + sql += " ORDER BY score LIMIT ?" + args.append(k) + try: + return list(self._db.execute(sql, args)) + except sqlite3.OperationalError: + return [] + + # ------------------------------------------------- the four #485 views + + def view_scratchpad(self, limit: int = 200): + return list( + self._db.execute("SELECT * FROM notes ORDER BY seq DESC LIMIT ?", (limit,)) + ) + + def view_insights(self): + return list( + self._db.execute( + "SELECT * FROM notes WHERE kind='insight' AND status='accepted'" + " ORDER BY seq" + ) + ) + + def view_tool_history(self, capability: str | None = None): + rows = self._db.execute( + "SELECT * FROM events WHERE type='capability_invoked' ORDER BY id" + ) + out = [dict(r) for r in rows] + if capability: + out = [r for r in out if json.loads(r["payload"]).get("capability") == capability] + return out + + def view_summary_tree(self): + return list(self._db.execute("SELECT * FROM segments ORDER BY level, lo")) + + # --------------------------------------------------- context compilation + + def compile_window( + self, + budget_tokens: int, + query: str = "", + lanes: dict[str, float] | None = None, + ) -> ContextWindow: + """Compile a context window under a per-lane token budget (claim C4). + + Lane fractions are POLICY, not constants: a deterministic leaf can ask + for {'tail': 1.0} and a planner for {'insights': .5, 'retrieved': .5}. + """ + lanes = lanes or {"tail": 0.35, "summaries": 0.25, "retrieved": 0.25, + "insights": 0.15} + win = ContextWindow() + seen: set[str] = set() + + def add(lane: str, ref: str, body: str, cap: int) -> bool: + t = n_tokens(body) + if ref in seen: + return False + if t > cap: + win.dropped.append(ref) + return False + seen.add(ref) + win.entries.append(WindowEntry(lane, ref, t, body)) + return True + + for lane, frac in lanes.items(): + cap = int(budget_tokens * frac) + used = 0 + if lane == "tail": + for row in self.notes_tail(cap): + b = self.get_blob(row["body_hash"]) or "" + if used + n_tokens(b) > cap: + break + if add(lane, f"note:{row['seq']}", b, cap - used): + used += n_tokens(b) + elif lane == "summaries": + for seg in self.top_segments(): + b = self.get_blob(seg["summary_hash"]) or "" + if used + n_tokens(b) > cap: + win.dropped.append(f"seg:{seg['id']}") + continue + if add(lane, f"seg:{seg['id']}", b, cap - used): + used += n_tokens(b) + elif lane == "retrieved" and query: + for row in self.search_notes(query, k=12): + b = self.get_blob(row["body_hash"]) or "" + if used + n_tokens(b) > cap: + break + if add(lane, f"note:{row['seq']}", b, cap - used): + used += n_tokens(b) + elif lane == "insights": + cands = self.search_notes(query, k=8, kind="insight") if query else [] + if not cands: + cands = self.view_insights()[-8:] + for row in cands: + b = self.get_blob(row["body_hash"]) or "" + if used + n_tokens(b) > cap: + break + if add(lane, f"insight:{row['seq']}", b, cap - used): + used += n_tokens(b) + return win diff --git a/scripts/prototypes/probe_optimism.py b/scripts/prototypes/probe_optimism.py new file mode 100644 index 00000000..ee7a1041 --- /dev/null +++ b/scripts/prototypes/probe_optimism.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Is a low ambiguity rate competence, or optimism? + +`measure_ambiguity.py` finds that swapping the planner model moves the +branching mean `m` by more than an order of magnitude (0.07 vs 0.88 on the same +library). That is either very good news -- a better planner really does know +how to do more with the same tools -- or the worst possible news, because the +cheapest way for a planner to drive `f` to zero is to call hard steps atomic +and let the runtime discover the lie later. + +`f` is only a safety metric if "atomic" means "this capability, alone, actually +produces this output". So this script takes each atomic step a planner emitted +and puts it in front of an INDEPENDENT judge model -- a third model that +authored neither plan, which is the separation-of-duty rule from #485 applied +to the measurement itself. + + optimism rate = atomic steps the judge says are not actually atomic + corrected f, m = recomputed counting those steps as ambiguous + + .venv/bin/python scripts/prototypes/probe_optimism.py [--refresh] +""" + +from __future__ import annotations + +import argparse +import json +import os +import statistics +import sys +from collections import defaultdict + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from measure_ambiguity import PROBLEMS, TIERS +from minikernel import LLMPlanner, load_env_key +from minikernel.ir import as_jsonable + +HERE = os.path.dirname(os.path.abspath(__file__)) +CACHE = os.path.join(HERE, "measurements", "optimism.json") + +# The problems where the two planners disagreed most, plus a control that both +# found easy. +HARD = [1, 2, 4, 5, 6, 10, 11, 13] + +JUDGE_SYSTEM = """You are an independent reviewer. You did not write the plan \ +you are shown. For EACH step marked "atomic", decide one thing only: + + Can the single named capability, on its own, produce that step's stated \ +output for this problem -- with at most trivial reformatting of its result? + +Answer NO if the step actually requires several capabilities, a judgement call \ +the capability cannot make, information the capability has no access to, or an \ +open-ended amount of work. Answer YES only if a competent engineer would agree \ +one call to that tool does it. + +Reply with ONLY JSON: {"verdicts": [{"id": "", "atomic": true|false, \ +"why": ""}]}""" + + +def judge_plan(judge: LLMPlanner, problem: str, plan, caps: list[str]) -> dict: + atomic = [s for s in plan.steps if s.kind == "capability"] + if not atomic: + return {"verdicts": []} + body = "\n".join( + f' - id={s.id} capability={s.ref} args={json.dumps(s.args)[:160]} ' + f'declared_output={s.output_schema}' + for s in atomic + ) + user = (f"PROBLEM: {problem}\n\nAVAILABLE CAPABILITIES: {', '.join(caps)}\n\n" + f"STEPS MARKED ATOMIC:\n{body}") + content, _, _ = judge._chat( + [{"role": "system", "content": JUDGE_SYSTEM}, + {"role": "user", "content": user}], max_tokens=1400) + return judge._extract_json(content) + + +def run(planner_models: list[str], judge_model: str, key: str, + base_url: str) -> list[dict]: + caps = TIERS["L2_working"] + maturities = {c: "trusted" for c in caps} + judge = LLMPlanner(judge_model, key, base_url) + rows: list[dict] = [] + for model in planner_models: + planner = LLMPlanner(model, key, base_url) + for idx in HARD: + problem = PROBLEMS[idx] + try: + draft = planner.decompose(problem, "any->report", maturities, 0) + except Exception as exc: + print(f" !! plan {model} p{idx}: {exc}") + continue + try: + verdicts = judge_plan(judge, problem, draft.plan, caps) + except Exception as exc: + print(f" !! judge {model} p{idx}: {exc}") + continue + vs = {v.get("id"): v for v in verdicts.get("verdicts", [])} + n_atomic = sum(1 for s in draft.plan.steps if s.kind == "capability") + overclaimed = sum( + 1 for s in draft.plan.steps + if s.kind == "capability" and vs.get(s.id, {}).get("atomic") is False + ) + corrected_amb = draft.n_ambiguous + overclaimed + rows.append({ + "planner": model, "judge": judge_model, "problem_index": idx, + "problem": problem[:80], "b": draft.fan_out, + "declared_ambiguous": draft.n_ambiguous, "atomic_steps": n_atomic, + "overclaimed": overclaimed, + "declared_f": round(draft.f, 3), + "corrected_f": round(corrected_amb / draft.fan_out, 3) + if draft.fan_out else 0.0, + "corrected_ambiguous": corrected_amb, + "plan": as_jsonable(draft.plan), + "verdicts": verdicts.get("verdicts", []), + }) + print(f" {model:<14} p{idx:<2d} b={draft.fan_out} " + f"declared_amb={draft.n_ambiguous} overclaimed={overclaimed}/" + f"{n_atomic} f: {draft.f:.2f} -> " + f"{corrected_amb / max(1, draft.fan_out):.2f}") + return rows + + +def report(rows: list[dict]) -> None: + by = defaultdict(list) + for r in rows: + by[r["planner"]].append(r) + print(f"\n{'='*96}") + print("OPTIMISM PROBE -- atomic steps re-judged by an independent model") + print(f"{'='*96}") + print(f"{'planner':<16} {'plans':>6} {'mean b':>7} {'declared m':>11} " + f"{'overclaim rate':>15} {'corrected m':>12} {'regime after':>15}") + print("-" * 96) + for model, rs in by.items(): + b = statistics.mean(r["b"] for r in rs) + dm = statistics.mean(r["declared_ambiguous"] for r in rs) + cm = statistics.mean(r["corrected_ambiguous"] for r in rs) + tot_atomic = sum(r["atomic_steps"] for r in rs) + over = sum(r["overclaimed"] for r in rs) + regime = ("SUPERCRITICAL" if cm > 1.05 else + "critical" if cm > 0.95 else "subcritical") + print(f"{model:<16} {len(rs):>6} {b:>7.2f} {dm:>11.2f} " + f"{over}/{tot_atomic} = {over / max(1, tot_atomic):>5.0%} " + f"{cm:>12.2f} {regime:>15}") + print("-" * 96) + print("\nexamples of steps the judge rejected as not-actually-atomic:") + shown = 0 + for r in rows: + for v in r["verdicts"]: + if v.get("atomic") is False and shown < 8: + step = next((s for s in r["plan"]["steps"] + if s["id"] == v.get("id")), None) + ref = step.get("ref") if step else "?" + print(f" [{r['planner']}] {ref:<18} {v.get('why','')[:64]}") + shown += 1 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--refresh", action="store_true") + ap.add_argument("--planners", default="gpt-5.4-mini,gpt-5.6-sol") + ap.add_argument("--judge", default="gpt-5.5") + ap.add_argument("--base-url", default="https://api.openai.com/v1") + args = ap.parse_args() + os.makedirs(os.path.dirname(CACHE), exist_ok=True) + if os.path.exists(CACHE) and not args.refresh: + rows = json.load(open(CACHE)) + print(f"(cached: {len(rows)} plans from {CACHE})") + report(rows) + return 0 + key = load_env_key("OPENAI_API_KEY") + if not key: + print("No OPENAI_API_KEY reachable; refusing to invent numbers.") + return 2 + rows = run(args.planners.split(","), args.judge, key, args.base_url) + json.dump(rows, open(CACHE, "w"), indent=1) + report(rows) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prototypes/run_scenarios.py b/scripts/prototypes/run_scenarios.py new file mode 100644 index 00000000..787dbc02 --- /dev/null +++ b/scripts/prototypes/run_scenarios.py @@ -0,0 +1,532 @@ +#!/usr/bin/env python3 +"""End-to-end scenarios against the #485 kernel. + +Nothing here is mocked. The crash scenario kills a real child process with a +real signal; the store is a real SQLite database on disk; the capabilities do +real work. Run: + + .venv/bin/python scripts/prototypes/run_scenarios.py +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from minikernel import (ArtifactVersion, Authority, BugReport, BugTracker, + Budget, Capability, CapabilityRegistry, Crash, + Criterion, Finding, InsightPool, Message, ReviewBoard, + Runtime, SeparationOfDuty, SolvedProblemLibrary, + StubPlanner, Store, Step, Plan, validate) + +PASS, FAIL = " PASS", " FAIL" +results: list[tuple[str, bool, str]] = [] + + +def check(name: str, ok: bool, detail: str = "") -> bool: + results.append((name, ok, detail)) + print(f"{PASS if ok else FAIL} {name}" + (f" -- {detail}" if detail else "")) + return ok + + +# ------------------------------------------------------------------ fixtures + +def seed_capabilities(reg: CapabilityRegistry) -> None: + reg.seed(Capability("echo", 1, lambda a: str(a.get("text", "")), "str", "str", + tests=[({"text": "x"}, "x")], source="return text")) + reg.seed(Capability("upper", 1, lambda a: str(a.get("text", "")).upper(), + "str", "str", tests=[({"text": "x"}, "X")], + source="return text.upper()")) + reg.seed(Capability("wordcount", 1, + lambda a: len(str(a.get("text", "")).split()), + "str", "int", tests=[({"text": "a b"}, 2)], + source="return len(text.split())")) + + +def fresh(tmp: str, name: str, **kw): + store = Store(os.path.join(tmp, f"{name}.db"), segment_tokens=800) + reg = CapabilityRegistry(store) + seed_capabilities(reg) + lib = SolvedProblemLibrary(store) + planner = kw.pop("planner", None) or StubPlanner(seed=7, b=4, f=0.25, + capability_pool=("echo@1",)) + rt = Runtime(store, reg, lib, planner, **kw) + return store, reg, lib, rt + + +ROOT_AUTH = Authority(net=frozenset({"example.com"}), subprocess=False, spend_usd=5.0) + + +# ----------------------------------------------------------------- scenarios + +def s1_nested_runs(tmp: str) -> None: + print("\nS1 durable nested runs (P1)") + # m = b*f = 4*0.15 = 0.6: subcritical, so the recursion is finite. + store, reg, lib, rt = fresh(tmp, "s1", max_depth=3, review_plans=False) + rt.planner = StubPlanner(seed=7, b=4, f=0.15, capability_pool=("echo@1",)) + res = rt.run("build a widget report", Budget(200_000, 5.0, 400, 600.0), + ROOT_AUTH, run_id="r1", signature="any->str") + proj = rt.projection("r1") + children = [r for r, v in proj["tree"].items() if v["parent"]] + linked = all(v["parent"] in proj["tree"] or v["parent"] is None + for v in proj["tree"].values()) + check("root run completes", res.status == "completed", res.status) + check("child runs exist with their own run_id", len(children) > 0, + f"{len(children)} child runs, depth {rt.stats.max_depth}") + check("every child records its parent_run_id", linked) + check("org tree is a projection of the log, not an object graph", + len(proj["tree"]) == len(children) + 1, + f"{len(proj['tree'])} runs rebuilt from {len(store.events('r1'))} events") + store.close() + + # The same kernel at the critical point (m = 4*0.25 = 1.0) must NOT quietly + # produce an answer -- it must run into the depth cap and say so. + store2, _, _, rt2 = fresh(tmp, "s1crit", max_depth=3, review_plans=False) + rt2.planner = StubPlanner(seed=7, b=4, f=0.25, capability_pool=("echo@1",)) + crit = rt2.run("build a widget report", Budget(200_000, 5.0, 400, 600.0), + ROOT_AUTH, run_id="rc", signature="any->str") + # m=1.0 is the critical point: a SINGLE realisation may well terminate -- + # it is the expectation that diverges. So the invariant is not "it fails", + # it is "it costs sharply more, and whatever it returns is an honest + # terminal status". + check("at m=1.0 the run still reaches an honest terminal status", + crit.status in ("completed", "escalated", "budget_exhausted"), + f"status={crit.status}, depth reached {rt2.stats.max_depth}") + store2.close() + # One realisation says nothing about a branching process. Average over + # seeds instead. + def sweep(f_: float, seeds: int = 16) -> tuple[float, float]: + total, escalated = 0, 0 + for sd in range(seeds): + st, _, _, r = fresh(tmp, f"s1sweep{f_}{sd}", max_depth=4, + review_plans=False) + r.planner = StubPlanner(seed=sd, b=4, f=f_, capability_pool=("echo@1",)) + res = r.run("build a widget report", Budget(400_000, 5.0, 4000, 600.0), + ROOT_AUTH, run_id="x", signature="any->str") + total += r.stats.nodes + escalated += res.status != "completed" + st.close() + return total / seeds, escalated / seeds + (n_sub, e_sub), (n_crit, e_crit) = sweep(0.15), sweep(0.30) + # With a depth cap in place, crossing m=1 does NOT mostly show up as cost: + # the cap truncates the tree. It shows up as ANSWERS NOT PRODUCED. + check("crossing m=1 shows up as escalation rate, not as cost (16 seeds)", + e_crit > 2 * max(e_sub, 0.01) and n_crit > n_sub, + f"m=0.6: {n_sub:.1f} nodes / {e_sub:.0%} escalated -> " + f"m=1.2: {n_crit:.1f} nodes / {e_crit:.0%} escalated") + + +def s2_crash_resume(tmp: str) -> None: + print("\nS2 crash and resume (P2)") + db = os.path.join(tmp, "s2.db") + clean_db = os.path.join(tmp, "s2_clean.db") + env = dict(os.environ, MK_DB=db, MK_CLEAN=clean_db, + PYTHONPATH=os.path.dirname(os.path.abspath(__file__))) + # 1. clean reference run in its own process + r0 = subprocess.run([sys.executable, __file__, "--child", "clean"], env=env, + capture_output=True, text=True) + # 2. a run that really dies mid-flight + r1 = subprocess.run([sys.executable, __file__, "--child", "crash"], env=env, + capture_output=True, text=True) + check("child process died with a hard exit code", r1.returncode == 137, + f"returncode={r1.returncode}") + # 3. resume in a third process + r2 = subprocess.run([sys.executable, __file__, "--child", "resume"], env=env, + capture_output=True, text=True) + if r2.returncode != 0: + check("resume ran", False, r2.stderr.strip().splitlines()[-1:] or "") + return + resumed = json.loads(r2.stdout.strip().splitlines()[-1]) + reference = json.loads(r0.stdout.strip().splitlines()[-1]) + check("resumed run reaches a terminal status", + resumed["status"] in ("completed", "budget_exhausted"), resumed["status"]) + check("resumed projection == uninterrupted projection", + resumed["projection"] == reference["projection"], + "byte-identical results/statuses/tree") + check("resume skipped already-completed work", + resumed["skipped"] > 0, f"{resumed['skipped']} nodes replayed from log") + + +def _child(mode: str) -> None: + db, clean = os.environ["MK_DB"], os.environ["MK_CLEAN"] + path = clean if mode == "clean" else db + store = Store(path, segment_tokens=800) + reg = CapabilityRegistry(store) + seed_capabilities(reg) + rt = Runtime(store, reg, SolvedProblemLibrary(store), + StubPlanner(seed=11, b=4, f=0.3, capability_pool=("echo@1",)), + max_depth=2, review_plans=False) + budget = Budget(200_000, 5.0, 400, 600.0) + if mode == "crash": + rt.crash_at = "r1/s2" + try: + rt.run("assemble the quarterly digest", budget, ROOT_AUTH, run_id="r1") + except Crash: + os._exit(137) + os._exit(0) + skipped = len(rt.completed_nodes("r1")) if mode == "resume" else 0 + res = rt.run("assemble the quarterly digest", budget, ROOT_AUTH, run_id="r1", + resume=(mode == "resume")) + print(json.dumps({"status": res.status, "skipped": skipped, + "projection": rt.projection("r1")})) + store.close() + + +def s3_budget(tmp: str) -> None: + print("\nS3 budget exhaustion is loud (P3)") + store, reg, lib, rt = fresh(tmp, "s3", max_depth=4, review_plans=False) + rt.planner = StubPlanner(seed=3, b=6, f=0.5, capability_pool=("echo@1",)) + tight = Budget(4_000, 0.05, 12, 60.0) + res = rt.run("an expensive recursive problem", tight, ROOT_AUTH, run_id="rb") + ev = [e["type"] for e in store.events("rb")] + check("tight budget does NOT report success", + res.status != "completed", res.status) + check("exhaustion is recorded as an event, not swallowed", + "node_budget_exhausted" in ev or "node_escalated" in ev, + f"{ev.count('node_budget_exhausted')} exhaustion events") + check("result is flagged partial", res.partial, res.reason) + check("parent reallocated from reserve before giving up", + "budget_reallocated" in ev, f"{ev.count('budget_reallocated')} top-ups") + # Money is not the binding constraint when m > 1. Same problem, same huge + # budget, two planners either side of the critical point. + def generous() -> Budget: + return Budget(2_000_000, 50.0, 20_000, 600.0) + + store2, _, _, rt2 = fresh(tmp, "s3b", max_depth=4, review_plans=False) + rt2.planner = StubPlanner(seed=3, b=6, f=0.5, capability_pool=("echo@1",)) + g2 = generous() + res2 = rt2.run("an expensive recursive problem", g2, ROOT_AUTH, run_id="rb", + signature="any->str") + check("supercritical (m=3.0) does NOT complete even on a huge budget", + res2.status != "completed", + f"status={res2.status}, {rt2.stats.nodes} nodes, {g2.spent_tokens} tokens") + store3, _, _, rt3 = fresh(tmp, "s3c", max_depth=4, review_plans=False) + rt3.planner = StubPlanner(seed=3, b=6, f=0.1, capability_pool=("echo@1",)) + g3 = generous() + res3 = rt3.run("an expensive recursive problem", g3, ROOT_AUTH, run_id="rb", + signature="any->str") + check("subcritical (m=0.6) completes on the same budget", + res3.status == "completed", + f"{rt3.stats.nodes} nodes, {g3.spent_tokens} tokens") + for st in (store, store2, store3): + st.close() + + +def s4_messages(tmp: str) -> None: + print("\nS4 messages at deterministic boundaries (P4)") + store, reg, lib, rt = fresh(tmp, "s4", max_depth=1, review_plans=False) + plan = Plan("p", "greet", ( + Step("s0", "capability", ref="echo@1", args={"text": "original"}, + output_schema="str"), + Step("s1", "capability", ref="upper@1", args={"text": "original"}, + output_schema="str"), + )) + rt.bus.send("rm", "root", Message("m1", "rm/s1", "scope_change", + {"args": {"text": "redirected"}})) + budget = Budget(100_000, 1.0, 50, 60.0) + store.append_event("rm", "rm", "run_created", {"problem": "greet"}) + res = rt._execute_plan(plan, budget, ROOT_AUTH, "rm", 0, {}) + delivered = [e for e in store.events("rm") if e["type"] == "message_delivered"] + check("message delivered at the step boundary", len(delivered) == 1) + check("message actually changed the node's inputs", + res.value == "REDIRECTED", str(res.value)) + check("delivery is an auditable event, not prompt injection", + json.loads(delivered[0]["payload"])["kind"] == "scope_change") + rt.bus.send("rm", "root", Message("m2", "rm/nobody", "instruction", {}, + hops=99, max_hops=8)) + check("over-budget hop count dead-letters instead of looping", + len(rt.bus.dead_letters) == 1) + store.close() + + +def s5_review(tmp: str) -> None: + print("\nS5 review protocol: separation of duty, evidence, ledger") + store, reg, lib, rt = fresh(tmp, "s5") + board = ReviewBoard(store, max_rounds=5) + author = "sess-author-1" + art = ArtifactVersion("report", 1, "the answer is 41", author) + crit = [Criterion("C1", "answer must be 42", + lambda b: ("42" in b, f"observed body: {b!r}"))] + try: + board.run(art, crit, author, revise=lambda a, f: a) + check("author cannot review own work", False, "no exception raised") + except SeparationOfDuty: + check("author cannot review own work", True) + + revisions = {"n": 0} + + def revise(a, findings): + revisions["n"] += 1 + return ArtifactVersion(a.artifact_id, a.version + 1, + "the answer is 42", "sess-author-2") + + final, outcome = board.run(art, crit, "sess-reviewer-9", revise=revise) + check("evidential finding blocks, then is fixed", + outcome.passed and revisions["n"] == 1, + f"{outcome.rounds} rounds, v{final.version}") + check("a passing review reports residual risk, not 'clean'", + outcome.residual_risk > 0, f"residual_risk={outcome.residual_risk}") + + # out-of-scope prose worry must not be able to block + art2 = ArtifactVersion("report2", 1, "the answer is 42", author) + board.freeze("report2", crit) + drifty = board.file(art2, "sess-reviewer-9", + Finding("F99", "C_NEW", "blocker", + "we should also rewrite the intro", None)) + check("finding citing no frozen criterion is auto-deferred", + drifty.status == "deferred" and not drifty.blocking, drifty.status) + prose = board.file(art2, "sess-reviewer-9", + Finding("F98", "C1", "blocker", "feels underspecified", None)) + check("in-scope finding with no evidence becomes a non-blocking risk", + prose.status == "risk" and not prose.blocking, prose.status) + store.close() + + +def s6_capabilities(tmp: str) -> None: + print("\nS6 capability lifecycle") + store, reg, lib, rt = fresh(tmp, "s6") + broken = Capability("flaky", 1, lambda a: 1 / 0 if a.get("boom") else "ok", + "str", "str", tests=[({"boom": False}, "ok")], + author_session="sess-author-3", source="1/0") + reg.register(broken) + try: + reg.invoke("flaky@1", {}, "r", "n", "s", ROOT_AUTH) + check("draft capability cannot be invoked", False) + except PermissionError: + check("draft capability cannot be invoked", True) + try: + reg.qualify("flaky@1", "sess-author-3") + check("author cannot qualify own capability", False) + except SeparationOfDuty: + check("author cannot qualify own capability", True) + ok, failures = reg.qualify("flaky@1", "sess-reviewer-4") + check("independent qualification RUNS the tests and promotes", + ok and reg.get("flaky@1").maturity == "trusted", str(failures)) + + privileged = Capability("fetch", 1, lambda a: "data", "str", "str", + authority=Authority(net=frozenset({"evil.test"})), + tests=[({}, "data")], author_session="sess-author-5") + reg.register(privileged) + reg.qualify("fetch@1", "sess-reviewer-4") + try: + reg.invoke("fetch@1", {}, "r", "n", "s", ROOT_AUTH) + check("capability cannot exceed caller authority", False) + except PermissionError as exc: + check("capability cannot exceed caller authority", True, str(exc)[:60]) + + tracker = BugTracker(store, reg) + for _ in range(5): + try: + reg.invoke("flaky@1", {"boom": True}, "r", "n", "s", ROOT_AUTH) + except RuntimeError: + pass + rid = tracker.file(BugReport("B1", "flaky@1", "call with boom", + "returns ok", "raises ZeroDivisionError", + "sess-worker-6")) + try: + tracker.triage(rid, "sess-worker-6") + check("reporter cannot triage own bug", False) + except SeparationOfDuty: + check("reporter cannot triage own bug", True) + decision = tracker.triage(rid, "sess-reviewer-7") + check("triage uses the call history as evidence", + decision == "revoked" and reg.get("flaky@1").maturity == "revoked", + decision) + store.close() + + +def s7_library(tmp: str) -> None: + print("\nS7 the library is the termination mechanism") + # (a) subcritical: repeated work should get cheaper via cache hits. + store, reg, lib, rt = fresh(tmp, "s7", max_depth=3, review_plans=False) + rt.planner = StubPlanner(seed=5, b=5, f=0.15, capability_pool=("echo@1",)) + nodes, plans = [], [] + for i in range(4): + b = Budget(400_000, 5.0, 2000, 600.0) + before, before_p = rt.stats.nodes, rt.stats.planner_calls + res = rt.run("summarise the sales corpus for region north", b, ROOT_AUTH, + run_id=f"m{i}", signature="any->str") + nodes.append(rt.stats.nodes - before) + plans.append(rt.stats.planner_calls - before_p) + check("subcritical mission completes", res.status == "completed", res.status) + check("library grows as problems are solved", len(lib) > 0, f"{len(lib)} entries") + check("a repeated problem needs strictly less PLANNING", + plans[-1] < plans[0], f"planner calls per mission: {plans}") + # The finding this scenario actually produced: a library hit removes the + # decomposition, not the work. Reuse drives `f` down (which is what + # termination needs) but it does NOT make execution cheaper unless results + # are memoised for identical inputs -- a separate mechanism #485 does not + # mention and the round-2 cost model quietly folded in. + check("but execution cost is UNCHANGED by a plan-level cache hit", + nodes[-1] == nodes[0], f"nodes per mission: {nodes}") + check("cache hits are recorded with similarity and reliability", + rt.stats.library_hits > 0, + f"{rt.stats.library_hits} hits / {rt.stats.library_misses} misses") + stmt = next(iter(lib.entries.values())).statement + check("identical text with an incompatible signature does not hit", + lib.lookup(stmt, "any->int") is None + and lib.lookup(stmt, "any->str") is not None) + check("an untyped solution is never published at all", + lib.publish("some untyped problem", "any->any", _plan_stub(), {}) is None) + store.close() + + # (b) supercritical: the mission CANNOT succeed. What must still improve is + # the cost of finding that out, and the honesty of the answer. + store2, _, lib2, rt2 = fresh(tmp, "s7b", max_depth=3, review_plans=False) + rt2.planner = StubPlanner(seed=5, b=5, f=0.35, capability_pool=("echo@1",)) + hard = [] + for i in range(4): + b = Budget(400_000, 5.0, 2000, 600.0) + before = rt2.stats.nodes + res2 = rt2.run("summarise the sales corpus for region north", b, ROOT_AUTH, + run_id=f"h{i}", signature="any->str") + hard.append(rt2.stats.nodes - before) + check("supercritical mission never reports success", + res2.status == "escalated", res2.status) + check("failure is remembered, so the retry is cheap", + hard[-1] < hard[0] / 2, f"nodes per attempt: {hard}") + check("negative results are first-class library entries", + len(lib2.dead_ends) > 0 and rt2.stats.dead_end_hits > 0, + f"{len(lib2.dead_ends)} dead ends, {rt2.stats.dead_end_hits} hits") + check("a dead end names how many attempts have hit it", + next(iter(lib2.dead_ends.values())).attempts >= 1) + store2.close() + + +def _plan_stub() -> Plan: + return Plan("stub", "x", (Step("a", "capability", ref="echo@1", + output_schema="str"),)) + + +def s8_validator(tmp: str) -> None: + print("\nS8 validator refuses plans the runtime cannot bound") + store, reg, lib, rt = fresh(tmp, "s8") + bad = Plan("p", "x", ( + Step("l", "loop", guard="true", then=(Step("i", "capability", + ref="echo@1"),)), + Step("u", "capability", ref="ghost@9"), + Step("p", "capability", ref="echo@1", + authority=Authority(subprocess=True)), + )) + errs = validate(bad, reg.maturities(), ROOT_AUTH) + check("unbounded loop rejected", any("static iteration bound" in e for e in errs)) + check("unknown capability rejected", any("unknown capability" in e for e in errs)) + check("authority widening rejected", any("beyond its parent" in e for e in errs)) + good = Plan("p", "x", (Step("a", "capability", ref="echo@1", + output_schema="str"),)) + check("valid plan passes", validate(good, reg.maturities(), ROOT_AUTH) == []) + store.close() + + +def s9_insights(tmp: str) -> None: + print("\nS9 insight pool gating") + store, reg, lib, rt = fresh(tmp, "s9") + pool = InsightPool(store, ReviewBoard(store)) + ok, ref = pool.propose("r", "n", "the calibration drifts above 40 degrees", + "sess-a", "sess-b") + check("insight accepted after independent review", ok, ref) + try: + pool.propose("r", "n", "another thought", "sess-a", "sess-a") + check("author cannot approve own insight", False) + except SeparationOfDuty: + check("author cannot approve own insight", True) + ok2, why = pool.propose("r", "n", + "the calibration does not drift above 40 degrees", + "sess-c", "sess-b") + check("contradicting insight is refused at insertion", not ok2, why) + store.close() + + +class OptimisticPlanner: + """A planner that marks everything atomic. Not a straw man: the best real + planner measured (probe_optimism.py) overclaimed 35% of its atomic steps, + and reported an apparent m of 0.12 while its corrected m was 1.62.""" + + def __init__(self, b: int = 4): + self.b = b + + def decompose(self, problem, signature, capabilities, depth): + from minikernel.planner import PlanDraft + steps = tuple( + Step(id=f"s{i}", kind="capability", ref="guess@1", + args={"text": f"{problem}#{i}"}, output_schema="str") + for i in range(self.b) + ) + return PlanDraft(Plan(f"opt-{depth}", problem, steps, + signature=signature), "everything looks easy") + + +def s10_admission(tmp: str) -> None: + print("\nS10 admission control: is 'atomic' a claim or a label?") + + def build(with_admission: bool): + store = Store(os.path.join(tmp, f"s10-{with_admission}.db")) + reg = CapabilityRegistry(store) + seed_capabilities(reg) + # A capability that always answers, and is always wrong on hard input. + reg.seed(Capability("guess", 1, lambda a: "PLAUSIBLE-BUT-UNVERIFIED", + "str", "str", tests=[({"text": "x"}, + "PLAUSIBLE-BUT-UNVERIFIED")])) + admission = None + if with_admission: + def admission(step, run_id): + # Stands in for the independent judge used in probe_optimism.py. + if step.ref == "guess@1" and "reproduce" in str(step.args): + return False, "guess@1 cannot reproduce a published figure" + return True, "" + rt = Runtime(store, reg, SolvedProblemLibrary(store), OptimisticPlanner(), + max_depth=2, review_plans=False, admission=admission) + res = rt.run("reproduce figure 3 from the released dataset", + Budget(200_000, 5.0, 400, 600.0), ROOT_AUTH, + run_id="ro", signature="any->str") + return store, rt, res + + st0, rt0, r0 = build(False) + check("without admission control the run reports SUCCESS", + r0.status == "completed", r0.status) + check("...and its answer is an unverified placeholder", + r0.value == "PLAUSIBLE-BUT-UNVERIFIED", str(r0.value)) + check("...and its measured m looks perfectly safe", + rt0.m_measured_safe() < 0.05 if hasattr(rt0, "m_measured_safe") + else rt0.stats.m_measured < 0.05, + f"m_measured={rt0.stats.m_measured:.2f} " + f"(declared {rt0.stats.m_declared:.2f})") + + st1, rt1, r1 = build(True) + check("with admission control the same run does NOT report success", + r1.status != "completed", r1.status) + check("overclaimed steps are demoted to decomposition, and recorded", + rt1.stats.demoted > 0, + f"{rt1.stats.demoted} steps demoted; " + f"declared m={rt1.stats.m_declared:.2f} -> " + f"corrected m={rt1.stats.m_measured:.2f}") + check("the demotion is an auditable event", + any(e["type"] == "atomicity_rejected" for e in st1.events("ro"))) + for st in (st0, st1): + st.close() + + +def main() -> int: + if len(sys.argv) > 2 and sys.argv[1] == "--child": + _child(sys.argv[2]) + return 0 + with tempfile.TemporaryDirectory() as tmp: + for scenario in (s1_nested_runs, s2_crash_resume, s3_budget, + s4_messages, s5_review, s6_capabilities, s7_library, + s8_validator, s9_insights, s10_admission): + scenario(tmp) + n_pass = sum(1 for _, ok, _ in results if ok) + print(f"\n{'='*70}\n{n_pass}/{len(results)} checks passed") + for name, ok, detail in results: + if not ok: + print(f" FAILED: {name} {detail}") + return 0 if n_pass == len(results) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/simulations/README.md b/scripts/simulations/README.md new file mode 100644 index 00000000..328d3b6b --- /dev/null +++ b/scripts/simulations/README.md @@ -0,0 +1,127 @@ +# Design simulations for #485 + +Four standalone models of the architecture proposed in +[#485](https://github.com/ContextLab/orchestrator/issues/485), written to +support the design review posted at +[#485 (comment)](https://github.com/ContextLab/orchestrator/issues/485#issuecomment-5384176128). + +**These are not product code.** Nothing under `src/` imports them and they are +not on any execution path. They exist so that the numbers quoted in that review +are reproducible, and so that a future agent can replace an assumption with a +measurement and see what moves. + +They are also an instance of the thing #485 itself asks for in component 1c: +turn a proposed solution into deterministic operations over objects of the +assumed shape, write down the predicted observables in advance, run it, and see +which predictions survive. + +## Running them + +Standard library only — no orchestrator import, no network, no API keys. + +```bash +.venv/bin/python scripts/simulations/decomposition_branching.py +.venv/bin/python scripts/simulations/scratchpad_contention.py +.venv/bin/python scripts/simulations/context_recursion.py +.venv/bin/python scripts/simulations/red_team_gate.py +``` + +Every stochastic model is seeded, so output is byte-identical run to run. The +invariants they demonstrate are asserted in +`tests/test_design_simulations.py` (marked `unit`), which is what keeps them +from rotting: + +```bash +.venv/bin/python -m pytest tests/test_design_simulations.py -q +``` + +## What each one answers + +| script | #485 component | review § | headline | +|-|-|-|-| +| `decomposition_branching.py` | 1 — recursive breakdown | §1, §5 | Decomposition is a Galton-Watson process: finite **iff `m = b·f < 1`**. At ~10 steps per pipeline, fewer than 1 step in 10 may be ambiguous, at every level. | +| `scratchpad_contention.py` | 3 — shared scratchpad | §2 | The LLM "consider" call sits inside the critical section, capping the whole fleet at `3600 / critical_section` steps/hour — 450/hr at 8s — regardless of fleet size. Re-summarising an append-only log per read is quadratic. | +| `context_recursion.py` | 1b — context recursion | §3 | The tree converges cheaply (depth ≤ 5 for 100M tokens, ~1.5× corpus cost), but retains `r**depth ≈ 0.0016` of the original at the top. It is a navigation structure, not a retrieval structure. | +| `red_team_gate.py` | 1a — critical review | §4 | The concern ledger removes scope drift entirely (8.6 → 3.6 rounds). But at `p_detect = 0.5`, only 45% of "clean" verdicts are truly clean. Same-family review has a hard `1-ρ` ceiling. | + +## Round 2: integrated-system models + +Six further scripts added after the three design reviews cross-compared +notes (posted as a follow-up comment on #485). Same rules: stdlib only, +seeded, `ASSUMPTIONS` dict at module scope, nothing under `src/` imports +them. Where round-1 scripts tested *mechanisms in isolation*, these resolve +the open disagreements between the reviews and assemble the whole proposal +into one runnable schematic. + +```bash +.venv/bin/python scripts/simulations/library_learning.py +.venv/bin/python scripts/simulations/review_policies.py +.venv/bin/python scripts/simulations/budget_allocation.py +.venv/bin/python scripts/simulations/summary_navigation.py +.venv/bin/python scripts/simulations/message_routing.py +.venv/bin/python scripts/simulations/system_integration.py +``` + +| script | question it answers | headline | +|-|-|-| +| `library_learning.py` | Does the solved-problem library rescue a supercritical start (`m0 ≈ 3`) without changing the planner? | **Yes.** `m_eff` falls below 1 by mission ~33 (complete-only) or ~15 (attempts-teach) — 2.2× faster when capped/failed missions also teach. Cold-start tax ≈ 5× nodes/mission. Stability is a property of the fleet's *memory*, not of any single run. | +| `review_policies.py` | Declarative vs ledger+adjudicator vs evidential red-team gates, head to head. | Declarative-unbounded hits the phase transition (63% convergence at FP/round=1.5). Evidential gating converges 100% everywhere at ~1.9× less reviewer time. Certification value (~60-70%) is bought by detection quality, not gate policy. | +| `budget_allocation.py` | How should a parent split its budget across children? | Equal-split-no-reserve silently truncates ≥1 leaf in 69% of missions while reporting success (#478's failure mode, generated by the resource model). Reserve-and-reallocate with escalation: 90% completion, zero silent truncations. Escalation beats prediction under heavy-tailed demand. | +| `summary_navigation.py` | Is the summary tree a retrieval mechanism or just a map? Few-hard vs many-gentle hops? | Navigate-only finds 30% of planted needles; index-only/hybrid 100%. At fixed top-level size, relative needle retention is (1−ε)^hops: r=0.1 via 2 hops keeps 90%, r=0.9 via 40 hops keeps 13%. Prefer fewer, harder compressions. | +| `message_routing.py` | Tree-only routing vs direct addressing (corrects an analytic sketch in the Opus review). | With greedy recovery + a ~2×-distance hop budget, tree delivery stays >94% even at 1M nodes — messages aren't lost, they burn calls (7.6 routing LLM-calls vs 1 addressed). Keep the tree for AUTHORITY; move TRANSPORT to an addressed channel. | +| `system_integration.py` | What does each subsystem buy? Full ablation of library / review / budgets vs "#485 as literally read". | Removing the library costs 5-6× tokens and 17 points of success; removing review doubles shipped-defect rate (30% → 16%); removing budgets costs graceful degradation. NAIVE ships defects in ~50% of declared successes at **~60× the token cost** of the governed system. The difference is not intelligence, it is governance. | + +### One finding that post-dates the review + +Hardening `context_recursion.py` against its tests separated two claims the +first draft had collapsed together. The recursion **converges iff `r < 1`** — +that part of the review stands — but the *depth* that takes explodes as `r` +approaches 1: `r = 0.5` needs 5 levels, `r = 0.9` needs 29, `r = 0.99` needs +299. Only the first of those fits under any sane depth cap. + +The useful corollary: top-level fidelity is `payload / N` **whatever `r` is**. +It is forced by the target size, not chosen. What `r` buys is *how many lossy +hops* you pass through to get there. So for a fixed final size, aggressive +single-hop summarisation accumulates less distortion than gentle multi-hop +summarisation — **prefer fewer, harder compressions**. That is a design +recommendation the review did not make, and it is asserted in +`test_top_level_fidelity_is_set_by_target_size_not_by_compression_ratio`. + +## Assumptions, and how to replace them + +Each script has an `ASSUMPTIONS` dict at module scope. **Every value in it is a +guess.** None was measured, because no live model was reachable when the review +was written: the `HF_TOKEN` in `~/.orchestrator/.env` authenticates against +`router.huggingface.co/v1/models` but returns `401` on `/v1/chat/completions`, +and there was no `DARTMOUTH_CHAT_API_KEY` on the machine. + +| assumption | used by | how to measure it | +|-|-|-| +| `f` — P(a step is ambiguous) | `decomposition_branching` | **The most important number in the project.** Decompose ~50 real subproblems, count how many need recursion. Decides whether the architecture works at all. | +| `r` — summary compression per level | `context_recursion`, `scratchpad_contention` | Summarise a real corpus at a fixed prompt; take output/input tokens. | +| `critical_section_locked_s` | `scratchpad_contention` | Time one real read-tail + decide + write cycle. | +| `p_detect` — P(reviewer finds a given defect) | `red_team_gate` | Seed known defects into artifacts, count how many a reviewer returns. Easier under evidential gating: count submitted artifacts. | +| `p_regress` — P(a fix introduces a defect) | `red_team_gate` | Track defects introduced by fix commits. | +| `ρ` — family-wide blind-spot rate | `red_team_gate` | Give the same seeded artifact to two model families; measure the overlap of what each misses. | +| `red_team_rounds`, `red_team_tokens` | `decomposition_branching` | Fall out of `red_team_gate` once `p_detect` is measured. | + +### Structural vs. numeric + +The **structural** conclusions do not depend on any of the above, and should be +treated as load-bearing: + +- termination iff `m = b·f < 1`; +- throughput ceiling of `1 / critical_section`, invariant to fleet size; +- quadratic-vs-linear summarisation churn on an append-only log; +- summary-tree fidelity decays as `r**depth`; +- catch rate for same-family reviewers is bounded above by `1 - ρ`. + +The **numeric** results — leaf counts, dollar figures, round counts, percentages +— are only as good as the table above. Quote them with the assumption attached. + +## If you change something + +1. Edit the relevant `ASSUMPTIONS` entry (or add a row to a table in `main()`). +2. Re-run the script and `tests/test_design_simulations.py`. +3. If a *structural* claim moved, that is a finding — post it on #485 rather + than quietly editing the number, so the design conversation sees it. diff --git a/scripts/simulations/budget_allocation.py b/scripts/simulations/budget_allocation.py new file mode 100644 index 00000000..5ffc0495 --- /dev/null +++ b/scripts/simulations/budget_allocation.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""How should a parent split a token budget across children? + +All three reviews demand budget credits instead of bare depth caps, but none +says HOW a parent should divide its allocation among children. This script +grows decomposition trees (branching process with heterogeneous — lognormal — +leaf costs) and compares three propagation policies at a fixed global budget: + + P1 **equal-split, no reserve** each child gets an equal share; a child + that runs out is silently truncated (the #478 "plausible wrong answer" + failure mode). + P2 **proportional, informed** the parent splits proportionally to noisy + per-child cost estimates. + P3 **reserve & reallocate** children get phi=70% of the parent's pool + split equally; 30% stays in reserve. An exhausted child ESCALATES with + partial results; the parent tops up from reserve or abandons it loudly. + +Metrics: mission completion within budget, silent-truncation rate (garbage +shipped as success), loud-failure rate, and wasted tokens. The claim under +test: escalation beats prediction when subtree demand is heavy-tailed, and +equal-split-without-escalation silently ships incomplete work exactly when +the tree is largest. +""" + +from __future__ import annotations + +import math +import random +import statistics + +ASSUMPTIONS = { + "trials": 2000, + "b": 10, # steps per pipeline (#485's target) + "f": 0.15, # ambiguity rate -> m = 1.5 supercritical start + "depth_cap": 3, + "leaf_cost_mu_ln": math.log(2_000), # mean leaf execution ~2k tokens... + "leaf_cost_sigma_ln": 1.0, # ...but heavy-tailed + "decomp_cost": 1_500, # tokens for one internal node to plan/decompose + "budget_multiple": 1.5, # global budget = multiple of E[total demand] + "reserve_fraction": 0.30, + "max_topups_per_child": 1, + "estimate_noise_sigma": 0.5, # lognormal noise on policy-2 estimates + "seed": 31, +} + + +def build_tree(rng: random.Random) -> tuple[int, float]: + """Grow one tree; return (n_leaves, total_demand_tokens).""" + b, f = ASSUMPTIONS["b"], ASSUMPTIONS["f"] + frontier = [0] + leaves = 0 + demand = ASSUMPTIONS["decomp_cost"] # root plans + while frontier: + depth = frontier.pop() + demand += ASSUMPTIONS["decomp_cost"] + for _ in range(b): + if depth + 1 < ASSUMPTIONS["depth_cap"] and rng.random() < f: + frontier.append(depth + 1) + else: + leaves += 1 + demand += rng.lognormvariate(ASSUMPTIONS["leaf_cost_mu_ln"], + ASSUMPTIONS["leaf_cost_sigma_ln"]) + return leaves, demand + + +def run_policy(policy: str, rng: random.Random) -> dict: + """One mission under one budget policy.""" + leaves, _true_demand = build_tree(rng) + + # Global budget is set from the EXPECTED demand of the average tree, not + # this tree's actual demand -- the governor cannot see the future. + expected_leaves = _expected_leaves() + expected_demand = (expected_leaves * + math.exp(ASSUMPTIONS["leaf_cost_mu_ln"] + + ASSUMPTIONS["leaf_cost_sigma_ln"] ** 2 / 2)) + budget_total = ASSUMPTIONS["budget_multiple"] * ( + expected_demand + ASSUMPTIONS["decomp_cost"] * _expected_nodes()) + + # Simulate leaf demands and walk the policies top-down. + leaf_costs = [rng.lognormvariate(ASSUMPTIONS["leaf_cost_mu_ln"], + ASSUMPTIONS["leaf_cost_sigma_ln"]) + for _ in range(leaves)] + + spent = 0.0 + completed = 0 + truncated = 0 # silent truncation (P1 failure mode) + abandoned = 0 # loud failure (P3) + + if policy == "P1": + share = budget_total / max(leaves, 1) + for c in leaf_costs: + spent += min(c, share) + if c <= share: + completed += 1 + else: + truncated += 1 # ships plausible garbage, claims success + + elif policy == "P2": + estimates = [c * math.exp(rng.gauss(0, ASSUMPTIONS["estimate_noise_sigma"])) + for c in leaf_costs] + total_est = sum(estimates) + scale = min(budget_total / max(total_est, 1), 50.0) # cap over-allocation + for c, e in zip(leaf_costs, estimates): + alloc = e * scale + spent += min(c, alloc) + if c <= alloc: + completed += 1 + else: + truncated += 1 + + else: # P3 reserve & reallocate + reserve = budget_total * ASSUMPTIONS["reserve_fraction"] + pool = budget_total - reserve + base = pool / max(leaves, 1) + escalations: list[float] = [] + for c in leaf_costs: + spent += min(c, base) + if c <= base: + completed += 1 + else: + escalations.append(c - base) # child reports shortfall honestly + escalations.sort(reverse=True) + for shortfall in escalations: + topup = min(shortfall, reserve) + reserve -= topup + spent += topup + if topup >= shortfall: + completed += 1 + else: + abandoned += 1 # explicit partial-failure status + + return { + "leaves": leaves, + "completed_frac": completed / max(leaves, 1), + "silent_truncation": truncated, + "loud_abandon": abandoned, + "spent": spent, + "budget": budget_total, + "mission_ok": completed == leaves, + } + + +def _expected_leaves() -> float: + """E[leaves] for b, f, depth_cap via the geometric series of the frontier.""" + b, f, d = ASSUMPTIONS["b"], ASSUMPTIONS["f"], ASSUMPTIONS["depth_cap"] + total = 0.0 + level = 1.0 + for depth in range(d): + total += level * b * (1 - f) + level *= b * f + total += level * b + return total + + +def _expected_nodes() -> float: + b, f, d = ASSUMPTIONS["b"], ASSUMPTIONS["f"], ASSUMPTIONS["depth_cap"] + total, level = 0.0, 1.0 + for _ in range(d): + total += level + level *= b * f + return total + level + + +def main() -> None: + print("== budget allocation policies ==") + print(f"b={ASSUMPTIONS['b']}, f={ASSUMPTIONS['f']} " + f"(m={ASSUMPTIONS['b'] * ASSUMPTIONS['f']:.2f}), " + f"depth cap {ASSUMPTIONS['depth_cap']}, leaf costs Lognormal(" + f"{ASSUMPTIONS['leaf_cost_mu_ln']:.1f}, {ASSUMPTIONS['leaf_cost_sigma_ln']})") + print(f"global budget = {ASSUMPTIONS['budget_multiple']}x expected demand; " + f"{ASSUMPTIONS['trials']} trials/policy") + + results = {} + for policy in ("P1", "P2", "P3"): + rng = random.Random(ASSUMPTIONS["seed"] + ord(policy[-1])) + rows = [run_policy(policy, rng) for _ in range(ASSUMPTIONS["trials"])] + results[policy] = rows + + names = {"P1": "P1 equal-split, no reserve", + "P2": "P2 proportional (noisy estimates)", + "P3": "P3 reserve & reallocate"} + hdr = (f"{'policy':>32} {'all-leaves-done%':>17} {'silent-trunc/mission':>21} " + f"{'loud-abandon/mission':>21} {'tokens wasted%':>15}") + print() + print(hdr) + print("-" * len(hdr)) + for p, rows in results.items(): + n = len(rows) + waste = statistics.fmean( + r["budget"] - r["spent"] for r in rows) / statistics.fmean( + r["budget"] for r in rows) + print(f"{names[p]:>32} " + f"{sum(r['mission_ok'] for r in rows) / n:>16.0%} " + f"{statistics.fmean(r['silent_truncation'] for r in rows):>21.1f} " + f"{statistics.fmean(r['loud_abandon'] for r in rows):>21.1f} " + f"{waste:>14.0%}") + + p1 = results["P1"] + n = len(p1) + any_trunc = sum(1 for r in p1 if r["silent_truncation"] > 0) / n + print("\n== headline ==") + print(f"* equal-split-no-reserve silently truncates >=1 leaf in " + f"{any_trunc:.0%} of missions and reports those missions as SUCCESS.") + print(" That is #478's 'plausible wrong answer' failure mode, generated by") + print(" the resource model itself rather than by a lying agent.") + print("* reserve-and-reallocate converts most of those into loud failures") + print(" or completions, at the price of holding reserve idle. Escalation") + print(" beats prediction because subtree demand is heavy-tailed.") + print("* design rule: budgets must be (a) visible per node, (b) topped up") + print(" only via recorded escalation events, (c) exhausted => explicit") + print(" partial-result status, never a green checkmark.") + + +if __name__ == "__main__": + main() diff --git a/scripts/simulations/context_recursion.py b/scripts/simulations/context_recursion.py new file mode 100644 index 00000000..20a71634 --- /dev/null +++ b/scripts/simulations/context_recursion.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Does the "context recursion" scheme in #485 converge, and what survives it? + +#485 proposes an inode-like scheme: content that exceeds 25% of a model's +context is split, each piece summarised, the summaries stitched back together, +and the whole thing repeated on the summaries until the top fits. The claim is +that this lets content of *any* length fit into any model's context. + +**What this answers** + +1. Does it terminate, and at what depth and cost? +2. How much of the original survives at the top? +3. What is left of the context window once #485's four allocations are paid? + +**Finding (report section 3).** The geometry is fine -- depth stays at or below +5 even for a 100M-token corpus, and total cost is about 1.5x the corpus. It +converges iff ``r < 1``, which any real summariser satisfies. The problem is +fidelity: at depth 4 with r=0.2 the top-level view retains r**4 = 0.0016 of the +original, so a *specific fact* is essentially invisible from the root. The +summary tree is a navigation structure, not a retrieval structure -- which +makes the search index load-bearing rather than a convenience. + +A second finding, surfaced by hardening this script against its tests: the +fidelity at the top is ``payload / n_tokens`` **whatever r is** -- it is forced +by the target size, not chosen. What ``r`` actually controls is how many lossy +hops you pass through to reach that size: r=0.1 gets there in 2 hops, r=0.9 +needs 29. So for a fixed final size, aggressive single-hop summarisation +accumulates less distortion than gentle multi-hop summarisation. Prefer fewer, +harder compressions. + +**Assumptions** (see ASSUMPTIONS; replace with measurements, then re-run). +The structural conclusions -- convergence iff r<1, depth logarithmic in N, +fidelity r**depth -- do not depend on the values. The specific numbers do. +""" + +from __future__ import annotations + +import math + +# --- assumptions ----------------------------------------------------------- +# Every value here is a guess, not a measurement. `r` in particular should be +# measured against a real summariser before any of the numbers are quoted. +ASSUMPTIONS = { + "r": 0.2, # summary tokens / input tokens, per level + "frac": 0.25, # share of context a payload may occupy (#485's rule) + "instr": 800, # prompt overhead per summariser call, tokens + "out_reserve_frac": 0.5, # share of payload budget reserved for the output + "depth_cap": 12, # backstop so a non-converging config still returns +} + +# #485's four context allocations, as literally specified. +BUDGET = [ + ("scratchpad, direct read (tail)", 0.05), + ("scratchpad, recursive summaries", 0.10), + ("insights RAG seed", 0.10), + ("one recursed document/problem payload", 0.25), +] + + +def summary_tree( + n_tokens: int, + context: int, + frac: float = ASSUMPTIONS["frac"], + r: float = ASSUMPTIONS["r"], + instr: int = ASSUMPTIONS["instr"], + out_reserve_frac: float = ASSUMPTIONS["out_reserve_frac"], + depth_cap: int = ASSUMPTIONS["depth_cap"], +) -> dict: + """Build the summary tree for ``n_tokens`` of content under ``context``. + + The usable chunk is smaller than the nominal ``frac * context``: the + summariser has to hold the chunk, its instructions, and its own output in + one window. Ignoring that overstates capacity by 5-20%. + + Returns depth, per-level shape, summariser calls, total tokens moved, the + size of the top-level view, and ``r ** depth`` (the share of the original + still represented up there). + """ + payload = frac * context + chunk = payload - instr - out_reserve_frac * payload * r + if chunk <= 0: + raise ValueError( + f"overheads ({instr} + reserve) exceed the payload budget ({payload:.0f})" + ) + + # Two different claims, easily conflated: the recursion terminates + # mathematically iff r < 1, but the depth that takes can be absurd as r + # approaches 1. Report both. + if r < 1 and n_tokens > payload: + depth_required = math.ceil(math.log(payload / n_tokens) / math.log(r)) + else: + depth_required = 0 if n_tokens <= payload else math.inf + + levels: list[tuple[int, float, int, float]] = [] + current, depth, calls, tokens = float(n_tokens), 0, 0, 0.0 + hit_cap = False + while current > payload: + if depth >= depth_cap: + hit_cap = True + break + n_chunks = math.ceil(current / chunk) + calls += n_chunks + tokens += current + n_chunks * instr + current * r + nxt = current * r + levels.append((depth + 1, current, n_chunks, nxt)) + current, depth = nxt, depth + 1 + + return { + "depth": depth, + "levels": levels, + "calls": calls, + "tokens": tokens, + "top": current, + "fidelity": r**depth, + "chunk": chunk, + "payload": payload, + "converges": r < 1, + "depth_required": depth_required, + "hit_cap": hit_cap, + } + + +def budget_subtotal() -> float: + """Fraction of context #485 spends before any work happens.""" + return sum(share for _, share in BUDGET) + + +def main() -> None: + print("== context-recursion tree geometry ==") + print("model context C, doc size N -> depth / summariser calls / tokens burned") + hdr = ( + f"{'C':>9} {'N':>12} {'r':>5} {'depth':>6} {'calls':>8} " + f"{'tokens_in+out':>14} {'top_tokens':>11} {'r^depth':>9}" + ) + print(hdr) + print("-" * len(hdr)) + for context in (128_000, 1_000_000): + for n_tokens in (250_000, 5_000_000, 100_000_000): + for r in (0.1, 0.2, 0.4): + t = summary_tree(n_tokens, context, r=r) + print( + f"{context:>9,} {n_tokens:>12,} {r:>5} {t['depth']:>6} " + f"{t['calls']:>8,} {int(t['tokens']):>14,} " + f"{int(t['top']):>11,} {t['fidelity']:>9.2e}" + ) + + print() + print("== termination condition (N=5M, C=1M) ==") + print(" converges iff r < 1 -- but the depth that takes explodes as r -> 1,") + print(" and fidelity r**depth is what is left of the original at the top.") + print() + hdr_t = f" {'r':<5} {'converges':>10} {'depth needed':>13} {'fidelity':>10} note" + print(hdr_t) + print(" " + "-" * (len(hdr_t) - 2)) + for r in (0.1, 0.5, 0.9, 0.99, 1.0, 1.1): + t = summary_tree(5_000_000, 1_000_000, r=r) + needed = t["depth_required"] + if not t["converges"]: + shown, fidelity, note = "never", "-", "summaries do not shrink" + else: + shown = f"{needed:.0f}" + fidelity = f"{r ** needed:.2e}" + note = ( + f"exceeds the depth cap of {ASSUMPTIONS['depth_cap']}" + if needed > ASSUMPTIONS["depth_cap"] + else "" + ) + print(f" {r:<5} {str(t['converges']):>10} {shown:>13} {fidelity:>10} {note}") + + print() + print("== usable chunk after overheads (C=1M, frac=0.25) ==") + for r in (0.1, 0.2, 0.4): + t = summary_tree(1, 1_000_000, r=r) + print( + f" r={r}: nominal payload {int(t['payload']):,} -> usable chunk " + f"{int(t['chunk']):,} ({100 * t['chunk'] / t['payload']:.0f}% of nominal)" + ) + + print() + print("== per-agent context budget, as literally specified in #485 ==") + for label, share in BUDGET: + print(f" {share * 100:>5.0f}% {label}") + subtotal = budget_subtotal() + print(f" {'-' * 5}") + print(f" {subtotal * 100:>5.0f}% SUBTOTAL (before system prompt, tool schemas,") + print(" pipeline spec, child outputs, and the agent's own reasoning)") + print(f" => {100 - subtotal * 100:.0f}% of context left for the actual work.") + for context in (128_000, 200_000, 1_000_000): + print( + f" C={context:>9,} -> {int(context * (1 - subtotal)):>9,} " + "tokens of working room" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/simulations/decomposition_branching.py b/scripts/simulations/decomposition_branching.py new file mode 100644 index 00000000..ea9909a6 --- /dev/null +++ b/scripts/simulations/decomposition_branching.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Does #485's recursive problem breakdown terminate, and what does it cost? + +Component 1 of #485 breaks a task into ~10 steps; each step is either atomic +(solve it now) or ambiguous (recurse, it becomes its own pipeline). That is a +Galton-Watson branching process: a node emits ``b`` steps, each ambiguous with +probability ``f``, so mean offspring is ``m = b * f``. + +**Finding (report sections 1 and 5).** The process is finite in expectation +**iff m < 1**. At #485's stated target of about 10 steps per pipeline, fewer +than 1 step in 10 may be ambiguous -- at every level -- or the tree does not +close. Nothing in #485 enforces that. + +Three consequences, in the report: + +1. The atomic/ambiguous classifier is the load-bearing component of the whole + design, and its false-"ambiguous" rate is the number to measure first. +2. The solved-problem library is the *termination mechanism*, not an + efficiency feature: every solved subtree that lands in it converts future + ambiguous steps into atomic ones, driving ``f`` down until the process goes + subcritical. +3. A depth cap alone is not enough. It truncates silently and returns a + plausible wrong answer. Budget credits that propagate down the tree, with + explicit escalation on exhaustion, fail loudly instead. + +The cost model shows where the tokens go: 34% to re-reading shared state before +any work happens, 57% to review, 8% to the work. + +**Assumptions** in ASSUMPTIONS. ``m = b * f < 1`` is structural and holds for +any values; the leaf counts and dollar figures do not. +""" + +from __future__ import annotations + +import random +import statistics + +# --- assumptions ----------------------------------------------------------- +ASSUMPTIONS = { + "depth_cap": 8, # #485 has none; this keeps supercritical runs finite + "node_cap": 2_000_000, # "runaway" threshold + "trials": 400, + "seed": 7, + "context_tokens": 200_000, + "red_team_rounds": 2.5, # see red_team_gate.py for where this comes from + "red_team_tokens": 34_000, # fresh session re-reads context each round + "usd_per_mtok": 3.0, + "steps_per_hour": 450.0, # the locked ceiling from scratchpad_contention.py +} + +# Per-node token cost, using #485's own context percentages on a 200k model. +NODE_COSTS = [ + ("read scratchpad tail + summaries (15%)", 30_000), + ("insight RAG seed (10%)", 20_000), + ("plan / decompose or execute", 12_000), + ("write scratchpad note", 1_000), +] +SHARED_STATE_COSTS = 2 # the first two rows are shared-state re-reading + + +def branch( + b: int, + f: float, + rng: random.Random, + depth_cap: int = ASSUMPTIONS["depth_cap"], + node_cap: int = ASSUMPTIONS["node_cap"], +) -> tuple[int, int, int] | None: + """One realisation of the decomposition tree. + + Starts from a single ambiguous root. Each ambiguous node emits ``b`` + children, each of which is itself ambiguous with probability ``f`` unless + ``depth_cap`` forces it atomic. Returns (leaves, internal nodes, max depth), + or None if the tree blew past ``node_cap`` before the cap bit. + """ + frontier = [0] + leaves = internal = max_depth = 0 + while frontier: + depth = frontier.pop() + internal += 1 + max_depth = max(max_depth, depth) + if internal + leaves > node_cap: + return None + for _ in range(b): + if depth + 1 < depth_cap and rng.random() < f: + frontier.append(depth + 1) + else: + leaves += 1 + return leaves, internal, max_depth + + +def regime(b: int, f: float) -> str: + """Subcritical / critical / supercritical, by mean offspring m = b*f.""" + m = b * f + if m < 1: + return "subcritical" + return "critical" if m == 1 else "SUPERCRITICAL" + + +def leaf_distribution(b: int, f: float, trials: int = ASSUMPTIONS["trials"], + seed: int = ASSUMPTIONS["seed"]) -> dict: + """Median and p95 leaf count over ``trials`` independent trees.""" + rng = random.Random(seed) + results = [branch(b, f, rng) for _ in range(trials)] + finite = [r[0] for r in results if r is not None] + runaway = sum(1 for r in results if r is None) + return { + "m": b * f, + "regime": regime(b, f), + "median": statistics.median(finite) if finite else float("nan"), + "p95": sorted(finite)[int(0.95 * len(finite))] if finite else float("nan"), + "runaway_pct": 100.0 * runaway / trials, + } + + +def tokens_per_node() -> float: + """Total tokens one node spends, review included.""" + return ( + sum(cost for _, cost in NODE_COSTS) + + ASSUMPTIONS["red_team_rounds"] * ASSUMPTIONS["red_team_tokens"] + ) + + +def shared_state_share() -> float: + """Fraction of a node's tokens spent re-reading shared state before working.""" + return sum(c for _, c in NODE_COSTS[:SHARED_STATE_COSTS]) / tokens_per_node() + + +def main() -> None: + print("== does recursive breakdown terminate? ==") + print("b = steps per pipeline, f = P(a step is 'ambiguous' -> recurse)") + print("Branching process: mean offspring m = b*f. Finite in expectation iff m < 1.") + print() + hdr = ( + f"{'b':>4} {'f':>6} {'m=b*f':>7} {'regime':>13} " + f"{'median leaves':>14} {'p95 leaves':>11} {'runaway%':>9}" + ) + print(hdr) + print("-" * len(hdr)) + for b in (5, 10): + for f in (0.05, 0.10, 0.15, 0.20, 0.30, 0.50): + d = leaf_distribution(b, f) + print( + f"{b:>4} {f:>6} {d['m']:>7.2f} {d['regime']:>13} " + f"{d['median']:>14,.0f} {d['p95']:>11,.0f} {d['runaway_pct']:>8.0f}%" + ) + print() + print( + f" (depth hard-capped at {ASSUMPTIONS['depth_cap']}; 'runaway' = more than " + f"{ASSUMPTIONS['node_cap']:,} nodes before the cap bit)" + ) + print(" => with b=10 you need f < 0.10: more than 90% of every pipeline's steps") + print(" must classify atomic, at EVERY level, or the tree does not close.") + + per_node = tokens_per_node() + print() + print("== token cost of one node, as #485 specifies it ==") + for label, cost in NODE_COSTS: + print(f" {cost:>8,} {label}") + print( + f" {ASSUMPTIONS['red_team_rounds'] * ASSUMPTIONS['red_team_tokens']:>8,.0f} " + f"red-team ({ASSUMPTIONS['red_team_rounds']} rounds x " + f"{ASSUMPTIONS['red_team_tokens']:,} in a fresh session)" + ) + print(f" {per_node:>8,.0f} TOTAL per node") + print( + f" of which {shared_state_share():.0%} is shared-state re-reading, " + "before any work happens" + ) + + print() + hdr2 = ( + f"{'nodes':>8} {'tokens':>14} {'$ @ $3/Mtok':>13} " + f"{'wall-clock @ 450 steps/hr':>27}" + ) + print(hdr2) + print("-" * len(hdr2)) + for n in (10, 100, 1_000, 10_000): + total = n * per_node + usd = total * ASSUMPTIONS["usd_per_mtok"] / 1e6 + hours = n / ASSUMPTIONS["steps_per_hour"] + print(f"{n:>8,} {int(total):>14,} {usd:>12,.0f} {hours:>25,.1f}h") + + worst = leaf_distribution(10, 0.30) + cost = worst["median"] * per_node * ASSUMPTIONS["usd_per_mtok"] / 1e6 + print() + print( + f" cross-reference: b=10, f=0.30 has a median of {worst['median']:,.0f} " + f"leaves -> about ${cost:,.0f} and " + f"{worst['median'] / ASSUMPTIONS['steps_per_hour']:,.0f}h for one question." + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/simulations/library_learning.py b/scripts/simulations/library_learning.py new file mode 100644 index 00000000..59c49bf5 --- /dev/null +++ b/scripts/simulations/library_learning.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Can the solved-problem library rescue supercritical decomposition? + +`decomposition_branching.py` established that recursive breakdown terminates +iff m = b*f < 1, and the Opus review of #485 argued (section 1b) that the +solved-problem library is the *termination mechanism*: solved ambiguous +subtrees become atomic cache hits next time, driving the effective ambiguity +rate down until the process goes subcritical. That claim had never been +simulated end to end. This script does it. + +Model +----- +* A universe of P distinct subproblem *types*; step instances draw types + i.i.d. from Zipf(s, P) -- demand concentrates on a few types. +* A type is intrinsically hard w.p. `h`: its FIRST encounter cannot be atomic, + it genuinely breaks down into `b` child steps (same law). Easy types + execute atomically immediately. +* Completing an ambiguous subtree stores its type in the library; later + instances are cache hits: cheaper and less defect-prone. +* Two escape hatches mirror the proposed governor: + - depth cap -> a node executes CRUDELY (5x cost, +1 defect) and the + mission continues; + - node budget cap -> the mission FAILS LOUDLY. +* Learning policy A ("complete-only"): only fully-expanded subtrees teach -- + crude depth-cap executions teach nothing. Policy B ("attempts"): everything + attempted teaches, including work inside capped runs. + +Questions answered: does repeated exposure drive a supercritical start +(m0 = b*h >> 1) below m = 1 WITHOUT changing the planner or classifier; how +long is cold start; and does learning from failed attempts matter? +""" + +from __future__ import annotations + +import random +import statistics + +ASSUMPTIONS = { + "types": 300, + "zipf_skew": 1.1, + "hard_rate": 0.30, # P(fresh type genuinely needs breakdown) + "steps_per_root": 10, # children per ambiguous node (#485 target ~10) + "node_cap": 600, # loud-failure budget per mission + "depth_cap": 5, # beyond this, crude execution + "missions": 250, + "trials": 60, + "seed": 11, +} + + +def zipf_table(n_types: int, skew: float) -> list[float]: + raw = [1.0 / (rank + 1) ** skew for rank in range(n_types)] + total = sum(raw) + return [p / total for p in raw] + + +def uncovered_hard_mass(learned: set[int], hard: set[int], + weights: list[float]) -> float: + """Remaining demand mass of hard types not yet in the library.""" + return sum(w for i, w in enumerate(weights) + if i in hard and i not in learned) + + +def run_mission(rng: random.Random, weights: list[float], hard: set[int], + learned: set[int], learn_policy: str) -> dict: + b = ASSUMPTIONS["steps_per_root"] + frontier = [(rng.choices(range(len(weights)), weights)[0], 1)] + nodes = 0 + crude = 0 + taught_complete: list[int] = [] + taught_attempted: list[int] = [] + + while frontier: + typ, depth = frontier.pop() + nodes += 1 + if nodes > ASSUMPTIONS["node_cap"]: + taught = taught_complete if learn_policy == "complete-only" \ + else taught_attempted + return {"ok": False, "nodes": nodes, "crude": crude, + "taught": taught} + if typ in learned: + continue # cache hit: atomic, cheap + if typ not in hard: + taught_complete.append(typ) # trivially solvable now and later + continue + taught_attempted.append(typ) + if depth >= ASSUMPTIONS["depth_cap"]: + crude += 1 # crude execution: expensive, flawed + taught_complete.append(typ) + continue + frontier.extend((rng.choices(range(len(weights)), weights)[0], depth + 1) + for _ in range(b)) + + taught = taught_complete if learn_policy == "complete-only" else taught_attempted + return {"ok": True, "nodes": nodes, "crude": crude, "taught": taught} + + +def timeline(learn_policy: str) -> list[dict]: + rng = random.Random(ASSUMPTIONS["seed"]) + weights = zipf_table(ASSUMPTIONS["types"], ASSUMPTIONS["zipf_skew"]) + hard = {i for i in range(ASSUMPTIONS["types"]) + if rng.random() < ASSUMPTIONS["hard_rate"]} + + per_index: list[list[dict]] = [[] for _ in range(ASSUMPTIONS["missions"])] + for _ in range(ASSUMPTIONS["trials"]): + learned: set[int] = set() + for mi in range(ASSUMPTIONS["missions"]): + res = run_mission(rng, weights, hard, learned, learn_policy) + learned.update(res["taught"]) + res["library"] = len(learned) + # P(a drawn child type is hard and unlearned) = remaining + # uncovered-hard demand mass -- this IS f(t). + res["p_recurse"] = uncovered_hard_mass(learned, hard, weights) + per_index[mi].append(res) + + agg = [] + for mi, rows in enumerate(per_index): + agg.append({ + "mission": mi + 1, + "success": statistics.fmean(r["ok"] for r in rows), + "nodes": statistics.mean(r["nodes"] for r in rows), + "crude": statistics.fmean(r["crude"] for r in rows), + "library": statistics.mean(r["library"] for r in rows), + # m_eff = b * f(t); p_recurse already IS P(child hard & unlearned) + "m_eff": ASSUMPTIONS["steps_per_root"] + * statistics.fmean(r["p_recurse"] for r in rows), + }) + return agg + + +def summarize(label: str, agg: list[dict]) -> None: + print(f"\n== {label} ==") + hdr = (f"{'mission':>8} {'success%':>9} {'mean nodes':>11} " + f"{'crude/miss':>11} {'library':>8} {'m_eff':>6}") + print(hdr) + print("-" * len(hdr)) + n = len(agg) + shown = sorted({1, 2, 3, 5, 10, 20, 40, 80, n}) + for m in shown: + a = agg[m - 1] + print(f"{a['mission']:>8} {a['success']:>8.0%} {a['nodes']:>11,.0f} " + f"{a['crude']:>11.1f} {a['library']:>8,.0f} {a['m_eff']:>6.2f}") + + +def main() -> None: + b, h = ASSUMPTIONS["steps_per_root"], ASSUMPTIONS["hard_rate"] + print("== can the library rescue a supercritical system? ==") + print(f"b={b}, fresh hardness h={h} -> cold-start m0 <= {b * h:.1f} " + f"by construction (supercritical); {ASSUMPTIONS['types']} types, Zipf " + f"{ASSUMPTIONS['zipf_skew']}, {ASSUMPTIONS['trials']} trials x " + f"{ASSUMPTIONS['missions']} missions") + + complete = timeline("complete-only") + attempts = timeline("attempts") + summarize("cold start, learn only from FULLY-COMPLETED subtrees", complete) + summarize("cold start, EVERY attempt teaches (capped runs included)", + attempts) + + def first_stable(agg: list[dict]) -> int: + for a in agg: + if a["m_eff"] < 1.0 and a["success"] >= 0.95: + return a["mission"] + return -1 + + c1, c2 = first_stable(complete), first_stable(attempts) + early = statistics.fmean(a["nodes"] for a in complete[:20]) + late = statistics.fmean(a["nodes"] for a in complete[-20:]) + print("\n== headline ==") + print(f"* m_eff drops below 1 with >=95% success at mission " + f"{max(c1, 1)} (complete-only) vs {max(c2, 1)} (attempts-teach).") + print(f"* cold-start tax: {early / max(late, 1):.0f}x more nodes per " + f"mission in the first 20 missions than the last 20.") + print("* stability is a property of the fleet's MEMORY, not of one run:") + print(" the same planner that melts on day one converges once the library") + print(" fills -- IF capped/failed attempts also teach. Design rules:") + print(" persist solutions from failed missions; warm-start the library;") + print(" measure f_eff continuously (it IS the system's vital sign).") + + +if __name__ == "__main__": + main() diff --git a/scripts/simulations/message_routing.py b/scripts/simulations/message_routing.py new file mode 100644 index 00000000..216d2e92 --- /dev/null +++ b/scripts/simulations/message_routing.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Tree-only routing vs direct addressing for the #485 org tree. + +#485 component 2 gives the org tree exactly two transport edges: a node routes +a message it cannot handle to its immediate parent (if higher-level) or to +selected immediate children. The Opus review (§6.5) sketched the consequence — +with b=10, d=4, two leaves sit 8 hops apart and an 8-hop route at 90% +per-hop accuracy arrives only 43% of the time — but never simulated it. + +This script grows complete b-ary trees of depth d, picks random sender/target +pairs, and compares: + + T **tree-only** every hop is an LLM routing decision that steers toward + the target w.p. `q_route`; a mis-steer moves to a random adjacent node; + messages carry `max_hops` and die as dead letters afterwards. + B **direct bus** nodes publish scope/capability descriptors; delivery is + addressed, one hop, w.p. `q_direct` (no routing decisions to get wrong). + +Metrics: P(delivered), expected hops (each hop = one model call = cost), +and dead-letter rate. Claim under test: tree-only transport collapses with +tree size while direct addressing stays flat; the tree should carry AUTHORITY, +not TRANSPORT. +""" + +from __future__ import annotations + +import random +import statistics + +ASSUMPTIONS = { + "trials_per_config": 4000, + "q_route": 0.90, # per-hop accuracy of an LLM routing decision + "q_direct": 0.995, # addressed delivery over a durable channel + "max_hops": 16, + "configs": [(3, 3), (4, 4), (8, 3), (10, 4), (10, 6), (16, 4)], + "seed": 59, +} + + +def build_tree(b: int, depth: int) -> dict[int, list[int]]: + """Adjacency lists for a complete b-ary tree; node ids in BFS order.""" + adj: dict[int, list[int]] = {} + nid = 0 + frontier = [0] + for _ in range(depth): + nxt: list[int] = [] + for parent in frontier: + kids = [] + for _ in range(b): + nid += 1 + kids.append(nid) + adj[parent] = kids + nxt.extend(kids) + frontier = nxt + return adj + + +def lca_depth(u: int, v: int, b: int) -> int: + """Depth of LCA via path-to-root unwinding (ids encode BFS positions).""" + du, dv = depth_of(u, b), depth_of(v, b) + while du > dv: + u = (u - 1) // b + du -= 1 + while dv > du: + v = (v - 1) // b + dv -= 1 + while u != v: + u = (u - 1) // b + v = (v - 1) // b + return depth_of(u, b) + + +def depth_of(u: int, b: int) -> int: + d = 0 + while u > 0: + u = (u - 1) // b + d += 1 + return d + + +def neighbors(u: int, b: int) -> set[int]: + nb = {(u - 1) // b} if u != 0 else set() + nb.update(children_of(u, b)) + if u != 0: + parent = (u - 1) // b + nb.update(k for k in children_of(parent, b) if k != u) + return nb + + +def children_of(u: int, b: int) -> list[int]: + first = u * b + 1 + return list(range(first, first + b)) + + +def deliver_tree(target: int, start: int, b: int, rng: random.Random) -> tuple[bool, int]: + """Steer greedily toward target; each hop correct w.p. q_route else random.""" + cur = start + for hop in range(1, ASSUMPTIONS["max_hops"] + 1): + options = sorted(neighbors(cur, b)) + # greedy choice: neighbor closest to target by id-path distance proxy + best = min(options, key=lambda n: _dist(n, target, b)) + cur = best if rng.random() < ASSUMPTIONS["q_route"] else rng.choice(list(neighbors(cur, b))) + if cur == target: + return True, hop + return False, ASSUMPTIONS["max_hops"] + + +def deliver_bus(start: int, target: int, rng: random.Random) -> tuple[bool, int]: + ok = rng.random() < ASSUMPTIONS["q_direct"] + return ok, 1 + + +def _dist(u: int, v: int, b: int) -> int: + """Hop distance between two nodes through their LCA.""" + lca = lca_depth(u, v, b) + return depth_of(u, b) + depth_of(v, b) - 2 * lca + + +def run_config(b: int, depth: int) -> None: + rng = random.Random(ASSUMPTIONS["seed"] + b * 100 + depth) + n_nodes = (b ** (depth + 1) - 1) // (b - 1) + + t_del, t_hops = [], [] + for _ in range(ASSUMPTIONS["trials_per_config"]): + s = rng.randrange(1, n_nodes) + t = rng.randrange(1, n_nodes) + while t == s: + t = rng.randrange(1, n_nodes) + ok, hops = deliver_tree(t, s, b, rng) + t_del.append(ok) + t_hops.append(hops) + + b_del, b_hops = [], [] + for _ in range(ASSUMPTIONS["trials_per_config"]): + s = rng.randrange(1, n_nodes) + t = rng.randrange(1, n_nodes) + while t == s: + t = rng.randrange(1, n_nodes) + ok, hops = deliver_bus(s, t, rng) + b_del.append(ok) + b_hops.append(hops) + + print(f"b={b:>2} d={depth} nodes={n_nodes:>7,} " + f"tree: {statistics.fmean(t_del):>6.1%} delivered, " + f"{statistics.fmean(t_hops):>5.1f} mean hops | " + f"bus: {statistics.fmean(b_del):>6.1%}, " + f"{statistics.fmean(b_hops):>3.1f} hops") + + # sanity anchor: the review's analytic example was q^8 = 43% for two + # leaves at opposite corners -- but that assumes every hop must be + # correct. Greedy steering RECOVERS from misroutes when the hop budget + # leaves slack (~2x distance), so simulated delivery stays high; what + # scales with distance is the number of routing LLM-calls burned. + if b == 10 and depth == 4: + analytic = ASSUMPTIONS["q_route"] ** 8 + print(f" (review's no-recovery bound for corner pairs: " + f"q^8 = {analytic:.0%}; with recovery the message survives --") + print(f" it just burns {statistics.fmean(t_hops):.1f} routing " + f"calls instead of 1)") + + +def main() -> None: + print("== message transport: org-tree hops vs direct addressing ==") + print(f"per-hop routing accuracy q={ASSUMPTIONS['q_route']}, " + f"direct delivery q={ASSUMPTIONS['q_direct']}, cap {ASSUMPTIONS['max_hops']} hops") + print() + run_all = [ + (3, 3), (4, 4), (8, 3), (10, 4), (10, 6), (16, 4), + ] + for b, d in run_all: + run_config(b, d) + print("\n== headline ==") + print("* CORRECTION to the review's sketch: with greedy recovery and a") + print(" ~2x-distance hop budget, tree-only delivery stays >94% even in") + print(" huge fleets -- messages are not lost, they take the scenic route.") + print(" The real cost is per-hop routing LLM-calls (7.6 mean at b=10,d=4") + print(" vs 1 addressed call), plus scope-leak risk at every relay hop.") + print("* direct addressing is flat in fleet size. Keep the tree for") + print(" AUTHORITY (who may commit, who reviews whom) and move TRANSPORT") + print(" onto an addressed channel; keep hop budget + visited set on any") + print(" relayed message regardless of mode.") + + +if __name__ == "__main__": + main() diff --git a/scripts/simulations/red_team_gate.py b/scripts/simulations/red_team_gate.py new file mode 100644 index 00000000..a03b268c --- /dev/null +++ b/scripts/simulations/red_team_gate.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""What does #485's red-team loop actually buy, and what does "clean" certify? + +#485 requires that every artifact be red-teamed by a separate agent, looping +until the reviewer's concerns "come back clean", with concerns tracked so scope +does not drift. No authoring agent may review its own work, though the same +*model* may review another instance of itself in a fresh session. + +Three questions, three models. + +**(a) Is the concern ledger worth it?** Yes, and it is the whole mechanism. +Unbounded scope drift turns a 3.6-round loop into 8.6 rounds with a 2.2% +never-terminating tail; the ledger removes drift's effect entirely. Concretely: +freeze the acceptance criteria *before* authoring; every concern gets an id, a +state, and a link to the frozen criterion it violates; a concern that maps to +no criterion is auto-filed as deferred rather than blocking. + +**(b) What does a clean verdict certify?** Much less than it looks like. At a +plausible 50% detection rate, only 45% of clean passes are actually clean -- +a clean verdict is wrong more often than it is right. The loop is +self-deceiving in a specific way: it terminates exactly when the reviewer stops +finding things, which tracks reviewer fatigue as much as artifact quality. +The report's fix is to change what a reviewer may *submit*: a blocking concern +must ship a reproducible artifact (a failing assertion, a counterexample input, +a contradicting citation, a command whose output differs from the prediction). +Prose worries are recorded as non-blocking risks. That terminates on evidence, +makes ``p_detect`` measurable by counting artifacts, and composes with #485's +own component 1c. + +**(c) How much independence does "same model, fresh session" buy?** Let ``rho`` +be the probability that a defect is systematically invisible to a model family. +Same-family reviewers are conditionally independent only on the ``1-rho`` +fraction, so no number of them beats a ``1-rho`` ceiling. One reviewer from a +different family beats five from the same one once ``rho > ~0.1``. This +collides with the two-provider policy in #430/#484. + +**Assumptions** in ASSUMPTIONS. The structural conclusions -- the ledger makes +rounds independent of drift, a clean verdict is weak evidence at low +``p_detect``, and the ``1-rho`` ceiling -- hold for any values. +""" + +from __future__ import annotations + +import random +import statistics + +# --- assumptions ----------------------------------------------------------- +ASSUMPTIONS = { + "initial_defects": 6, + "p_detect": 0.6, # P(reviewer finds a given defect in a round) + "p_regress": 0.15, # P(a fix introduces a new defect) + "max_rounds": 25, + "trials": 2_000, + "seed": 11, + "p_catch": 0.6, # per-reviewer catch rate, for the independence model +} + + +def review_loop( + initial_defects: int, + p_detect: float, + p_regress: float, + p_drift: float, + ledger: bool, + rng: random.Random, + max_rounds: int = ASSUMPTIONS["max_rounds"], +) -> tuple[int, int, bool]: + """One author/red-team loop. + + Each round the reviewer finds each outstanding defect with probability + ``p_detect``; each fix introduces a new defect with probability + ``p_regress``; the reviewer also raises a new-*scope* concern with + probability ``p_drift``. With ``ledger=True`` an out-of-scope concern is + recorded and deferred rather than blocking the gate. + + Returns (rounds used, defects remaining, converged). + """ + defects, rounds = initial_defects, 0 + while rounds < max_rounds: + rounds += 1 + found = sum(1 for _ in range(defects) if rng.random() < p_detect) + drifted = rng.random() < p_drift + blocking = found + (0 if ledger else int(drifted)) + if blocking == 0: + return rounds, defects, True + defects -= found + defects += sum(1 for _ in range(found) if rng.random() < p_regress) + if not ledger and drifted: + defects += 1 # a redefinition becomes real new work + return rounds, defects, False + + +def loop_stats(p_drift: float, ledger: bool, p_detect: float = ASSUMPTIONS["p_detect"], + trials: int = ASSUMPTIONS["trials"], + seed: int = ASSUMPTIONS["seed"]) -> dict: + """Aggregate ``trials`` review loops.""" + rng = random.Random(seed) + runs = [ + review_loop( + ASSUMPTIONS["initial_defects"], p_detect, ASSUMPTIONS["p_regress"], + p_drift, ledger, rng, + ) + for _ in range(trials) + ] + converged = [r for r in runs if r[2]] + return { + "mean_rounds": statistics.mean(r[0] for r in converged) if converged else float("nan"), + "p95_rounds": sorted(r[0] for r in converged)[int(0.95 * len(converged))] + if converged + else float("nan"), + "never_clean_pct": 100.0 * (1 - len(converged) / trials), + "defects_left": statistics.mean(r[1] for r in runs), + "p_truly_clean": ( + sum(1 for r in converged if r[1] == 0) / len(converged) if converged else float("nan") + ), + "residual_given_clean": ( + statistics.mean(r[1] for r in converged) if converged else float("nan") + ), + } + + +def catch_rate(rho: float, n_reviewers: int, p_catch: float = ASSUMPTIONS["p_catch"]) -> float: + """P(defect caught) by ``n_reviewers`` from the SAME model family. + + A share ``rho`` of defects is invisible to the whole family; reviewers are + conditionally independent only on the rest. Ceiling is ``1 - rho``. + """ + return 1 - rho - (1 - rho) * (1 - p_catch) ** n_reviewers + + +def cross_family_catch_rate(rho: float, p_catch: float = ASSUMPTIONS["p_catch"]) -> float: + """P(defect caught) by two reviewers from families with independent blind spots.""" + joint_blind = rho * rho + return 1 - joint_blind - (1 - joint_blind) * (1 - p_catch) ** 2 + + +def main() -> None: + print("== rounds to a clean red-team pass ==") + print( + f"{ASSUMPTIONS['trials']:,} trials; D0={ASSUMPTIONS['initial_defects']} latent " + f"defects, p_detect={ASSUMPTIONS['p_detect']}, " + f"p_regress={ASSUMPTIONS['p_regress']}" + ) + print() + hdr = ( + f"{'p_drift':>8} {'ledger':>8} {'mean rounds':>12} {'p95 rounds':>11} " + f"{'never clean':>12} {'defects left':>13}" + ) + print(hdr) + print("-" * len(hdr)) + for p_drift in (0.0, 0.2, 0.4, 0.6): + for ledger in (False, True): + s = loop_stats(p_drift, ledger) + print( + f"{p_drift:>8} {str(ledger):>8} {s['mean_rounds']:>12.1f} " + f"{s['p95_rounds']:>11} {s['never_clean_pct']:>11.1f}% " + f"{s['defects_left']:>13.2f}" + ) + + print() + print("== what a clean pass actually certifies ==") + print("P(zero defects remain | the reviewer returned clean), by detection rate") + print() + hdr2 = ( + f"{'p_detect':>9} {'mean rounds':>12} {'P(truly clean)':>15} " + f"{'mean residual defects':>22}" + ) + print(hdr2) + print("-" * len(hdr2)) + for p_detect in (0.3, 0.5, 0.7, 0.9): + s = loop_stats(0.0, True, p_detect=p_detect, trials=4_000, seed=3) + print( + f"{p_detect:>9} {s['mean_rounds']:>12.1f} {s['p_truly_clean']:>14.0%} " + f"{s['residual_given_clean']:>22.2f}" + ) + + print() + print("== reviewer independence ceiling ==") + print("rho = P(a defect is systematically invisible to this model family).") + print("Same-family reviewers are conditionally independent only on the 1-rho part.") + print() + hdr3 = ( + f"{'rho':>6} {'1 rev':>8} {'2 rev':>8} {'3 rev':>8} {'5 rev':>8} " + f"{'inf rev':>9} {'+1 other family':>17}" + ) + print(hdr3) + print("-" * len(hdr3)) + for rho in (0.0, 0.1, 0.2, 0.3, 0.5): + row = " ".join(f"{catch_rate(rho, n):>7.0%}" for n in (1, 2, 3, 5)) + print( + f"{rho:>6} {row} {1 - rho:>8.0%} " + f"{cross_family_catch_rate(rho):>16.0%}" + ) + print() + print(" No number of same-family reviewers beats the 1-rho ceiling.") + print(" One reviewer from a different family beats five from the same one") + print(" whenever rho > ~0.1.") + + +if __name__ == "__main__": + main() diff --git a/scripts/simulations/review_policies.py b/scripts/simulations/review_policies.py new file mode 100644 index 00000000..8368b1dd --- /dev/null +++ b/scripts/simulations/review_policies.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Which red-team gate should #485 use? Declarative vs adjudicated vs evidential. + +The three reviews agree the "loop until clean" gate needs bounding, but propose +different fixes: + + A. **declarative, unbounded** -- as #485 literally reads: loop until the + reviewer returns no concerns. + B. **ledger + cap + adjudicator** -- bounded rounds; an independent + adjudicator confirms/dismisses each finding correctly w.p. q_adj + (the Ox Alpha E3 proposal). + C. **evidential** -- a BLOCKING concern must ship a reproducible artifact + (failing assertion / counterexample / contradicting citation); prose + worries are recorded as non-blocking risks that force no changes (the + Opus review section 4b proposal). + +Model per round (faithful to exp3_review_convergence.py, extended): +* True defects D: reviewer finds each w.p. p_detect. +* False positives arrive ~ Poisson(lambda_fp0 * (0.5 + 0.5*scope)) -- the FP + rate scales with artifact size, and churn grows the artifact, which is what + creates A's non-termination spiral. +* Fixing anything regresses w.p. p_regress and grows scope by 5%/item. +* Adjudication (B): each finding confirmed w.p. q_adj; only confirmed ones + drive churn. Evidence gate (C): detection recall drops to + p_detect * evidence_recall, but blocker-FPs drop to + lambda_fp * fp_evidence_ratio (fabricating a reproducible counterexample is + hard), and prose risks never churn. +* Round cap R everywhere; overflow = escalation to the user (not silence). + +Reported per policy: convergence rate within R rounds, mean rounds, residual +true defects at ship, scope growth, certification value P(clean | pass), and +reviewer tokens. Structural claim under test: A's termination collapses as +lambda_fp0 grows past ~0.7 while B and C stay flat; C dominates on +certification value per token. +""" + +from __future__ import annotations + +import math +import random +import statistics + +ASSUMPTIONS = { + "trials": 4000, + "round_cap": 12, + "lambda_def": 3.0, + "p_detect": 0.6, + "evidence_recall": 0.85, + "lambda_fp0": 1.0, + "fp_evidence_ratio": 0.15, + "p_regress": 0.15, + "q_adj": 0.85, + "review_tokens_per_round": 34_000, + "seed": 23, +} + + +def poisson(rng: random.Random, lam: float) -> int: + if lam <= 0: + return 0 + limit, k, p = math.exp(-lam), 0, 1.0 + while True: + p *= rng.random() + if p <= limit: + return k + k += 1 + + +def binom(n: int, p: float, rng: random.Random) -> int: + return sum(1 for _ in range(n) if rng.random() < p) + + +def simulate(policy: str, rng: random.Random) -> dict: + lam0 = ASSUMPTIONS["lambda_fp0"] + D = poisson(rng, ASSUMPTIONS["lambda_def"]) + scope = 1.0 + verdict_pass = False + + rounds = 0 + for rounds in range(1, ASSUMPTIONS["round_cap"] + 1): + found = binom(D, ASSUMPTIONS["p_detect"], rng) + lam = lam0 * (0.5 + 0.5 * scope) + + if policy == "A": + fps = poisson(rng, lam) + if found == 0 and fps == 0: + verdict_pass = True + break + D -= found # fixes are clean... + D += binom(found + fps, ASSUMPTIONS["p_regress"], rng) # ...mostly + scope *= 1.0 + 0.05 * fps + + elif policy == "B": + fps = poisson(rng, lam) + confirmed = binom(fps, ASSUMPTIONS["q_adj"], rng) + wrongly_dismissed = found - binom(found, ASSUMPTIONS["q_adj"], rng) + if confirmed + (found - wrongly_dismissed) == 0: + verdict_pass = True + D += wrongly_dismissed # dismissed but real + break + D -= found + D += binom(found + confirmed, ASSUMPTIONS["p_regress"], rng) + scope *= 1.0 + 0.05 * confirmed + + else: # C: evidential + found_ev = round(found * ASSUMPTIONS["evidence_recall"]) + fps_block = poisson(rng, lam * ASSUMPTIONS["fp_evidence_ratio"]) + if found_ev + fps_block == 0: + verdict_pass = True + break # prose risks recorded, non-blocking + D -= round(found * ASSUMPTIONS["evidence_recall"]) + D += binom(found_ev + fps_block, ASSUMPTIONS["p_regress"], rng) + scope *= 1.0 + 0.05 * (found_ev + fps_block) + + return { + "pass": verdict_pass, + "residual": max(D, 0), + "rounds_used": rounds, + "scope": scope, + "tokens": rounds * ASSUMPTIONS["review_tokens_per_round"], + } + + +def run_policy(policy: str, trials: int | None = None) -> dict: + rng = random.Random(ASSUMPTIONS["seed"] + ord(policy)) + n = trials or ASSUMPTIONS["trials"] + rows = [simulate(policy, rng) for _ in range(n)] + passed = [r for r in rows if r["pass"]] + return { + "converged": statistics.fmean(r["pass"] for r in rows), + "mean_rounds": statistics.fmean(r["rounds_used"] for r in rows), + "residual_at_pass": (statistics.fmean(r["residual"] for r in passed) + if passed else float("nan")), + "cert_value": (sum(1 for r in passed if r["residual"] == 0) / len(passed) + if passed else float("nan")), + "mean_scope": statistics.fmean(r["scope"] for r in rows), + "mean_tokens": statistics.fmean(r["tokens"] for r in rows), + } + + +def main() -> None: + print("== which red-team gate? ==") + print(f"D~Poisson({ASSUMPTIONS['lambda_def']}) latent defects; " + f"p_detect={ASSUMPTIONS['p_detect']}; FP~Poisson(" + f"{ASSUMPTIONS['lambda_fp0']}*(0.5+0.5*scope))/round; " + f"cap {ASSUMPTIONS['round_cap']} rounds") + print() + + names = {"A": "A declarative-unbounded", "B": "B ledger+cap+adjudicator", + "C": "C evidential"} + hdr = (f"{'policy':>26} {'conv%':>6} {'rounds':>7} {'P(clean|pass)':>14} " + f"{'defects@pass':>13} {'scope':>6} {'tokens':>9}") + print(hdr) + print("-" * len(hdr)) + results = {} + for p in ("A", "B", "C"): + r = run_policy(p) + results[p] = r + print(f"{names[p]:>26} {r['converged']:>6.0%} {r['mean_rounds']:>7.1f} " + f"{r['cert_value']:>13.0%} {r['residual_at_pass']:>12.2f} " + f"{r['mean_scope']:>6.2f} {r['mean_tokens']:>9,.0f}") + + print("\n== convergence%% vs base FP rate ==") + hdr2 = f"{'FP/round':>9} {'A':>6} {'B':>6} {'C':>6}" + print(hdr2) + print("-" * len(hdr2)) + saved = dict(ASSUMPTIONS) + for lam in (0.10, 0.35, 0.50, 0.75, 1.00, 1.50): + ASSUMPTIONS["lambda_fp0"] = lam + cells = [] + for p in ("A", "B", "C"): + r = run_policy(p, trials=1500) + cells.append(r["converged"]) + print(f"{lam:>9.2f} {cells[0]:>5.0%} {cells[1]:>5.0%} {cells[2]:>5.0%}") + ASSUMPTIONS.clear() + ASSUMPTIONS.update(saved) + + a, c = results["A"], results["C"] + print("\n== headline ==") + print("* declarative-unbounded (A) hits the phase transition: convergence") + print(" collapses as FP/round passes ~0.7 (sweep above), matching the") + print(f" earlier E3 result. At the default rate it burns " + f"{a['mean_rounds']:.1f} rounds/artifact.") + print(f"* evidential (C) converges everywhere, ships " + f"{c['cert_value']:.0%}-clean passes, at " + f"{a['mean_tokens'] / max(c['mean_tokens'], 1):.1f}x less reviewer") + print(" time than A: it stops on evidence, not on imagination running out.") + print("* recommended default: C as the gate, B's ledger + cap as the safety") + print(" net, escalation on cap overflow. Prose worries stay visible as") + print(" risks -- they are signal, they just cannot block.") + + +if __name__ == "__main__": + main() diff --git a/scripts/simulations/scratchpad_contention.py b/scripts/simulations/scratchpad_contention.py new file mode 100644 index 00000000..c18867d7 --- /dev/null +++ b/scripts/simulations/scratchpad_contention.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""What does #485's shared-scratchpad semaphore cost the fleet? + +#485 specifies: "before executing a step, wait for the lock to become +available. then grab the lock. then read recent messages and consider whether +it alters the current plans ... then write down your plans. then release the +lock." + +The "consider" is an LLM call, so it sits *inside* the critical section. This +is a discrete-event simulation of that, and of the same fleet with the model +call moved outside the lock. + +It also models the second-order problem: the scratchpad is an append-only log, +not a static document, so the context-recursion tree over it is rebuilt +constantly unless segments are sealed. + +**Findings (report section 2).** + +* With the model call inside the lock, fleet throughput saturates at + ``3600 / critical_section`` agent-steps per hour -- 450/hr at 8s -- no matter + how many agents run. At 32 agents, 72% of every step's wall clock is lock + wait. Adding agents past ~16 buys latency and cost and no throughput. +* Moving the call outside (the lock then guards only an append and a sequence + bump) gives ~61x the throughput at 512 agents, and the read path needs no + lock at all. +* Re-summarising the log on every read is quadratic in log length: 2.0e12 + tokens at 100k notes, against 5.0e7 for sealed immutable segments. + +**Assumptions** in ASSUMPTIONS below. The structural conclusions -- a hard +throughput ceiling of 1/critical-section, and quadratic-vs-linear churn -- hold +for any values; the specific numbers do not. +""" + +from __future__ import annotations + +import heapq +import random +import statistics + +# --- assumptions ----------------------------------------------------------- +ASSUMPTIONS = { + "critical_section_locked_s": 8.0, # read tail + LLM decide + write + "critical_section_append_s": 0.03, # append + version bump only + "useful_work_s": 60.0, # per agent step, outside the lock + "horizon_s": 3600.0, # simulated hour + "context_tokens": 200_000, + "tail_frac": 0.05, # #485's "direct read" share + "seal_frac": 0.25, # sealed segment size, as a share of context + "tokens_per_note": 400, + "r": 0.2, # summary compression, per level +} + + +def simulate_lock( + n_agents: int, + critical_section_s: float, + work_s: float, + horizon_s: float = ASSUMPTIONS["horizon_s"], + seed: int = 0, +) -> dict: + """Agents loop: [wait for lock -> critical section -> release] -> work. + + One global lock, exponential service and work times. Service that would run + past the horizon is truncated so utilisation cannot exceed 100% and + throughput cannot exceed the ``horizon / critical_section`` ceiling. + """ + rng = random.Random(seed) + events: list[tuple[float, int]] = [] + for agent in range(n_agents): + heapq.heappush(events, (rng.random() * work_s, agent)) + + lock_free_at, busy, completed = 0.0, 0.0, 0 + waits: list[float] = [] + while events: + now, agent = heapq.heappop(events) + if now > horizon_s: + break + start = max(now, lock_free_at) + if start > horizon_s: + break + waits.append(start - now) + duration = rng.expovariate(1 / critical_section_s) + lock_free_at = start + duration + busy += min(lock_free_at, horizon_s) - start + completed += 1 + heapq.heappush(events, (lock_free_at + rng.expovariate(1 / work_s), agent)) + + return { + "steps": completed, + "steps_per_hour": completed * 3600.0 / horizon_s, + "mean_wait": statistics.mean(waits) if waits else 0.0, + "p95_wait": sorted(waits)[int(0.95 * len(waits))] if waits else 0.0, + "utilisation": busy / horizon_s, + } + + +def throughput_ceiling(critical_section_s: float, horizon_s: float = 3600.0) -> float: + """Steps per hour a single global lock can ever admit.""" + return horizon_s / critical_section_s + + +def churn( + n_notes: int, + context: int = ASSUMPTIONS["context_tokens"], + tail_frac: float = ASSUMPTIONS["tail_frac"], + tokens_per_note: int = ASSUMPTIONS["tokens_per_note"], + seal_frac: float = ASSUMPTIONS["seal_frac"], + r: float = ASSUMPTIONS["r"], +) -> dict: + """Summarisation tokens for an append-only log, naive vs sealed segments. + + naive: every read re-summarises the whole non-tail body. + sealed: a segment is summarised once when it fills, then never again. + """ + log_tokens = n_notes * tokens_per_note + naive = sum( + max(0.0, i * tokens_per_note - context * tail_frac) for i in range(n_notes) + ) + sealed, current = 0.0, float(log_tokens) + while current > context * seal_frac: + sealed += current + current *= r + return {"log_tokens": log_tokens, "naive": naive, "sealed": sealed} + + +def main() -> None: + locked = ASSUMPTIONS["critical_section_locked_s"] + work = ASSUMPTIONS["useful_work_s"] + + print("== global scratchpad lock, LLM call INSIDE the critical section ==") + print(f"critical section = {locked}s; useful work = {work}s/step") + hdr = ( + f"{'agents':>7} {'steps/hr':>9} {'lock util':>10} " + f"{'mean wait':>11} {'p95 wait':>10} {'wait/step':>10}" + ) + print(hdr) + print("-" * len(hdr)) + for n in (2, 4, 8, 16, 32, 64, 128): + r = simulate_lock(n, locked, work) + share = r["mean_wait"] / (r["mean_wait"] + work + locked) + print( + f"{n:>7} {r['steps']:>9} {r['utilisation']:>9.0%} " + f"{r['mean_wait']:>10.1f}s {r['p95_wait']:>9.1f}s {share:>9.0%}" + ) + print() + print( + f"theoretical ceiling with a {locked}s critical section: " + f"{throughput_ceiling(locked):.0f} agent-steps/hour, regardless of fleet size" + ) + + append = ASSUMPTIONS["critical_section_append_s"] + print() + print("== same fleet, LLM call moved OUTSIDE the lock ==") + print(f"lock now guards an append + version bump only (~{append * 1000:.0f}ms)") + print(hdr) + print("-" * len(hdr)) + for n in (2, 8, 32, 128, 512): + r = simulate_lock(n, append, work + locked) + share = r["mean_wait"] / (r["mean_wait"] + work + locked) + print( + f"{n:>7} {r['steps']:>9} {r['utilisation']:>9.0%} " + f"{r['mean_wait']:>10.3f}s {r['p95_wait']:>9.3f}s {share:>9.1%}" + ) + + print() + print("== summary churn (append-only log, not a static document) ==") + print("Agents append notes; every reader needs summaries of all but the tail.") + hdr2 = ( + f"{'notes':>8} {'log tokens':>11} {'naive re-sum tokens':>21} " + f"{'sealed-segment tokens':>23} {'ratio':>13}" + ) + print(hdr2) + print("-" * len(hdr2)) + for n_notes in (100, 1_000, 10_000, 100_000): + c = churn(n_notes) + ratio = ( + f"{c['naive'] / c['sealed']:>6.0f}x" if c["sealed"] else "fits directly" + ) + print( + f"{n_notes:>8,} {c['log_tokens']:>11,} {int(c['naive']):>21,} " + f"{int(c['sealed']):>23,} {ratio:>13}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/simulations/summary_navigation.py b/scripts/simulations/summary_navigation.py new file mode 100644 index 00000000..2a67cd37 --- /dev/null +++ b/scripts/simulations/summary_navigation.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""Is the summary tree a retrieval mechanism or just a map? + +The Opus review (§3) argued the inode-style summary DAG of #485 component 1b +is a *navigation* structure, not a *retrieval* structure: at depth d it retains +only r^d of the original, so a specific fact has ~zero chance of surviving to +the top. The follow-up comment then corrected the geometry: for a fixed +top-level size, top-level fidelity is set by the target size whatever r is — +what r controls is HOW MANY lossy hops you pass through. + +Two questions get quantified here: + +PART A — needle retrieval strategies on one synthetic corpus: + N1 navigate-only: descend the summary tree by matching keywords. + N2 index-only: perfect direct lookup (the FTS/RAG idealisation). + N3 hybrid: index nominates candidates; summaries verify/rank them. + Metrics: P(needle found), expected tokens read. Claim under test: + navigation alone degrades with corpus size; the index is load-bearing. + +PART B — hop distortion at fixed endpoint: + Same corpus, same final top-level size S_top; reach it via few-hard hops + (r=0.1) vs many-gentle hops (r=0.9). Each hop keeps r of what its child + kept and independently drops eps of the survivors (per-hop noise). + Needle survival after d hops ~ (r*(1-eps))^d with d = ceil(ln(N/S_top)/ln(1/r)). + Claim under test (asserted): fewer, harder compressions preserve needles + better than gentle multi-hop chains whenever per-hop noise > 0. + +Corpus model +------------ +* D documents over V words clustered into T topics; each doc draws most words + from one topic (salience signal) plus background noise words. +* K needles: unique rare words, each planted in exactly one document. +* A "summary" of a span = the s most frequent non-noise words in the span + (a keyword sketch parameterised by sketch size s — the stand-in for r). +""" + +from __future__ import annotations + +import math +import random +import statistics + +ASSUMPTIONS = { + "docs": 2_000, + "words_per_doc": 60, + "topics": 20, + "vocab_per_topic": 40, + "needles": 40, + "fanout": 8, # summary-tree branching factor + "sketch_size": 12, # terms kept in each summary node + "noise_rate": 0.25, # fraction of a doc's words that are background + "trials": 300, + "seed": 47, +} + + +class Corpus: + def __init__(self, rng: random.Random): + self.topic_words = { + t: [f"w{t}_{j}" for j in range(ASSUMPTIONS["vocab_per_topic"])] + for t in range(ASSUMPTIONS["topics"]) + } + # Topical LOCALITY: contiguous document bands share a dominant topic + # (with 15% cross-band strays), so sibling summary branches differ -- + # otherwise every bucket covers every topic and there is nothing to + # navigate BY. + self.docs: list[list[str]] = [] + band = max(1, ASSUMPTIONS["docs"] // ASSUMPTIONS["topics"]) + for i in range(ASSUMPTIONS["docs"]): + t_dom = min(i // band, ASSUMPTIONS["topics"] - 1) + t = t_dom if rng.random() > 0.15 else rng.randrange(ASSUMPTIONS["topics"]) + n_signal = int(ASSUMPTIONS["words_per_doc"] * (1 - ASSUMPTIONS["noise_rate"])) + n_noise = ASSUMPTIONS["words_per_doc"] - n_signal + words = [rng.choice(self.topic_words[t]) for _ in range(n_signal)] + words += [f"bg{rng.randrange(1000)}" for _ in range(n_noise)] + self.docs.append(words) + self.needle_doc: dict[str, int] = {} + for k in range(ASSUMPTIONS["needles"]): + word = f"NEEDLE_{k}" + di = rng.randrange(ASSUMPTIONS["docs"]) + self.docs[di].append(word) + self.needle_doc[word] = di + + def sketch(self, doc_ids: list[int]) -> set[str]: + """Keyword sketch of a span: `sketch_size` most frequent real words.""" + counts: dict[str, int] = {} + for di in doc_ids: + for w in self.docs[di]: + if w.startswith("bg") or w.startswith("NEEDLE"): + continue # sketches carry topic words, not noise + counts[w] = counts.get(w, 0) + 1 + ranked = sorted(counts.items(), key=lambda kv: -kv[1]) + return {w for w, _ in ranked[:ASSUMPTIONS["sketch_size"]]} + + +def build_summary_tree(corpus: Corpus) -> list[dict]: + """Leaves up; each level groups fanout siblings into a sketched parent.""" + fanout = ASSUMPTIONS["fanout"] + levels: list[list[dict]] = [ + [{"doc_ids": [i], "keywords": None} for i in range(len(corpus.docs))] + ] + while len(levels[-1]) > fanout: + prev = levels[-1] + parents = [] + for i in range(0, len(prev), fanout): + group = prev[i:i + fanout] + ids = [d for node in group for d in node["doc_ids"]] + parents.append({"doc_ids": ids, "children": group, "keywords": None}) + levels.append(parents) + if len(levels[-1]) > 1: # single root spanning the whole corpus + prev = levels[-1] + ids = [d for node in prev for d in node["doc_ids"]] + levels.append([{"doc_ids": ids, "children": prev, "keywords": None}]) + root = levels[-1][0] + root["keywords"] = corpus.sketch(root["doc_ids"]) + return levels + + +def navigate(corpus: Corpus, levels: list[dict], needle: str) -> tuple[bool, int]: + """Descend by keyword match; returns (found, docs_consulted_tokens).""" + node = levels[-1][0] + reads = 1 # reading the root sketch costs 1 unit + di_target = corpus.needle_doc[needle] + while "children" in node: + best, best_score = None, -1 + for child in node["children"]: + if child["keywords"] is None: + child["keywords"] = corpus.sketch(child["doc_ids"]) + score = sum(1 for w in child["keywords"] + for dw in corpus.docs[di_target] + if not w.startswith("bg") and w == dw) + # cheap proxy: overlap between child sketch and target doc's topic words + if score > best_score: + best, best_score = child, score + reads += 1 + if best is None or best_score <= 0: + return False, reads # lost: term invisible from here + node = best + # leaf level: check the actual documents in the chosen bucket + bucket = [i for i in node["doc_ids"]] + reads += max(1, len(bucket) // 10) # scanning a bucket costs proportionally + return di_target in bucket, reads + + +def part_a(corpus: Corpus, levels: list[dict], rng: random.Random) -> None: + print("== PART A: needle retrieval strategies ==") + hdr = f"{'strategy':>22} {'found%':>7} {'mean read units':>16}" + print(hdr) + print("-" * len(hdr)) + needles = list(corpus.needle_doc) + + found_nav, reads_nav = [], [] + for nd in needles: + ok, r = navigate(corpus, levels, nd) + found_nav.append(ok) + reads_nav.append(r) + print(f"{'N1 navigate-only':>22} {statistics.fmean(found_nav):>6.0%} " + f"{statistics.fmean(reads_nav):>16.1f}") + + # Index idealisation: near-perfect recall, tiny cost, but not perfect + # (tokenisation/staleness misses happen -- assume 3% miss). + index_recall = 0.97 + found_idx = [rng.random() < index_recall for _ in needles] + reads_idx = [3] * len(needles) + print(f"{'N2 index-only':>22} {statistics.fmean(found_idx):>6.0%} " + f"{statistics.fmean(reads_idx):>16.1f}") + + # Hybrid: index first; on an index MISS fall back to summary navigation, + # whose success rate applies to exactly those cases. + hybrid_found = [i or n for i, n in zip(found_idx, found_nav)] + hybrid_reads = [ri if i else rn + for ri, rn, i in zip(reads_idx, reads_nav, found_idx)] + print(f"{'N3 hybrid (idx+nav)':>22} {statistics.fmean(hybrid_found):>6.0%} " + f"{statistics.fmean(hybrid_reads):>16.1f}") + + print("* navigation succeeds only while the needle's TOPIC survives every") + print(" hop; the specific needle never does. The index is load-bearing;") + print(" the tree earns its keep as the fallback lane and the query planner.") + print(f"* hybrid recovers to {statistics.fmean(hybrid_found):.0%} vs index-only " + f"{statistics.fmean(found_idx):.0%} at near-index cost.") + + +def part_b() -> None: + print("\n== PART B: hop distortion at fixed endpoint ==") + eps = 0.05 # per-hop independent drop probability + n_docs = ASSUMPTIONS["docs"] + s_top = math.ceil(n_docs / (ASSUMPTIONS["fanout"] ** 2)) # fixed endpoint + print(f"same corpus -> same top size (~{s_top} docs); " + f"per-hop drop eps={eps}; survival shown RELATIVE to the") + print("endpoint baseline S_top/N (which is the same for every r)") + hdr = f"{'r':>5} {'hops needed':>12} {'relative needle retention':>26}" + print(hdr) + print("-" * len(hdr)) + for r in (0.1, 0.3, 0.5, 0.9): + d = math.ceil(math.log(n_docs / s_top) / math.log(1 / r)) + relative = (1 - eps) ** d + print(f"{r:>5.1f} {d:>12} {relative:>25.0%}") + print("* gentle multi-hop chains lose needles faster than few hard") + print(" compressions to the SAME top size: retention = (1-eps)^d and") + print(" d grows as r -> 1. Prefer fewer, harder compressions.") + + +def main() -> None: + rng = random.Random(ASSUMPTIONS["seed"]) + corpus = Corpus(rng) + levels = build_summary_tree(corpus) + print(f"corpus: {ASSUMPTIONS['docs']} docs x {ASSUMPTIONS['words_per_doc']}w, " + f"{ASSUMPTIONS['topics']} topics, {ASSUMPTIONS['needles']} needles; " + f"summary tree fanout={ASSUMPTIONS['fanout']}, " + f"levels={len(levels)}, sketch={ASSUMPTIONS['sketch_size']} words") + part_a(corpus, levels, rng) + part_b() + + +if __name__ == "__main__": + main() diff --git a/scripts/simulations/system_integration.py b/scripts/simulations/system_integration.py new file mode 100644 index 00000000..50ffa97c --- /dev/null +++ b/scripts/simulations/system_integration.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""The whole #485 proposal as one runnable schematic -- then ablated. + +Component 1c of #485 says: turn the proposed solution into deterministic +operations over objects of the assumed shape, write predictions down, run, +and see what holds. Prior scripts test mechanisms in isolation; this one +assembles them into a mission-level Monte Carlo: + + * branching decomposition (b steps/pipeline, ambiguity rate f0) + * solved-problem library (Zipf demand pool; cache hits are cheaper and + less defective; failed attempts teach) + * review gates (none / declarative-loop / evidential) + * budget governor (global token budget; exhaustion escalates, + never silently succeeds) + +and then ablates subsystems to answer "what works and what doesn't": + + FULL library + evidential review + budgets + -LIBRARY no cross-mission caching + -REVIEW outputs ship unchecked + -BUDGETS depth cap only (silent truncation possible) + NAIVE #485 as literally read: declarative unbounded review, depth cap, + no budgets, library resets every mission + +Outputs: success-within-budget rate, defective-shipped rate (declared success +but flawed result), mean token spend -- plus a scan of FULL over f0 x budget. +""" + +from __future__ import annotations + +import math +import random +import statistics + +ASSUMPTIONS = { + "types": 150, + "zipf_skew": 1.2, + "hard_rate": 0.35, + "steps_per_root": 8, + "fanout": 4, + "missions_per_fleet": 40, + "fleets": 24, + "node_cost_atomic": 2_000, + "node_cost_cached": 800, + "decomp_cost": 1_500, + "review_tokens_round": 8_000, + "p_detect": 0.6, + "lambda_fp_prose": 1.0, + "fp_evidence_ratio": 0.20, + "lambda_drift": 0.5, + "round_cap": 12, + "defect_rate_novel": 0.30, + "defect_rate_cached": 0.08, + "budget_multiple": 2.0, + "depth_cap_naive": 8, + "seed": 71, +} + + +def poisson(rng: random.Random, lam: float) -> int: + if lam <= 0: + return 0 + limit, k, p = math.exp(-lam), 0, 1.0 + while True: + p *= rng.random() + if p <= limit: + return k + k += 1 + + +def binom(n: int, p: float, rng: random.Random) -> int: + return sum(1 for _ in range(n) if rng.random() < p) + + +class Fleet: + """One replicated world: demand law, library, review mode, budget mode.""" + + def __init__(self, rng: random.Random, use_library: bool, review: str, + use_budgets: bool): + self.rng = rng + self.use_library = use_library + self.review = review + self.use_budgets = use_budgets + n = ASSUMPTIONS["types"] + raw = [1 / (r + 1) ** ASSUMPTIONS["zipf_skew"] for r in range(n)] + total = sum(raw) + self.weights = [p / total for p in raw] + self.hard = {i for i in range(n) + if rng.random() < ASSUMPTIONS["hard_rate"]} + self.library: set[int] = set() + + def draw_type(self) -> int: + return self.rng.choices(range(len(self.weights)), self.weights)[0] + + def review_artifact(self, defects: int) -> tuple[int, int]: + """Configured gate on one node artifact -> (residual defects, tokens).""" + if self.review == "none": + return defects, 0 + lam_ev = ASSUMPTIONS["lambda_fp_prose"] * ASSUMPTIONS["fp_evidence_ratio"] + open_fps = 0 + rnd = 0 + for rnd in range(1, ASSUMPTIONS["round_cap"] + 1): + found = binom(defects, ASSUMPTIONS["p_detect"], self.rng) + if self.review == "evidential": + found = round(found * 0.85) + fps = poisson(self.rng, lam_ev) + drift = 0 + else: + fps = poisson(self.rng, ASSUMPTIONS["lambda_fp_prose"]) + drift = (poisson(self.rng, ASSUMPTIONS["lambda_drift"]) + if self.review == "declarative" else 0) + if found + fps + drift == 0 and open_fps == 0: + return defects, rnd * ASSUMPTIONS["review_tokens_round"] + defects -= found + defects += binom(found, 0.15, self.rng) + churn = fps + drift + open_fps += churn + defects += binom(churn, 0.15, self.rng) + open_fps -= binom(open_fps, 0.85, self.rng) + return defects, rnd * ASSUMPTIONS["review_tokens_round"] + + def run_mission(self) -> dict: + """One mission through decomposition + execution + gates.""" + b = ASSUMPTIONS["steps_per_root"] + budget = None + reserve = 0.0 + if self.use_budgets: + expected_nodes = 40 + budget = ASSUMPTIONS["budget_multiple"] * expected_nodes * ( + ASSUMPTIONS["node_cost_atomic"] * 0.7 + + ASSUMPTIONS["review_tokens_round"] * 1.2) + reserve = budget * 0.3 + budget -= reserve + + touched: list[int] = [] + spent = 0.0 + residual_defects = 0 + nodes = 0 + exhausted = False + frontier = [(self.draw_type(), 1)] + while frontier: + typ, depth = frontier.pop() + nodes += 1 + if not self.use_budgets and depth > ASSUMPTIONS["depth_cap_naive"]: + residual_defects += 1 # silent truncation ships a flaw + continue + if typ in self.library: + cost = ASSUMPTIONS["node_cost_cached"] + defects = 1 if self.rng.random() < ASSUMPTIONS["defect_rate_cached"] else 0 + else: + cost = ASSUMPTIONS["node_cost_atomic"] + defects = 1 if self.rng.random() < ASSUMPTIONS["defect_rate_novel"] else 0 + res_def, rev_tok = self.review_artifact(defects) + step_cost = cost + rev_tok + if self.use_budgets: + if step_cost > budget: + topup = min(step_cost - budget, reserve) + reserve -= topup + budget += topup + if step_cost > budget: + exhausted = True + break + budget -= step_cost + spent += step_cost + residual_defects += res_def + if typ not in self.library: + if typ in self.hard and depth < 6: + touched.append(typ) + frontier.extend((self.draw_type(), depth + 1) + for _ in range(ASSUMPTIONS["fanout"])) + if self.use_library: + self.library.update(touched) # failed attempts teach too + + return { + "ok": not exhausted, + "spent": spent, + "residual": residual_defects, + "nodes": nodes, + "library": len(self.library), + } + + +def run_config(name: str, use_library: bool, review: str, + use_budgets: bool) -> dict: + # str hash() is process-randomized; derive the seed deterministically + rng = random.Random(ASSUMPTIONS["seed"] + sum(map(ord, name))) + rows = [] + for _ in range(ASSUMPTIONS["fleets"]): + fleet = Fleet(rng, use_library, review, use_budgets) + for _mi in range(ASSUMPTIONS["missions_per_fleet"]): + rows.append(fleet.run_mission()) + n = len(rows) + ok_rows = [r for r in rows if r["ok"]] + return { + "name": name, + "success": len(ok_rows) / n, + "defective_of_declared": shipped_defected_rate(ok_rows), + "mean_spend": statistics.fmean(r["spent"] for r in rows), + "_n": n, + } + + +def shipped_defected_rate(ok_rows: list[dict]) -> float: + if not ok_rows: + return float("nan") + return sum(1 for r in ok_rows if r["residual"] > 0) / len(ok_rows) + + +def ablation_table() -> None: + configs = [ + ("FULL", True, "evidential", True), + ("-LIBRARY", False, "evidential", True), + ("-REVIEW", True, "none", True), + ("-BUDGETS", True, "evidential", False), + ("NAIVE", False, "declarative", False), + ] + hdr = (f"{'config':>10} {'success%':>9} {'defect%|declared':>17} " + f"{'mean spend':>12} {'missions':>9}") + print(hdr) + print("-" * len(hdr)) + for name, lib, rev, bud in configs: + r = run_config(name, lib, rev, bud) + print(f"{name:>10} {r['success']:>8.0%} {r['defective_of_declared']:>16.0%} " + f"{r['mean_spend']:>12,.0f} {r['_n']:>9,}") + + +def phase_scan() -> None: + saved = {k: v for k, v in ASSUMPTIONS.items()} + print("\n== FULL config: success%% over hardness x budget ==") + grid_hard = [0.20, 0.35, 0.50] + grid_bud = [0.25, 0.5, 1.0] # tight budgets: where the governor binds + row_label = "hard vs budget" + hdr = f"{row_label:>14}" + "".join(f"{b:>9.1f}x" for b in grid_bud) + print(hdr) + for h in grid_hard: + ASSUMPTIONS["hard_rate"] = h + cells = [] + for bm in grid_bud: + ASSUMPTIONS["budget_multiple"] = bm + rng = random.Random(ASSUMPTIONS["seed"] + int(h * 100) + int(bm * 10)) + outcomes = [] + for _ in range(ASSUMPTIONS["fleets"] // 2): + fleet = Fleet(rng, True, "evidential", True) + for _mi in range(ASSUMPTIONS["missions_per_fleet"]): + outcomes.append(fleet.run_mission()["ok"]) + cells.append(statistics.fmean(outcomes)) + print(f"{h:>13.2f}" + "".join(f"{c:>8.0%}" for c in cells)) + ASSUMPTIONS.clear() + ASSUMPTIONS.update(saved) + + +def main() -> None: + print("== integrated system Monte Carlo (component 1c applied to #485) ==") + print(f"b={ASSUMPTIONS['steps_per_root']}, fresh-type hardness " + f"f0={ASSUMPTIONS['hard_rate']} -> m0={ASSUMPTIONS['steps_per_root'] * ASSUMPTIONS['hard_rate']:.1f}; " + f"novel-defect rate {ASSUMPTIONS['defect_rate_novel']}, " + f"cached {ASSUMPTIONS['defect_rate_cached']}; " + f"{ASSUMPTIONS['fleets']} fleets x {ASSUMPTIONS['missions_per_fleet']} missions/config") + ablation_table() + phase_scan() + print("\n== reading ==") + print("* Every ablation should HURT somewhere: library cuts spend over time,") + print(" review cuts shipped defects, budgets convert silent truncation into") + print(" loud failure. If removing one barely moves anything, that subsystem") + print(" is not earning its complexity yet.") + print("* NAIVE vs FULL is the whole argument of this issue thread in one row:") + print(" the difference is not intelligence, it is governance.") + + +if __name__ == "__main__": + main() diff --git a/src/orchestrator/__init__.py b/src/orchestrator/__init__.py index 4d13fc1e..474e8d98 100644 --- a/src/orchestrator/__init__.py +++ b/src/orchestrator/__init__.py @@ -64,14 +64,20 @@ # --- Models --- "ModelRegistry": ".models.model_registry", "get_model_registry": ".models.registry_singleton", - # --- Model integrations (each needs its provider extra) --- - "HuggingFaceModel": ".integrations.huggingface_model", - "OllamaModel": ".integrations.ollama_model", + # --- Model integrations --- + # The retired providers (Anthropic/OpenAI/Google/Ollama/local-HF, #430) + # are deliberately absent here: not importable, not advertised. # Dartmouth Chat needs no extra: it is an OpenAI-compatible HTTP gateway # spoken with aiohttp (a core dep), and it serves free models. "DartmouthModel": ".models.dartmouth_model", "DartmouthProvider": ".models.providers.dartmouth_provider", "resolve_dartmouth_api_key": ".models.dartmouth_credentials", + # The HuggingFace Inference API (#484): the hosted OpenAI-compatible + # router, also spoken with plain aiohttp. `HuggingFaceInferenceModel` is + # named to stay distinct from the retired local-transformers adapter. + "HuggingFaceInferenceModel": ".models.huggingface_model", + "HuggingFaceProvider": ".models.providers.huggingface_provider", + "resolve_huggingface_api_key": ".models.huggingface_credentials", # --- State --- "StateManager": ".state.state_manager", # --- Tools / MCP --- @@ -82,13 +88,10 @@ "compile": "._api", "compile_async": "._api", "OrchestratorPipeline": "._api", - # --- Optional API layer --- - "PipelineAPI": ".api", - "AdvancedPipelineCompiler": ".api", - "PipelineExecutor": ".api", - "create_pipeline_api": ".api", - "create_advanced_pipeline_compiler": ".api", - "create_pipeline_executor": ".api", + # The api/ layer (PipelineAPI et al.) is frozen and was removed from the + # public surface with the provider retirement: it imported the competing + # model registry that #430 deleted, so those names could no longer + # resolve. Its own removal is a later #430 cut. # --- Validation, with its findings --- "validate_pipeline_file": ".validation.pipeline_report", "validate_pipeline_text": ".validation.pipeline_report", diff --git a/src/orchestrator/_api.py b/src/orchestrator/_api.py index e00f4a09..c187a34d 100644 --- a/src/orchestrator/_api.py +++ b/src/orchestrator/_api.py @@ -49,6 +49,10 @@ def init_models(config_path: str = None) -> ModelRegistry: #: must degrade to "no Dartmouth models" rather than stall the pipeline. _DARTMOUTH_DISCOVERY_TIMEOUT = 10.0 +#: Same constraint as the Dartmouth catalog: a slow router must degrade to +#: "no HuggingFace models" rather than stall the path to a user's first model. +_HUGGINGFACE_DISCOVERY_TIMEOUT = 10.0 + def _register_free_dartmouth_models(registry: ModelRegistry) -> int: """Register the free Dartmouth Chat models, if a credential is present. @@ -112,58 +116,100 @@ def _register_free_dartmouth_models(registry: ModelRegistry) -> int: return registered +def _register_free_huggingface_models(registry: ModelRegistry) -> int: + """Register the HuggingFace router models that currently have a free route. + + Like Dartmouth, these models are not read from ``models.yaml``: a free + route is a promo that starts and ends upstream, so the live catalog is the + only trustworthy source and **only** its zero-cost entries are registered. + Each registered model is pinned to its free provider -- an unpinned + request routes ``:fastest``, which may bill the account. + + Never raises: a missing credential, an unreachable router or a slow one + all mean "no HuggingFace models available", exactly as a missing Ollama + install does. + + Returns: + How many models were registered. + """ + from .models.huggingface_credentials import resolve_huggingface_api_key + from .models.huggingface_model import DEFAULT_BASE_URL as HF_DEFAULT_BASE_URL + from .models.huggingface_model import HuggingFaceInferenceModel + from .models.providers.huggingface_provider import ( + fetch_catalog_sync, + free_models_from_catalog, + free_route_from_catalog, + ) + + credential = resolve_huggingface_api_key(required=False) + if credential is None: + logger.debug("No HuggingFace token found; skipping HuggingFace models") + return 0 + + try: + catalog = fetch_catalog_sync( + HF_DEFAULT_BASE_URL, credential.key, _HUGGINGFACE_DISCOVERY_TIMEOUT + ) + free = free_models_from_catalog(catalog) + except Exception as exc: # noqa: BLE001 - discovery is best-effort + logger.info("Could not reach the HuggingFace catalog, skipping: %s", exc) + return 0 + + from .models.huggingface_model import HuggingFaceModelError + + registered = 0 + for model_id, cost in free.items(): + try: + registry.register_model( + HuggingFaceInferenceModel( + name=model_id, + api_key=credential.key, + cost=cost, + route=free_route_from_catalog(catalog[model_id]), + ) + ) + registered += 1 + except (HuggingFaceModelError, ValueError) as exc: + # Deliberately NOT `except Exception`: one unusable catalog entry + # must not stop the rest, but a programming error here should + # surface rather than be logged as a per-model hiccup. + logger.warning( + "Could not register HuggingFace model %s: %s", model_id, exc + ) + + if registered: + logger.info( + "Registered %d free-routed HuggingFace models (%d in catalog)", + registered, + len(catalog), + ) + return registered + + def populate_model_registry(registry: ModelRegistry) -> ModelRegistry: """Register every model the current environment can actually serve. - Reads ``~/.orchestrator/.env`` for provider credentials, loads - ``models.yaml`` and probes for a local Ollama install. This is the step - that touches the user's credentials, so it must only run when a model is - genuinely required. - """ - import os + The supported providers are Dartmouth Chat (registered from its live + catalog when a credential is present) and the HuggingFace Inference API + (#484). A ``models.yaml`` written before the provider retirement (#430) + may still name ``ollama``/``openai``/``anthropic``/``google``/ + ``huggingface`` sources; each such entry is skipped with a warning, never + raised on -- an old config file is not an error. - from .integrations.anthropic_model import AnthropicModel - from .integrations.google_model import GoogleModel - from .integrations.openai_model import OpenAIModel - from .utils.model_utils import check_ollama_installed + This is the step that touches the user's credentials, so it must only run + when a model is genuinely required. + """ from .utils.model_config_loader import get_model_config_loader - from .utils.api_keys_flexible import load_api_keys_optional logger.info("Initializing model pool") - # Load available API keys (doesn't require all keys to be present) - available_keys = load_api_keys_optional() - if available_keys: - logger.info("Found API keys for: %s", ", ".join(sorted(available_keys))) - # Also set them in environment for backward compatibility - provider_env_map = { - "anthropic": "ANTHROPIC_API_KEY", - "google": "GOOGLE_AI_API_KEY", - "huggingface": "HF_TOKEN", - "openai": "OPENAI_API_KEY", - } - for provider, api_key in available_keys.items(): - env_var = provider_env_map.get(provider) - if env_var and not os.environ.get(env_var): - os.environ[env_var] = api_key - else: - logger.info("No API keys found - only local models will be available") + _register_free_dartmouth_models(registry) + _register_free_huggingface_models(registry) - # Load model configuration using the new loader loader = get_model_config_loader() config = loader.load_config() models_config = config.get("models", {}) - # Check if Ollama is installed - ollama_available = check_ollama_installed() - if not ollama_available: - logger.info( - "Ollama not found - Ollama models unavailable (install from https://ollama.ai)" - ) - - _register_free_dartmouth_models(registry) - - # Process each model in configuration (list format) if not isinstance(models_config, list): logger.warning( "Invalid models configuration format: expected a list, got %s", @@ -171,117 +217,18 @@ def populate_model_registry(registry: ModelRegistry) -> ModelRegistry: ) models_config = [] - # Process each model for model_config in models_config: provider = model_config.get("source") name = model_config.get("name") - - # Parse size - size_str = str(model_config.get("size", "1b")) - if size_str.endswith("b"): - size_billions = float(size_str[:-1]) - else: - size_billions = float(size_str) - - # Get expertise - expertise = model_config.get("expertise", ["general"]) - if not provider or not name: continue - - try: - if provider == "ollama": - if not ollama_available: - continue - - # Register model for lazy loading (will be downloaded on first use) - # Use a lazy wrapper that doesn't check availability yet - from .integrations.lazy_ollama_model import LazyOllamaModel - - model = LazyOllamaModel(model_name=name, timeout=60) - # Add dynamic attributes for model selection - setattr(model, "_expertise", expertise) - setattr(model, "_size_billions", size_billions) - registry.register_model(model) - logger.info( - "Registered Ollama model %s (%sB) - downloads on first use", - name, - size_billions, - ) - - elif provider == "huggingface": - # Skip HuggingFace models if disabled via environment variable - if ( - os.environ.get("ORCHESTRATOR_SKIP_HUGGINGFACE", "").lower() - == "true" - ): - continue - - # Check if transformers is available - try: - import importlib.util - - if importlib.util.find_spec("transformers") is not None: - # Register for lazy loading (will be downloaded on first use) - from .integrations.lazy_huggingface_model import ( - LazyHuggingFaceModel, - ) - - hf_model = LazyHuggingFaceModel(model_name=name) - # Add dynamic attributes for model selection - setattr(hf_model, "_expertise", expertise) - setattr(hf_model, "_size_billions", size_billions) - registry.register_model(hf_model) - logger.info( - "Registered HuggingFace model %s (%sB) - downloads on first use", - name, - size_billions, - ) - except ImportError: - logger.info( - "HuggingFace model %s configured but transformers is not " - "installed (pip install 'py-orc[multimedia]')", - name, - ) - except Exception as e: - logger.warning( - "Could not register HuggingFace model %s: %s", name, e - ) - - elif provider == "openai" and "openai" in available_keys: - # Only register if API key is available - model = OpenAIModel(model_name=name, api_key=available_keys["openai"]) - # Add dynamic attributes for model selection - setattr(model, "_expertise", expertise) - setattr(model, "_size_billions", size_billions) - registry.register_model(model) - logger.info("Registered OpenAI model %s (%sB)", name, size_billions) - - elif provider == "anthropic" and "anthropic" in available_keys: - # Only register if API key is available - model = AnthropicModel( - model_name=name, api_key=available_keys["anthropic"] - ) - # Add dynamic attributes for model selection - setattr(model, "_expertise", expertise) - setattr(model, "_size_billions", size_billions) - registry.register_model(model) - logger.info("Registered Anthropic model %s (%sB)", name, size_billions) - - elif provider == "google" and "google" in available_keys: - # Only register if API key is available - model = GoogleModel(model_name=name, api_key=available_keys["google"]) - # Add dynamic attributes for model selection - setattr(model, "_expertise", expertise) - setattr(model, "_size_billions", size_billions) - registry.register_model(model) - logger.info("Registered Google model %s (%sB)", name, size_billions) - - except Exception as e: - # One line per provider, naming the real cause. Registering a - # model is best-effort: a missing provider SDK must not stop the - # models that *are* usable from being registered. - logger.warning("Could not register %s model %s: %s", provider, name, e) + logger.warning( + "Skipping models.yaml entry %r: provider %r was retired (#430). " + "Supported providers: dartmouth (live catalog) and the " + "HuggingFace Inference API (#484).", + name, + provider, + ) registered = registry.list_models() if registered: diff --git a/src/orchestrator/config/models.yaml b/src/orchestrator/config/models.yaml index ee78c35a..344f6e05 100644 --- a/src/orchestrator/config/models.yaml +++ b/src/orchestrator/config/models.yaml @@ -1,400 +1,11 @@ # Model configuration for Orchestrator -# This file defines available models and their properties - -models: - # Ollama models (downloaded on first use) - - source: ollama - name: deepseek-r1:1.5b - expertise: - - reasoning - - code - - math - size: 1.5b - - - source: ollama - name: deepseek-r1:8b - expertise: - - reasoning - - code - - math - size: 8b - - - source: ollama - name: deepseek-r1:32b - expertise: - - reasoning - - code - - math - - analysis - size: 32b - - - source: ollama - name: gemma3:1b - expertise: - - general - - fast - - compact - size: 1b - - - source: ollama - name: gemma3:4b - expertise: - - general - - reasoning - size: 4b - - - source: ollama - name: gemma3:12b - expertise: - - general - - reasoning - - analysis - size: 12b - - - source: ollama - name: gemma3:27b - expertise: - - general - - reasoning - - analysis - size: 27b - - - source: ollama - name: gemma3n:e4b - expertise: - - general - - efficient - size: 4b - - - source: ollama - name: llama3.1:8b - expertise: - - general - - reasoning - - multilingual - size: 8b - - - source: ollama - name: llama3.2:1b - expertise: - - general - - fast - size: 1b - - - source: ollama - name: llama3.2:3b - expertise: - - general - - fast - size: 3b - - - source: ollama - name: mistral:7b - expertise: - - general - - code - size: 7b - - - source: ollama - name: qwen2.5-coder:7b - expertise: - - code - - programming - size: 7b - - - source: ollama - name: qwen2.5-coder:14b - expertise: - - code - - programming - - analysis - size: 14b - - - source: ollama - name: qwen2.5-coder:32b - expertise: - - code - - programming - - analysis - - reasoning - size: 32b - - # HuggingFace models (downloaded on first use) - # Top instruct models - - source: huggingface - name: meta-llama/Llama-3.2-11B-Vision-Instruct - expertise: - - general - - vision - - multimodal - size: 11b - - - source: huggingface - name: meta-llama/Llama-3.1-8B-Instruct - expertise: - - general - - reasoning - - multilingual - size: 8b - - - source: huggingface - name: Qwen/Qwen2.5-1.5B-Instruct - expertise: - - general - - multilingual - - fast - size: 1.5b - - - source: huggingface - name: Qwen/Qwen2-VL-7B-Instruct - expertise: - - general - - vision - - multimodal - size: 7b - - - source: huggingface - name: tencent/Hunyuan-A13B-Instruct - expertise: - - general - - reasoning - - math - - science - size: 13b - - - source: huggingface - name: microsoft/Phi-3.5-mini-instruct - expertise: - - reasoning - - code - - compact - size: 3.8b - - - source: huggingface - name: SmolLM-1.7B-Instruct - expertise: - - general - - compact - - fast - size: 1.7b - - - source: huggingface - name: stabilityai/stable-code-instruct-3b - expertise: - - code - - programming - - compact - size: 3b - - # Top coding models - - source: huggingface - name: Qwen/Qwen2.5-Coder-32B-Instruct - expertise: - - code - - programming - - reasoning - size: 32b - - - source: huggingface - name: deepseek-ai/DeepSeek-R1-Distill-Qwen-32B - expertise: - - reasoning - - code - - math - size: 32b - - - source: huggingface - name: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B - expertise: - - reasoning - - code - - compact - size: 1.5b - - - source: huggingface - name: Qwen/Qwen2.5-Coder-7B-Instruct - expertise: - - code - - programming - size: 7b - - - source: huggingface - name: codellama/CodeLlama-7b-Instruct-hf - expertise: - - code - - programming - size: 7b - - - source: huggingface - name: bigcode/starcoder2-15b - expertise: - - code - - programming - size: 15b - - - source: huggingface - name: WizardLM/WizardCoder-Python-34B-V1.0 - expertise: - - code - - python - - programming - size: 34b - - # OpenAI models (require OPENAI_API_KEY) - # GPT-5 Series (Latest - Released Jan 2025) - - source: openai - name: gpt-5 - expertise: - - general - - reasoning - - code - - analysis - - instruction-following - - vision - - multimodal - size: 2000b # Estimated - - - source: openai - name: gpt-5-mini - expertise: - - general - - fast - - efficient - - reasoning - size: 100b # Estimated - - - source: openai - name: gpt-5-nano - expertise: - - fast - - compact - - efficient - size: 10b # Estimated - - # Anthropic models (require ANTHROPIC_API_KEY) - # Claude 4 Series - - source: anthropic - name: claude-opus-4-20250514 - expertise: - - general - - reasoning - - analysis - - code - - complex-tasks - size: 2500b # Estimated - - - source: anthropic - name: claude-sonnet-4-20250514 - expertise: - - general - - reasoning - - efficient - size: 600b # Estimated - - # Claude 3.x Series (Deprecated - use 4th generation models instead) - # - source: anthropic - # name: claude-3-7-sonnet-20250219 - # expertise: - # - general - # - reasoning - # - analysis - # - extended-thinking - # size: 400b # Estimated - # - # - source: anthropic - # name: claude-3-5-sonnet-20241022 - # expertise: - # - general - # - fast - # - balanced - # size: 200b # Estimated - # - # - source: anthropic - # name: claude-3-5-haiku-20241022 - # expertise: - # - fast - # - efficient - # - compact - # size: 20b # Estimated - # - # # Legacy models - # - source: anthropic - # name: claude-3-opus-20240229 - # expertise: - # - general - # - reasoning - # - analysis - # size: 2000b # Estimated - # - # - source: anthropic - # name: claude-3-haiku-20240307 - # expertise: - # - fast - # - efficient - # size: 20b # Estimated - - # Google models (require GOOGLE_API_KEY) - # Gemini 2.5 Series - - source: google - name: gemini-2.5-pro - expertise: - - general - - reasoning - - code - - math - - stem - - long-context - size: 1500b # Estimated - - - source: google - name: gemini-2.5-flash - expertise: - - general - - fast - - efficient - - thinking - size: 80b # Estimated - - - source: google - name: gemini-2.5-flash-lite-preview-06-17 - expertise: - - fast - - efficient - - compact - - classification - size: 8b # Estimated - - # Gemini 2.0 Series - - source: google - name: gemini-2.0-flash - expertise: - - general - - fast - - multimodal - - native-tools - size: 70b # Estimated - - - source: google - name: gemini-2.0-flash-lite - expertise: - - fast - - efficient - - compact - size: 8b # Estimated - -defaults: - expertise_preferences: - code: qwen2.5-coder:32b - reasoning: deepseek-r1:32b - fast: llama3.2:1b - general: llama3.1:8b - analysis: gemma3:27b - compact: gemma3:1b - vision: meta-llama/Llama-3.2-11B-Vision-Instruct - - fallback_chain: - - llama3.1:8b - - gemma3:27b - - mistral:7b - - llama3.2:3b - - llama3.2:1b \ No newline at end of file +# +# The shipped default pool is intentionally EMPTY. The supported providers +# are Dartmouth Chat -- whose free models register from the live catalog when +# DARTMOUTH_CHAT_API_KEY is present, and are deliberately not listed here +# because which models are free changes upstream -- and the HuggingFace +# Inference API (#484). The Anthropic / OpenAI / Google / Ollama / +# local-HuggingFace adapters were retired (#430); entries naming those +# sources are skipped with a warning at population time. + +models: [] diff --git a/src/orchestrator/control_systems/hybrid_control_system.py b/src/orchestrator/control_systems/hybrid_control_system.py index 07447ea9..58919853 100644 --- a/src/orchestrator/control_systems/hybrid_control_system.py +++ b/src/orchestrator/control_systems/hybrid_control_system.py @@ -1137,9 +1137,11 @@ async def _handle_analyze_text(self, task: Task, context: Dict[str, Any]) -> Any # Get specific model model = self.model_registry.get_model(model_spec) else: - # Fallback to creating a model directly - from ..models.openai_model import OpenAIModel - model = OpenAIModel(name="gpt-4") + # No registry, no model. There is no fallback provider to + # construct: the retired adapters used to be built here silently, + # which made a misconfigured run spend money on an API the user + # never chose (#430). + model = None if not model: return { diff --git a/src/orchestrator/integrations/__init__.py b/src/orchestrator/integrations/__init__.py index 9f2e7717..3f72287e 100644 --- a/src/orchestrator/integrations/__init__.py +++ b/src/orchestrator/integrations/__init__.py @@ -1,13 +1,8 @@ -"""Model integrations for the orchestrator framework.""" +"""Model integrations. -from .anthropic_model import AnthropicModel -from .google_model import GoogleModel -from .huggingface_model import HuggingFaceModel -from .openai_model import OpenAIModel - -__all__ = [ - "OpenAIModel", - "AnthropicModel", - "GoogleModel", - "HuggingFaceModel", -] +The provider adapters that lived here (Anthropic, OpenAI, Google, Ollama, +local HuggingFace) were retired under #430. The supported providers are +Dartmouth Chat (:mod:`orchestrator.models.providers.dartmouth_provider`) and +the HuggingFace Inference API (#484). What remains is provider-independent +support code. +""" diff --git a/src/orchestrator/integrations/anthropic_model.py b/src/orchestrator/integrations/anthropic_model.py deleted file mode 100644 index 2f5414df..00000000 --- a/src/orchestrator/integrations/anthropic_model.py +++ /dev/null @@ -1,557 +0,0 @@ -"""Anthropic model integration for the orchestrator framework.""" - -from __future__ import annotations - -import logging - -import json -import os -from typing import Any, Dict, List, Optional - -try: - import anthropic - from anthropic import Anthropic - - ANTHROPIC_AVAILABLE = True -except ImportError: - ANTHROPIC_AVAILABLE = False - Anthropic = None - anthropic = None - -from orchestrator.core.model import ( - Model, - ModelCapabilities, - ModelMetrics, - ModelRequirements, -) - - -logger = logging.getLogger(__name__) - -class AnthropicModel(Model): - """Anthropic model implementation.""" - - # Model configurations - MODEL_CONFIGS = { - "claude-3-5-sonnet-20241022": { - "capabilities": ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "vision", - "summarize", - "extract", - ], - context_window=200000, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=4096, - temperature_range=(0.0, 1.0), - ), - "requirements": ModelRequirements( - memory_gb=0.1, - cpu_cores=1, - disk_space_gb=0.1, - min_python_version="3.8", - requires_gpu=False, - ), - "metrics": ModelMetrics( - latency_p50=2.5, - latency_p95=6.0, - throughput=12.0, - accuracy=0.97, - cost_per_token=0.000015, - success_rate=0.99, - ), - }, - "claude-3-opus-20240229": { - "capabilities": ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "vision", - "summarize", - "extract", - ], - context_window=200000, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=4096, - temperature_range=(0.0, 1.0), - ), - "requirements": ModelRequirements( - memory_gb=0.1, - cpu_cores=1, - disk_space_gb=0.1, - min_python_version="3.8", - requires_gpu=False, - ), - "metrics": ModelMetrics( - latency_p50=3.0, - latency_p95=7.0, - throughput=8.0, - accuracy=0.98, - cost_per_token=0.000075, - success_rate=0.99, - ), - }, - "claude-3-haiku-20240307": { - "capabilities": ModelCapabilities( - supported_tasks=["generate", "analyze", "transform", "code", "summarize", "extract"], - context_window=200000, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=4096, - temperature_range=(0.0, 1.0), - ), - "requirements": ModelRequirements( - memory_gb=0.1, - cpu_cores=1, - disk_space_gb=0.1, - min_python_version="3.8", - requires_gpu=False, - ), - "metrics": ModelMetrics( - latency_p50=1.0, - latency_p95=2.5, - throughput=20.0, - accuracy=0.90, - cost_per_token=0.00000125, - success_rate=0.98, - ), - }, - } - - def __init__( - self, - model_name: str = "claude-3-5-sonnet-20241022", - api_key: Optional[str] = None, - base_url: Optional[str] = None, - max_retries: int = 3, - timeout: float = 30.0, - **kwargs: Any, - ) -> None: - """ - Initialize Anthropic model. - - Args: - model_name: Anthropic model name - api_key: Anthropic API key (if not provided, will use ANTHROPIC_API_KEY env var) - base_url: Base URL for API calls - max_retries: Maximum number of retries for failed requests - timeout: Request timeout in seconds - **kwargs: Additional arguments passed to parent class - """ - global ANTHROPIC_AVAILABLE, anthropic, Anthropic - if not ANTHROPIC_AVAILABLE: - # Try to install on demand - import subprocess - import sys - - # Installing at runtime reaches the network and mutates the - # live environment on an ordinary pipeline run, so it is - # gated behind the same explicit opt-in as utils.auto_install. - from ..utils.auto_install import ( - AUTO_INSTALL_ENV_VAR, - auto_install_enabled, - ) - - if not auto_install_enabled(): - # This is the whole story: the library is absent and we are - # not allowed to fetch it. Re-wrapping it as "Failed to - # install ..." reported one cause twice and named the wrong - # one -- nothing was attempted, so nothing failed to install. - raise ImportError( - "Anthropic library is not installed. Install it with: " - "pip install 'py-orc[anthropic]' " - f"(or set {AUTO_INSTALL_ENV_VAR}=1 to install automatically)." - ) - - logger.info("Anthropic library not found; installing it.") - try: - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "anthropic"] - ) - # Re-import after installation - import anthropic - from anthropic import Anthropic - - ANTHROPIC_AVAILABLE = True - except Exception as e: - raise ImportError( - f"Could not install the Anthropic library automatically: {e}. " - "Install it with: pip install anthropic" - ) from e - - # Get model configuration - config = self.MODEL_CONFIGS.get( - model_name, self.MODEL_CONFIGS["claude-3-5-sonnet-20241022"] - ) - - super().__init__( - name=model_name, - provider="anthropic", - capabilities=config["capabilities"], - requirements=config["requirements"], - metrics=config["metrics"], - **kwargs, - ) - - # Initialize Anthropic client - self.api_key = api_key or os.getenv("ANTHROPIC_API_KEY") - if not self.api_key: - raise ValueError( - "Anthropic API key not provided. Set ANTHROPIC_API_KEY environment variable " - "or pass api_key parameter." - ) - - self.client = Anthropic( - api_key=self.api_key, - base_url=base_url, - max_retries=max_retries, - timeout=timeout, - ) - - self.model_name = model_name - self.max_retries = max_retries - self.timeout = timeout - - # Rate limiting - self._rate_limiter = None - self._last_request_time = 0.0 - self._min_request_interval = 0.1 # 10 requests per second max - - async def generate( - self, - prompt: str, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """ - Generate text using Anthropic API. - - Args: - prompt: Input prompt (can be string or list of content blocks) - temperature: Sampling temperature (0.0 to 1.0) - max_tokens: Maximum tokens to generate - **kwargs: Additional Anthropic parameters (including 'messages' for multimodal) - - Returns: - Generated text - """ - await self._rate_limit() - - # Validate temperature - temp_min, temp_max = self.capabilities.temperature_range - if not temp_min <= temperature <= temp_max: - raise ValueError( - f"Temperature {temperature} not in valid range {self.capabilities.temperature_range}" - ) - - # Set default max_tokens if not provided - if max_tokens is None: - max_tokens = self.capabilities.max_tokens - - try: - # Check if multimodal messages are provided - if "messages" in kwargs: - messages = kwargs.pop("messages") - else: - # Create messages from prompt - messages = [{"role": "user", "content": prompt}] - - response = self.client.messages.create( - model=self.model_name, - max_tokens=max_tokens, - temperature=temperature, - messages=messages, - **kwargs, - ) - - return response.content[0].text if response.content else "" - - except Exception as e: - raise RuntimeError(f"Anthropic API error: {str(e)}") from e - - async def generate_structured( - self, - prompt: str, - schema: Dict[str, Any], - temperature: float = 0.7, - **kwargs: Any, - ) -> Dict[str, Any]: - """ - Generate structured output using Anthropic API. - - Args: - prompt: Input prompt - schema: JSON schema for output structure - temperature: Sampling temperature - **kwargs: Additional Anthropic parameters - - Returns: - Structured output matching schema - """ - if not self.capabilities.supports_structured_output: - raise ValueError(f"Model {self.name} does not support structured output") - - await self._rate_limit() - - # Create prompt with schema instructions - structured_prompt = f""" - {prompt} - - Please respond with a JSON object that matches this schema: - {json.dumps(schema, indent=2)} - - Return only the JSON object, no additional text. - """ - - try: - response = self.client.messages.create( - model=self.model_name, - max_tokens=self.capabilities.max_tokens, - temperature=temperature, - messages=[{"role": "user", "content": structured_prompt}], - **kwargs, - ) - - content = response.content[0].text if response.content else "{}" - - # Parse JSON response - try: - return json.loads(content) - except json.JSONDecodeError: - # Try to extract JSON from response - import re - - json_match = re.search(r"\{.*\}", content, re.DOTALL) - if json_match: - return json.loads(json_match.group()) - else: - raise ValueError("Could not parse JSON from response") - - except Exception as e: - raise RuntimeError( - f"Anthropic structured generation error: {str(e)}" - ) from e - - async def health_check(self) -> bool: - """ - Check if Anthropic API is available and healthy. - - Returns: - True if healthy, False otherwise - """ - try: - # Run synchronous client in thread pool to avoid blocking - import asyncio - - loop = asyncio.get_event_loop() - - def _sync_health_check(): - self.client.messages.create( - model=self.model_name, - max_tokens=1, - temperature=0.0, - messages=[{"role": "user", "content": "Test"}], - timeout=5.0, # Add explicit timeout - ) - return True - - # Run in executor with timeout - result = await asyncio.wait_for( - loop.run_in_executor(None, _sync_health_check), timeout=10.0 - ) - self._is_available = result - return result - - except Exception: - self._is_available = False - return False - - async def estimate_cost( - self, - prompt: str, - max_tokens: Optional[int] = None, - ) -> float: - """ - Estimate cost for generation. - - Args: - prompt: Input prompt - max_tokens: Maximum tokens to generate - - Returns: - Estimated cost in USD - """ - # Rough token estimation (1 token ≈ 4 characters) - input_tokens = len(prompt) // 4 - output_tokens = max_tokens or 100 - - total_tokens = input_tokens + output_tokens - return total_tokens * self.metrics.cost_per_token - - async def _rate_limit(self) -> None: - """Apply rate limiting to API requests.""" - import asyncio - import time - - current_time = time.time() - time_since_last = current_time - self._last_request_time - - if time_since_last < self._min_request_interval: - await asyncio.sleep(self._min_request_interval - time_since_last) - - self._last_request_time = time.time() - - def supports_streaming(self) -> bool: - """Check if model supports streaming.""" - return self.capabilities.supports_streaming - - async def generate_stream( - self, - prompt: str, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ): - """ - Generate text with streaming. - - Args: - prompt: Input prompt - temperature: Sampling temperature - max_tokens: Maximum tokens to generate - **kwargs: Additional Anthropic parameters - - Yields: - Streaming text chunks - """ - if not self.supports_streaming(): - raise ValueError(f"Model {self.name} does not support streaming") - - await self._rate_limit() - - try: - stream = self.client.messages.create( - model=self.model_name, - max_tokens=max_tokens or self.capabilities.max_tokens, - temperature=temperature, - messages=[{"role": "user", "content": prompt}], - stream=True, - **kwargs, - ) - - for chunk in stream: - if chunk.type == "content_block_delta": - yield chunk.delta.text - - except Exception as e: - raise RuntimeError(f"Anthropic streaming error: {str(e)}") from e - - def supports_function_calling(self) -> bool: - """Check if model supports function calling.""" - return self.capabilities.supports_function_calling - - async def generate_multimodal( - self, - messages: List[Dict[str, Any]], - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """ - Generate text from multimodal input using Anthropic's native vision support. - - Args: - messages: List of message dicts with role and content (can include images) - temperature: Sampling temperature - max_tokens: Maximum tokens to generate - **kwargs: Additional Anthropic parameters - - Returns: - Generated text - """ - # For Anthropic models with vision support, we can pass messages directly - if "vision" in self.capabilities.supported_tasks: - kwargs["messages"] = messages - return await self.generate("", temperature, max_tokens, **kwargs) - else: - # Fall back to default text-only implementation - return await super().generate_multimodal( - messages, temperature, max_tokens, **kwargs - ) - - async def generate_with_tools( - self, - prompt: str, - tools: List[Dict[str, Any]], - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> Dict[str, Any]: - """ - Generate text with tool/function calling. - - Args: - prompt: Input prompt - tools: List of tool definitions - temperature: Sampling temperature - max_tokens: Maximum tokens to generate - **kwargs: Additional Anthropic parameters - - Returns: - Response with tool calls - """ - if not self.supports_function_calling(): - raise ValueError(f"Model {self.name} does not support function calling") - - await self._rate_limit() - - try: - response = self.client.messages.create( - model=self.model_name, - max_tokens=max_tokens or self.capabilities.max_tokens, - temperature=temperature, - messages=[{"role": "user", "content": prompt}], - tools=tools, - **kwargs, - ) - - return { - "content": response.content[0].text if response.content else "", - "tool_calls": [ - { - "name": block.name, - "input": block.input, - } - for block in response.content - if hasattr(block, "name") - ], - } - - except Exception as e: - raise RuntimeError(f"Anthropic function calling error: {str(e)}") from e - - def get_available_models(self) -> List[str]: - """Get list of available Anthropic models.""" - return list(self.MODEL_CONFIGS.keys()) - - @classmethod - def create_from_config(cls, config: Dict[str, Any]) -> "AnthropicModel": - """Create Anthropic model from configuration.""" - return cls(**config) diff --git a/src/orchestrator/integrations/google_model.py b/src/orchestrator/integrations/google_model.py deleted file mode 100644 index 8ac341b3..00000000 --- a/src/orchestrator/integrations/google_model.py +++ /dev/null @@ -1,666 +0,0 @@ -"""Google AI model integration for the orchestrator framework.""" - -from __future__ import annotations - -import logging - -import json -import os -from typing import Any, Dict, List, Optional - -try: - import google.generativeai as genai - from google.generativeai import GenerativeModel - - GOOGLE_AI_AVAILABLE = True -except ImportError: - GOOGLE_AI_AVAILABLE = False - genai = None - GenerativeModel = None - -from orchestrator.core.model import ( - Model, - ModelCapabilities, - ModelMetrics, - ModelRequirements, -) - - -logger = logging.getLogger(__name__) - -class GoogleModel(Model): - """Google AI model implementation.""" - - # Model configurations - MODEL_CONFIGS = { - "gemini-1.5-pro": { - "capabilities": ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "vision", - "summarize", - "extract", - ], - context_window=2000000, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=8192, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=0.1, - cpu_cores=1, - disk_space_gb=0.1, - min_python_version="3.8", - requires_gpu=False, - ), - "metrics": ModelMetrics( - latency_p50=2.0, - latency_p95=5.0, - throughput=10.0, - accuracy=0.95, - cost_per_token=0.0000035, - success_rate=0.99, - ), - }, - "gemini-1.5-flash": { - "capabilities": ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "vision", - "summarize", - "extract", - ], - context_window=1000000, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=8192, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=0.1, - cpu_cores=1, - disk_space_gb=0.1, - min_python_version="3.8", - requires_gpu=False, - ), - "metrics": ModelMetrics( - latency_p50=1.0, - latency_p95=2.5, - throughput=20.0, - accuracy=0.92, - cost_per_token=0.00000035, - success_rate=0.98, - ), - }, - "gemini-1.0-pro": { - "capabilities": ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "summarize", - ], - context_window=32768, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=8192, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=0.1, - cpu_cores=1, - disk_space_gb=0.1, - min_python_version="3.8", - requires_gpu=False, - ), - "metrics": ModelMetrics( - latency_p50=1.5, - latency_p95=3.0, - throughput=15.0, - accuracy=0.90, - cost_per_token=0.0000005, - success_rate=0.97, - ), - }, - } - - def __init__( - self, - model_name: str = "gemini-1.5-flash", - api_key: Optional[str] = None, - max_retries: int = 3, - timeout: float = 30.0, - **kwargs: Any, - ) -> None: - """ - Initialize Google AI model. - - Args: - model_name: Google AI model name - api_key: Google AI API key (if not provided, will use GOOGLE_AI_API_KEY env var) - max_retries: Maximum number of retries for failed requests - timeout: Request timeout in seconds - **kwargs: Additional arguments passed to parent class - """ - global GOOGLE_AI_AVAILABLE, genai, GenerativeModel - if not GOOGLE_AI_AVAILABLE: - # Try to install on demand - import subprocess - import sys - - # Installing at runtime reaches the network and mutates the - # live environment on an ordinary pipeline run, so it is - # gated behind the same explicit opt-in as utils.auto_install. - from ..utils.auto_install import ( - AUTO_INSTALL_ENV_VAR, - auto_install_enabled, - ) - - if not auto_install_enabled(): - # This is the whole story: the library is absent and we are - # not allowed to fetch it. Re-wrapping it as "Failed to - # install ..." reported one cause twice and named the wrong - # one -- nothing was attempted, so nothing failed to install. - raise ImportError( - "Google AI library is not installed. Install it with: " - "pip install 'py-orc[google]' " - f"(or set {AUTO_INSTALL_ENV_VAR}=1 to install automatically)." - ) - - logger.info("Google AI library not found; installing it.") - try: - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "google-generativeai"] - ) - # Re-import after installation - import google.generativeai as genai - from google.generativeai import GenerativeModel - - GOOGLE_AI_AVAILABLE = True - except Exception as e: - raise ImportError( - f"Could not install the Google AI library automatically: {e}. " - "Install it with: pip install google-generativeai" - ) from e - - # Get model configuration - config = self.MODEL_CONFIGS.get( - model_name, self.MODEL_CONFIGS["gemini-1.5-flash"] - ) - - super().__init__( - name=model_name, - provider="google", - capabilities=config["capabilities"], - requirements=config["requirements"], - metrics=config["metrics"], - **kwargs, - ) - - # Initialize Google AI client - self.api_key = api_key or os.getenv("GOOGLE_AI_API_KEY") - if not self.api_key: - raise ValueError( - "Google AI API key not provided. Set GOOGLE_AI_API_KEY environment variable " - "or pass api_key parameter." - ) - - # Configure the API - genai.configure(api_key=self.api_key) - - # Create model instance - self.model = GenerativeModel(model_name) - self.model_name = model_name - self.max_retries = max_retries - self.timeout = timeout - - # Rate limiting - self._rate_limiter = None - self._last_request_time = 0.0 - self._min_request_interval = 0.1 # 10 requests per second max - - async def generate( - self, - prompt: str, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """ - Generate text using Google AI API. - - Args: - prompt: Input prompt - temperature: Sampling temperature (0.0 to 2.0) - max_tokens: Maximum tokens to generate - **kwargs: Additional Google AI parameters (including 'contents' for multimodal) - - Returns: - Generated text - """ - await self._rate_limit() - - # Validate temperature - temp_min, temp_max = self.capabilities.temperature_range - if not temp_min <= temperature <= temp_max: - raise ValueError( - f"Temperature {temperature} not in valid range {self.capabilities.temperature_range}" - ) - - # Set up generation config - generation_config = { - "temperature": temperature, - "max_output_tokens": max_tokens or self.capabilities.max_tokens, - } - - # Remove generation config params from kwargs - for key in ["temperature", "max_output_tokens"]: - kwargs.pop(key, None) - - try: - # Check if multimodal contents are provided - if "contents" in kwargs: - contents = kwargs.pop("contents") - response = self.model.generate_content( - contents, - generation_config=generation_config, - **kwargs, - ) - else: - response = self.model.generate_content( - prompt, - generation_config=generation_config, - **kwargs, - ) - - return response.text if response.text else "" - - except Exception as e: - raise RuntimeError(f"Google AI API error: {str(e)}") from e - - async def generate_structured( - self, - prompt: str, - schema: Dict[str, Any], - temperature: float = 0.7, - **kwargs: Any, - ) -> Dict[str, Any]: - """ - Generate structured output using Google AI API. - - Args: - prompt: Input prompt - schema: JSON schema for output structure - temperature: Sampling temperature - **kwargs: Additional Google AI parameters - - Returns: - Structured output matching schema - """ - if not self.capabilities.supports_structured_output: - raise ValueError(f"Model {self.name} does not support structured output") - - await self._rate_limit() - - # Create prompt with schema instructions - structured_prompt = f""" - {prompt} - - Please respond with a JSON object that matches this schema: - {json.dumps(schema, indent=2)} - - Return only the JSON object, no additional text. - """ - - try: - generation_config = { - "temperature": temperature, - "max_output_tokens": self.capabilities.max_tokens, - **kwargs, - } - - response = self.model.generate_content( - structured_prompt, - generation_config=generation_config, - ) - - content = response.text if response.text else "{}" - - # Parse JSON response - try: - return json.loads(content) - except json.JSONDecodeError: - # Try to extract JSON from response - import re - - json_match = re.search(r"\{.*\}", content, re.DOTALL) - if json_match: - return json.loads(json_match.group()) - else: - raise ValueError("Could not parse JSON from response") - - except Exception as e: - raise RuntimeError( - f"Google AI structured generation error: {str(e)}" - ) from e - - async def health_check(self) -> bool: - """ - Check if Google AI API is available and healthy. - - Returns: - True if healthy, False otherwise - """ - try: - # Run synchronous client in thread pool to avoid blocking - import asyncio - - loop = asyncio.get_event_loop() - - def _sync_health_check(): - generation_config = { - "temperature": 0.0, - "max_output_tokens": 1, - } - self.model.generate_content( - "Test", - generation_config=generation_config, - request_options={"timeout": 5.0}, # Add timeout - ) - return True - - # Run in executor with timeout - result = await asyncio.wait_for( - loop.run_in_executor(None, _sync_health_check), timeout=10.0 - ) - self._is_available = result - return result - - except Exception: - self._is_available = False - return False - - async def estimate_cost( - self, - prompt: str, - max_tokens: Optional[int] = None, - ) -> float: - """ - Estimate cost for generation. - - Args: - prompt: Input prompt - max_tokens: Maximum tokens to generate - - Returns: - Estimated cost in USD - """ - # Rough token estimation (1 token ≈ 4 characters) - input_tokens = len(prompt) // 4 - output_tokens = max_tokens or 100 - - total_tokens = input_tokens + output_tokens - return total_tokens * self.metrics.cost_per_token - - async def _rate_limit(self) -> None: - """Apply rate limiting to API requests.""" - import asyncio - import time - - current_time = time.time() - time_since_last = current_time - self._last_request_time - - if time_since_last < self._min_request_interval: - await asyncio.sleep(self._min_request_interval - time_since_last) - - self._last_request_time = time.time() - - def supports_streaming(self) -> bool: - """Check if model supports streaming.""" - return self.capabilities.supports_streaming - - async def generate_stream( - self, - prompt: str, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ): - """ - Generate text with streaming. - - Args: - prompt: Input prompt - temperature: Sampling temperature - max_tokens: Maximum tokens to generate - **kwargs: Additional Google AI parameters - - Yields: - Streaming text chunks - """ - if not self.supports_streaming(): - raise ValueError(f"Model {self.name} does not support streaming") - - await self._rate_limit() - - try: - generation_config = { - "temperature": temperature, - "max_output_tokens": max_tokens or self.capabilities.max_tokens, - **kwargs, - } - - response = self.model.generate_content( - prompt, - generation_config=generation_config, - stream=True, - ) - - for chunk in response: - if chunk.text: - yield chunk.text - - except Exception as e: - raise RuntimeError(f"Google AI streaming error: {str(e)}") from e - - def supports_function_calling(self) -> bool: - """Check if model supports function calling.""" - return self.capabilities.supports_function_calling - - async def generate_with_tools( - self, - prompt: str, - tools: List[Dict[str, Any]], - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> Dict[str, Any]: - """ - Generate text with tool/function calling. - - Args: - prompt: Input prompt - tools: List of tool definitions - temperature: Sampling temperature - max_tokens: Maximum tokens to generate - **kwargs: Additional Google AI parameters - - Returns: - Response with tool calls - """ - if not self.supports_function_calling(): - raise ValueError(f"Model {self.name} does not support function calling") - - await self._rate_limit() - - try: - generation_config = { - "temperature": temperature, - "max_output_tokens": max_tokens or self.capabilities.max_tokens, - **kwargs, - } - - # Convert tools to Google AI format - google_tools = [] - for tool in tools: - google_tools.append({"function_declarations": [tool]}) - - response = self.model.generate_content( - prompt, - generation_config=generation_config, - tools=google_tools, - ) - - # Extract tool calls - tool_calls = [] - if response.candidates and response.candidates[0].content.parts: - for part in response.candidates[0].content.parts: - if hasattr(part, "function_call"): - tool_calls.append( - { - "name": part.function_call.name, - "args": dict(part.function_call.args), - } - ) - - return { - "content": response.text if response.text else "", - "tool_calls": tool_calls, - } - - except Exception as e: - raise RuntimeError(f"Google AI function calling error: {str(e)}") from e - - def supports_vision(self) -> bool: - """Check if model supports vision/image processing.""" - return "vision" in self.capabilities.supported_tasks - - async def generate_with_image( - self, - prompt: str, - image_data: bytes, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """ - Generate text with image input. - - Args: - prompt: Input prompt - image_data: Image data as bytes - temperature: Sampling temperature - max_tokens: Maximum tokens to generate - **kwargs: Additional Google AI parameters - - Returns: - Generated text - """ - if not self.supports_vision(): - raise ValueError(f"Model {self.name} does not support vision") - - await self._rate_limit() - - try: - generation_config = { - "temperature": temperature, - "max_output_tokens": max_tokens or self.capabilities.max_tokens, - **kwargs, - } - - # Create image part - image_part = { - "mime_type": "image/jpeg", # Assume JPEG for simplicity - "data": image_data, - } - - response = self.model.generate_content( - [prompt, image_part], - generation_config=generation_config, - ) - - return response.text if response.text else "" - - except Exception as e: - raise RuntimeError(f"Google AI vision error: {str(e)}") from e - - async def generate_multimodal( - self, - messages: List[Dict[str, Any]], - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """ - Generate text from multimodal input using Google's native vision support. - - Args: - messages: List of message dicts with role and content (can include images) - temperature: Sampling temperature - max_tokens: Maximum tokens to generate - **kwargs: Additional Google parameters - - Returns: - Generated text - """ - # Convert our format to Google format - contents = [] - - for msg in messages: - if isinstance(msg.get("content"), list): - # Build parts for multimodal content - parts = [] - for block in msg["content"]: - if block["type"] == "text": - parts.append(block["text"]) - elif block["type"] == "image": - if block.get("source", {}).get("type") == "base64": - # Google expects PIL Image or bytes - import base64 - from PIL import Image - import io - - image_data = base64.b64decode(block["source"]["data"]) - image = Image.open(io.BytesIO(image_data)) - parts.append(image) - contents.extend(parts) - else: - # Simple text content - contents.append(msg["content"]) - - kwargs["contents"] = contents - return await self.generate("", temperature, max_tokens, **kwargs) - - def get_available_models(self) -> List[str]: - """Get list of available Google AI models.""" - return list(self.MODEL_CONFIGS.keys()) - - def list_models(self) -> List[str]: - """List all available models from the API.""" - try: - models = genai.list_models() - return [model.name for model in models] - except Exception: - return self.get_available_models() - - @classmethod - def create_from_config(cls, config: Dict[str, Any]) -> "GoogleModel": - """Create Google model from configuration.""" - return cls(**config) diff --git a/src/orchestrator/integrations/huggingface_model.py b/src/orchestrator/integrations/huggingface_model.py deleted file mode 100644 index b77ea4f6..00000000 --- a/src/orchestrator/integrations/huggingface_model.py +++ /dev/null @@ -1,586 +0,0 @@ -"""HuggingFace model integration for the orchestrator framework.""" - -from __future__ import annotations - -import json -import os -from typing import Any, Dict, List, Optional - -from orchestrator.utils.auto_install import safe_import, ensure_packages - -# Try to import required packages with auto-installation -try: - import torch - import transformers - from transformers import ( - AutoModelForCausalLM, - AutoTokenizer, - BitsAndBytesConfig, - pipeline, - ) - TRANSFORMERS_AVAILABLE = True -except ImportError: - # Try safe import with auto-installation - torch = safe_import("torch") - transformers = safe_import("transformers") - - if transformers and torch: - try: - from transformers import ( - AutoModelForCausalLM, - AutoTokenizer, - BitsAndBytesConfig, - pipeline, - ) - TRANSFORMERS_AVAILABLE = True - except ImportError: - TRANSFORMERS_AVAILABLE = False - AutoModelForCausalLM = None - AutoTokenizer = None - pipeline = None - BitsAndBytesConfig = None - else: - TRANSFORMERS_AVAILABLE = False - AutoModelForCausalLM = None - AutoTokenizer = None - pipeline = None - BitsAndBytesConfig = None - -from orchestrator.core.model import ( - Model, - ModelCapabilities, - ModelMetrics, - ModelRequirements, -) - - -class HuggingFaceModel(Model): - """HuggingFace model implementation.""" - - # Model configurations - MODEL_CONFIGS = { - "microsoft/DialoGPT-medium": { - "capabilities": ModelCapabilities( - supported_tasks=["generate", "chat", "analyze", "transform"], - context_window=1024, - supports_function_calling=False, - supports_structured_output=False, - supports_streaming=False, - languages=["en"], - max_tokens=512, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=2.0, - gpu_memory_gb=1.0, - cpu_cores=2, - supports_quantization=["8bit", "4bit"], - min_python_version="3.8", - requires_gpu=False, - disk_space_gb=1.0, - ), - "metrics": ModelMetrics( - latency_p50=0.5, - latency_p95=1.5, - throughput=50.0, - accuracy=0.80, - cost_per_token=0.0, - success_rate=0.95, - ), - }, - "microsoft/DialoGPT-small": { - "capabilities": ModelCapabilities( - supported_tasks=["generate", "chat", "analyze", "transform"], - context_window=1024, - supports_function_calling=False, - supports_structured_output=False, - supports_streaming=False, - languages=["en"], - max_tokens=512, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=1.0, - gpu_memory_gb=0.5, - cpu_cores=1, - supports_quantization=["8bit", "4bit"], - min_python_version="3.8", - requires_gpu=False, - disk_space_gb=0.5, - ), - "metrics": ModelMetrics( - latency_p50=0.3, - latency_p95=1.0, - throughput=100.0, - accuracy=0.75, - cost_per_token=0.0, - success_rate=0.95, - ), - }, - "gpt2": { - "capabilities": ModelCapabilities( - supported_tasks=["generate", "complete", "analyze", "transform"], - context_window=1024, - supports_function_calling=False, - supports_structured_output=False, - supports_streaming=False, - languages=["en"], - max_tokens=512, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=1.5, - gpu_memory_gb=0.8, - cpu_cores=2, - supports_quantization=["8bit", "4bit"], - min_python_version="3.8", - requires_gpu=False, - disk_space_gb=0.8, - ), - "metrics": ModelMetrics( - latency_p50=0.4, - latency_p95=1.2, - throughput=75.0, - accuracy=0.78, - cost_per_token=0.0, - success_rate=0.95, - ), - }, - "distilgpt2": { - "capabilities": ModelCapabilities( - supported_tasks=["generate", "complete", "analyze", "transform"], - context_window=1024, - supports_function_calling=False, - supports_structured_output=False, - supports_streaming=False, - languages=["en"], - max_tokens=512, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=0.5, - gpu_memory_gb=0.3, - cpu_cores=1, - supports_quantization=["8bit", "4bit"], - min_python_version="3.8", - requires_gpu=False, - disk_space_gb=0.3, - ), - "metrics": ModelMetrics( - latency_p50=0.2, - latency_p95=0.6, - throughput=150.0, - accuracy=0.70, - cost_per_token=0.0, - success_rate=0.95, - ), - }, - "TinyLlama/TinyLlama-1.1B-Chat-v1.0": { - "capabilities": ModelCapabilities( - supported_tasks=["generate", "chat", "reasoning", "analyze", "transform"], - context_window=2048, - supports_function_calling=False, - supports_structured_output=True, - supports_streaming=False, - languages=["en"], - max_tokens=512, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=1.5, - gpu_memory_gb=1.0, - cpu_cores=2, - supports_quantization=["8bit", "4bit"], - min_python_version="3.8", - requires_gpu=False, - disk_space_gb=1.2, - ), - "metrics": ModelMetrics( - latency_p50=0.8, - latency_p95=2.0, - throughput=40.0, - accuracy=0.82, - cost_per_token=0.0, - success_rate=0.92, - ), - }, - } - - def __init__( - self, - model_name: str = "distilgpt2", - device: Optional[str] = None, - quantization: Optional[str] = None, - cache_dir: Optional[str] = None, - token: Optional[str] = None, - **kwargs: Any, - ) -> None: - """ - Initialize HuggingFace model. - - Args: - model_name: HuggingFace model name or path - device: Device to load model on ('cpu', 'cuda', 'auto') - quantization: Quantization mode ('8bit', '4bit', None) - cache_dir: Directory to cache models - token: HuggingFace authentication token - **kwargs: Additional arguments passed to parent class - """ - if not TRANSFORMERS_AVAILABLE: - raise ImportError( - "Transformers library not available. Install with: pip install transformers torch" - ) - - # Get model configuration - config = self.MODEL_CONFIGS.get( - model_name, - { - "capabilities": ModelCapabilities( - supported_tasks=["generate", "analyze", "transform"], - context_window=1024, - supports_function_calling=False, - supports_structured_output=False, - supports_streaming=False, - languages=["en"], - max_tokens=512, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=2.0, - gpu_memory_gb=1.0, - cpu_cores=2, - supports_quantization=["8bit", "4bit"], - min_python_version="3.8", - requires_gpu=False, - disk_space_gb=1.0, - ), - "metrics": ModelMetrics( - latency_p50=1.0, - latency_p95=3.0, - throughput=30.0, - accuracy=0.80, - cost_per_token=0.0, - success_rate=0.95, - ), - }, - ) - - super().__init__( - name=model_name, - provider="huggingface", - capabilities=config["capabilities"], - requirements=config["requirements"], - metrics=config["metrics"], - **kwargs, - ) - - self.model_name = model_name - self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") - self.quantization = quantization - self.cache_dir = cache_dir - self.token = token or os.getenv("HF_TOKEN") - - # Initialize model and tokenizer - self.model = None - self.tokenizer = None - self.pipeline = None - self._model_loaded = False - - # Quantization config - self.quantization_config = None - if quantization: - if quantization == "8bit": - self.quantization_config = BitsAndBytesConfig(load_in_8bit=True) - elif quantization == "4bit": - self.quantization_config = BitsAndBytesConfig(load_in_4bit=True) - - async def _load_model(self) -> None: - """Load model and tokenizer if not already loaded.""" - if self._model_loaded: - return - - print(f"[HuggingFace] Loading model: {self.model_name}") - print(f"[HuggingFace] Device: {self.device}, Quantization: {self.quantization}") - print(f"[HuggingFace] Cache dir: {self.cache_dir}") - print(f"[HuggingFace] Auth token: {'Set' if self.token else 'Not set'}") - - try: - # Load tokenizer - print(f"[HuggingFace] Loading tokenizer...") - self.tokenizer = AutoTokenizer.from_pretrained( - self.model_name, - cache_dir=self.cache_dir, - token=self.token, - ) - - # Add pad token if not present - if self.tokenizer.pad_token is None: - self.tokenizer.pad_token = self.tokenizer.eos_token - - # Load model - print(f"Loading model (this may take a while)...") - model_kwargs = { - "cache_dir": self.cache_dir, - "token": self.token, - } - - if self.quantization_config: - model_kwargs["quantization_config"] = self.quantization_config - - self.model = AutoModelForCausalLM.from_pretrained( - self.model_name, - **model_kwargs, - ) - - # Move to device - if self.device != "auto" and not self.quantization: - self.model = self.model.to(self.device) - - # Create pipeline - self.pipeline = pipeline( - "text-generation", - model=self.model, - tokenizer=self.tokenizer, - device=0 if self.device == "cuda" else -1, - ) - - self._model_loaded = True - self._is_available = True - - except Exception as e: - raise RuntimeError(f"Failed to load HuggingFace model: {str(e)}") from e - - async def generate( - self, - prompt: str, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """ - Generate text using HuggingFace model. - - Args: - prompt: Input prompt - temperature: Sampling temperature (0.0 to 2.0) - max_tokens: Maximum tokens to generate - **kwargs: Additional generation parameters - - Returns: - Generated text - """ - await self._load_model() - - # Validate temperature - temp_min, temp_max = self.capabilities.temperature_range - if not temp_min <= temperature <= temp_max: - raise ValueError( - f"Temperature {temperature} not in valid range {self.capabilities.temperature_range}" - ) - - # Set default max_tokens if not provided - if max_tokens is None: - max_tokens = self.capabilities.max_tokens - - try: - # Generate with pipeline - outputs = self.pipeline( - prompt, - max_new_tokens=max_tokens, - temperature=temperature, - do_sample=temperature > 0.0, - pad_token_id=self.tokenizer.pad_token_id, - **kwargs, - ) - - # Extract generated text (remove prompt) - generated_text = outputs[0]["generated_text"] - if generated_text.startswith(prompt): - generated_text = generated_text[len(prompt) :].strip() - - return generated_text - - except Exception as e: - raise RuntimeError(f"HuggingFace generation error: {str(e)}") from e - - async def generate_structured( - self, - prompt: str, - schema: Dict[str, Any], - temperature: float = 0.7, - **kwargs: Any, - ) -> Dict[str, Any]: - """ - Generate structured output using HuggingFace model. - - Args: - prompt: Input prompt - schema: JSON schema for output structure - temperature: Sampling temperature - **kwargs: Additional generation parameters - - Returns: - Structured output matching schema - """ - if not self.capabilities.supports_structured_output: - raise ValueError(f"Model {self.name} does not support structured output") - - # Create prompt with schema instructions - structured_prompt = f""" - {prompt} - - Please respond with a JSON object that matches this schema: - {json.dumps(schema, indent=2)} - - Return only the JSON object, no additional text. - """ - - try: - response = await self.generate( - structured_prompt, temperature=temperature, **kwargs - ) - - # Parse JSON response - try: - return json.loads(response) - except json.JSONDecodeError: - # Try to extract JSON from response - import re - - json_match = re.search(r"\{.*\}", response, re.DOTALL) - if json_match: - return json.loads(json_match.group()) - else: - raise ValueError("Could not parse JSON from response") - - except Exception as e: - raise RuntimeError( - f"HuggingFace structured generation error: {str(e)}" - ) from e - - async def health_check(self) -> bool: - """ - Check if HuggingFace model is available and healthy. - - Returns: - True if healthy, False otherwise - """ - try: - print(f"[HuggingFace] Starting health check for {self.model_name}") - - # Check if transformers is available - if not TRANSFORMERS_AVAILABLE: - print(f"[HuggingFace] Transformers library not available") - return False - - # Try to load the model - print(f"[HuggingFace] Loading model...") - await self._load_model() - - # Check if model was loaded - if not hasattr(self, 'model') or self.model is None: - print(f"[HuggingFace] Model failed to load") - return False - - if not hasattr(self, 'tokenizer') or self.tokenizer is None: - print(f"[HuggingFace] Tokenizer failed to load") - return False - - # Simple test generation - print(f"[HuggingFace] Testing generation...") - result = await self.generate("Test", max_tokens=1, temperature=0.0) - print(f"[HuggingFace] Test generation result: {result}") - - self._is_available = True - print(f"[HuggingFace] Health check passed") - return True - - except Exception as e: - print(f"[HuggingFace] Health check failed for {self.model_name}: {type(e).__name__}: {e}") - import traceback - traceback.print_exc() - self._is_available = False - return False - - async def estimate_cost( - self, - prompt: str, - max_tokens: Optional[int] = None, - ) -> float: - """ - Estimate cost for generation (local models are free). - - Args: - prompt: Input prompt - max_tokens: Maximum tokens to generate - - Returns: - Estimated cost in USD (0.0 for local models) - """ - return 0.0 # Local models are free - - def supports_quantization(self) -> bool: - """Check if model supports quantization.""" - return len(self.requirements.supports_quantization) > 0 - - def get_model_info(self) -> Dict[str, Any]: - """Get model information.""" - return { - "name": self.model_name, - "device": self.device, - "quantization": self.quantization, - "loaded": self._model_loaded, - "supports_quantization": self.supports_quantization(), - "supported_quantization": self.requirements.supports_quantization, - } - - def get_memory_usage(self) -> Dict[str, float]: - """Get memory usage information.""" - if not self._model_loaded or not torch.cuda.is_available(): - return {"gpu_memory_mb": 0.0, "gpu_memory_gb": 0.0} - - try: - memory_mb = torch.cuda.memory_allocated() / (1024**2) - return { - "gpu_memory_mb": memory_mb, - "gpu_memory_gb": memory_mb / 1024, - } - except Exception: - return {"gpu_memory_mb": 0.0, "gpu_memory_gb": 0.0} - - def unload_model(self) -> None: - """Unload model to free memory.""" - if self.model is not None: - del self.model - self.model = None - - if self.tokenizer is not None: - del self.tokenizer - self.tokenizer = None - - if self.pipeline is not None: - del self.pipeline - self.pipeline = None - - # Clear GPU cache if available - if torch.cuda.is_available(): - torch.cuda.empty_cache() - - self._model_loaded = False - self._is_available = False - - def get_available_models(self) -> List[str]: - """Get list of available HuggingFace models.""" - return list(self.MODEL_CONFIGS.keys()) - - @classmethod - def create_from_config(cls, config: Dict[str, Any]) -> "HuggingFaceModel": - """Create HuggingFace model from configuration.""" - return cls(**config) - - def __del__(self) -> None: - """Clean up when model is destroyed.""" - try: - self.unload_model() - except Exception: - pass diff --git a/src/orchestrator/integrations/lazy_huggingface_model.py b/src/orchestrator/integrations/lazy_huggingface_model.py deleted file mode 100644 index 9a7d726d..00000000 --- a/src/orchestrator/integrations/lazy_huggingface_model.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Lazy loading wrapper for HuggingFace models.""" - -from .huggingface_model import HuggingFaceModel - - -class LazyHuggingFaceModel(HuggingFaceModel): - """HuggingFace model that downloads on first use.""" - - def __init__(self, model_name: str, **kwargs): - """Initialize lazy HuggingFace model without loading it.""" - # Initialize parent without loading the model - super().__init__(model_name=model_name, **kwargs) - # Override the model loading behavior - self._model_loaded = False - self._is_available = True # Assume available until proven otherwise - - async def _load_model(self) -> None: - """Load model and tokenizer if not already loaded.""" - if self._model_loaded: - return - - print( - f">> 📥 Downloading HuggingFace model: {self.model_name} (this may take a while on first use)" - ) - - try: - # Call parent's load method - await super()._load_model() - print(f">> ✅ Successfully loaded {self.model_name}") - except Exception as e: - print(f">> ❌ Failed to load {self.model_name}: {str(e)}") - raise - - async def health_check(self) -> bool: - """Check if model is healthy without downloading.""" - # For lazy models, we assume they're healthy if they could be downloaded - # We don't actually download during health check - if self._model_loaded: - return await super().health_check() - - # Model not loaded yet, but could be - return True - return True diff --git a/src/orchestrator/integrations/lazy_ollama_model.py b/src/orchestrator/integrations/lazy_ollama_model.py deleted file mode 100644 index a2443a3c..00000000 --- a/src/orchestrator/integrations/lazy_ollama_model.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Lazy loading wrapper for Ollama models.""" - -from typing import Any, Dict, Optional - -from ..utils.model_utils import check_ollama_model, install_ollama_model, start_ollama_server -from .ollama_model import OllamaModel - - -class LazyOllamaModel(OllamaModel): - """Ollama model that downloads on first use.""" - - def __init__(self, model_name: str, **kwargs): - """Initialize lazy Ollama model without checking availability.""" - self._model_downloaded = False - self._download_attempted = False - # Initialize parent without checking availability - super().__init__(model_name=model_name, **kwargs) - # Override availability to True initially (we'll check on first use) - self._is_available = True - - def _check_ollama_availability(self) -> None: - """Override parent's availability check - we check lazily on first use.""" - # Don't check availability during init - we'll do it on first use - self._is_available = True - - def _pull_model(self) -> None: - """Override parent's pull method - we do this lazily.""" - # Don't pull during init - we'll do it on first use - pass - - async def _ensure_model_available(self) -> bool: - """Ensure model is downloaded before use.""" - if self._model_downloaded: - return True - - if self._download_attempted: - # Already tried and failed - return False - - self._download_attempted = True - - # First ensure Ollama server is running - if not start_ollama_server(): - print(">> ❌ Ollama is not installed or could not be started") - self._is_available = False - return False - - # Check if model is already available - if check_ollama_model(self.model_name): - self._model_downloaded = True - return True - - # Try to download the model - print( - f">> 📥 Downloading Ollama model: {self.model_name} (this may take a while on first use)" - ) - if install_ollama_model(self.model_name): - self._model_downloaded = True - self._is_available = True - print(f">> ✅ Successfully downloaded {self.model_name}") - return True - else: - self._is_available = False - print(f">> ❌ Failed to download {self.model_name}") - return False - - async def generate( - self, - prompt: str, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """Generate text, downloading model if needed.""" - if not await self._ensure_model_available(): - raise RuntimeError( - f"Ollama model {self.model_name} is not available and could not be downloaded" - ) - - return await super().generate(prompt, temperature, max_tokens, **kwargs) - - async def generate_structured( - self, - prompt: str, - schema: Dict[str, Any], - temperature: float = 0.7, - **kwargs: Any, - ) -> Dict[str, Any]: - """Generate structured output, downloading model if needed.""" - if not await self._ensure_model_available(): - raise RuntimeError( - f"Ollama model {self.model_name} is not available and could not be downloaded" - ) - - return await super().generate_structured(prompt, schema, temperature, **kwargs) - - async def chat( - self, - messages: list, - temperature: float = 0.7, - **kwargs: Any, - ) -> str: - """Chat with model, downloading if needed.""" - if not await self._ensure_model_available(): - raise RuntimeError( - f"Ollama model {self.model_name} is not available and could not be downloaded" - ) - - return await super().chat(messages, temperature, **kwargs) - - async def health_check(self) -> bool: - """Check if model is healthy (download if needed).""" - # For health check, we just check if model can be made available - # but don't actually download it - if self._model_downloaded: - return await super().health_check() - - # Check if model exists locally - if check_ollama_model(self.model_name): - self._model_downloaded = True - return await super().health_check() - - # Model not downloaded yet, but could be - return True - return True diff --git a/src/orchestrator/integrations/ollama_model.py b/src/orchestrator/integrations/ollama_model.py deleted file mode 100644 index 0e9ef0f5..00000000 --- a/src/orchestrator/integrations/ollama_model.py +++ /dev/null @@ -1,607 +0,0 @@ -"""Ollama model integration for the orchestrator framework.""" - -from __future__ import annotations - -import asyncio -import json -import logging -import subprocess -from typing import Any, Dict, List, Optional - -try: - import requests - - REQUESTS_AVAILABLE = True -except ImportError: - REQUESTS_AVAILABLE = False - requests = None - -from orchestrator.core.model import ( - Model, - ModelCapabilities, - ModelMetrics, - ModelRequirements, -) - -logger = logging.getLogger(__name__) - - -class OllamaModel(Model): - """Ollama model implementation.""" - - # Model configurations for popular Ollama models - MODEL_CONFIGS = { - "gemma2:27b": { - "capabilities": ModelCapabilities( - supported_tasks=["generate", "chat", "reasoning", "code", "analyze", "transform", "summarize", "extract"], - context_window=8192, - supports_function_calling=False, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ja", "ko", "zh"], - max_tokens=2048, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=16.0, - gpu_memory_gb=12.0, - cpu_cores=8, - supports_quantization=["q4_0", "q4_1", "q5_0", "q5_1", "q8_0"], - min_python_version="3.8", - requires_gpu=False, - disk_space_gb=15.0, - ), - "metrics": ModelMetrics( - latency_p50=2.5, - latency_p95=8.0, - throughput=15.0, - accuracy=0.88, - cost_per_token=0.0, - success_rate=0.96, - ), - }, - "gemma2:9b": { - "capabilities": ModelCapabilities( - supported_tasks=["generate", "chat", "reasoning", "code", "analyze", "transform", "summarize", "extract"], - context_window=8192, - supports_function_calling=False, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ja", "ko", "zh"], - max_tokens=2048, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=8.0, - gpu_memory_gb=6.0, - cpu_cores=4, - supports_quantization=["q4_0", "q4_1", "q5_0", "q5_1", "q8_0"], - min_python_version="3.8", - requires_gpu=False, - disk_space_gb=5.5, - ), - "metrics": ModelMetrics( - latency_p50=1.8, - latency_p95=5.0, - throughput=25.0, - accuracy=0.85, - cost_per_token=0.0, - success_rate=0.95, - ), - }, - "llama3.2:3b": { - "capabilities": ModelCapabilities( - supported_tasks=["generate", "chat", "reasoning", "analyze", "transform", "summarize", "extract"], - context_window=4096, - supports_function_calling=False, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt"], - max_tokens=1024, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=4.0, - gpu_memory_gb=3.0, - cpu_cores=2, - supports_quantization=["q4_0", "q4_1", "q5_0", "q5_1", "q8_0"], - min_python_version="3.8", - requires_gpu=False, - disk_space_gb=2.0, - ), - "metrics": ModelMetrics( - latency_p50=1.2, - latency_p95=3.5, - throughput=35.0, - accuracy=0.82, - cost_per_token=0.0, - success_rate=0.94, - ), - }, - "llama3.2:1b": { - "capabilities": ModelCapabilities( - supported_tasks=["generate", "chat", "analyze", "transform", "summarize", "extract"], - context_window=4096, - supports_function_calling=False, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr"], - max_tokens=1024, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=2.0, - gpu_memory_gb=1.5, - cpu_cores=2, - supports_quantization=["q4_0", "q4_1", "q5_0", "q5_1", "q8_0"], - min_python_version="3.8", - requires_gpu=False, - disk_space_gb=1.0, - ), - "metrics": ModelMetrics( - latency_p50=0.8, - latency_p95=2.0, - throughput=50.0, - accuracy=0.78, - cost_per_token=0.0, - success_rate=0.93, - ), - }, - } - - def __init__( - self, - model_name: str = "llama3.2:3b", - base_url: str = "http://localhost:11434", - timeout: int = 30, - **kwargs: Any, - ) -> None: - """ - Initialize Ollama model. - - Args: - model_name: Ollama model name (e.g., "gemma2:27b", "llama3.2:3b") - base_url: Ollama server URL - timeout: Request timeout in seconds - **kwargs: Additional arguments passed to parent class - """ - if not REQUESTS_AVAILABLE: - raise ImportError( - "Requests library not available. Install with: pip install requests" - ) - - # Get model configuration - config = self.MODEL_CONFIGS.get( - model_name, - { - "capabilities": ModelCapabilities( - supported_tasks=["generate", "chat", "analyze", "transform", "summarize", "extract"], - context_window=4096, - supports_function_calling=False, - supports_structured_output=True, - supports_streaming=True, - languages=["en"], - max_tokens=1024, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=4.0, - gpu_memory_gb=2.0, - cpu_cores=2, - supports_quantization=["q4_0", "q4_1", "q5_0", "q5_1", "q8_0"], - min_python_version="3.8", - requires_gpu=False, - disk_space_gb=2.0, - ), - "metrics": ModelMetrics( - latency_p50=2.0, - latency_p95=6.0, - throughput=20.0, - accuracy=0.80, - cost_per_token=0.0, - success_rate=0.90, - ), - }, - ) - - super().__init__( - name=model_name, - provider="ollama", - capabilities=config["capabilities"], - requirements=config["requirements"], - metrics=config["metrics"], - **kwargs, - ) - - self.model_name = model_name - self.base_url = base_url.rstrip("/") - self.timeout = timeout - - # Set Issue 194 enhanced attributes - self._expertise = self._get_model_expertise(model_name) - self._size_billions = self._estimate_model_size(model_name) - - # Set cost information (Ollama models are free) - from ..core.model import ModelCost - self.cost = ModelCost(is_free=True) - - # Check if Ollama is available - self._check_ollama_availability() - - def _check_ollama_availability(self) -> None: - """Check if Ollama is running and available.""" - try: - response = requests.get(f"{self.base_url}/api/tags", timeout=5) - if response.status_code == 200: - self._is_available = True - # Check if our specific model is available - models = response.json().get("models", []) - model_names = [model["name"] for model in models] - if self.model_name not in model_names: - # Try to pull the model - self._pull_model() - else: - self._is_available = False - except Exception: - # Ollama might not be running, try to start it - if self._start_ollama_if_installed(): - # Try again after starting - try: - response = requests.get(f"{self.base_url}/api/tags", timeout=5) - if response.status_code == 200: - self._is_available = True - # Check if our specific model is available - models = response.json().get("models", []) - model_names = [model["name"] for model in models] - if self.model_name not in model_names: - # Try to pull the model - self._pull_model() - else: - self._is_available = False - except Exception: - self._is_available = False - else: - self._is_available = False - - def _start_ollama_if_installed(self) -> bool: - """Try to start Ollama service if it's installed but not running.""" - try: - # Use the enhanced service manager for better service control - from orchestrator.utils.service_manager import SERVICE_MANAGERS - ollama_manager = SERVICE_MANAGERS.get("ollama") - if ollama_manager: - return ollama_manager.ensure_running() - else: - logger.error("Ollama service manager not found") - return False - except ImportError: - # Fallback to old method - try: - from orchestrator.utils.model_utils import start_ollama_server - return start_ollama_server() - except ImportError: - logger.warning("Service management modules not found, cannot auto-start Ollama") - return False - - def _pull_model(self) -> None: - """Pull model if not available locally.""" - try: - # Try using the enhanced service manager first - from orchestrator.utils.service_manager import SERVICE_MANAGERS - ollama_manager = SERVICE_MANAGERS.get("ollama") - if ollama_manager and hasattr(ollama_manager, 'ensure_model_available'): - if ollama_manager.ensure_model_available(self.model_name): - self._is_available = True - return - - # Fallback to direct CLI call - result = subprocess.run( - ["ollama", "pull", self.model_name], - capture_output=True, - text=True, - timeout=300, # 5 minutes timeout for model pull - ) - if result.returncode == 0: - self._is_available = True - else: - print( - f"Warning: Could not pull model {self.model_name}: {result.stderr}" - ) - except (subprocess.TimeoutExpired, FileNotFoundError): - print( - f"Warning: Could not pull model {self.model_name} (ollama CLI not available or timeout)" - ) - - async def generate( - self, - prompt: str, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """ - Generate text using Ollama model. - - Args: - prompt: Input prompt - temperature: Sampling temperature (0.0 to 2.0) - max_tokens: Maximum tokens to generate - **kwargs: Additional generation parameters - - Returns: - Generated text - """ - if not self._is_available: - raise RuntimeError( - "Ollama model not available. Check if Ollama is running." - ) - - # Validate temperature - temp_min, temp_max = self.capabilities.temperature_range - if not temp_min <= temperature <= temp_max: - raise ValueError( - f"Temperature {temperature} not in valid range {self.capabilities.temperature_range}" - ) - - # Set default max_tokens if not provided - if max_tokens is None: - max_tokens = self.capabilities.max_tokens - - # Prepare request payload - payload = { - "model": self.model_name, - "prompt": prompt, - "stream": False, - "options": { - "temperature": temperature, - "num_predict": max_tokens, - **kwargs, - }, - } - - try: - # Run the synchronous request in a thread pool to avoid blocking - response = await asyncio.to_thread( - requests.post, - f"{self.base_url}/api/generate", - json=payload, - timeout=self.timeout, - ) - response.raise_for_status() - - result = response.json() - return result.get("response", "").strip() - - except Exception as e: - raise RuntimeError(f"Ollama generation error: {str(e)}") from e - - async def generate_structured( - self, - prompt: str, - schema: Dict[str, Any], - temperature: float = 0.7, - **kwargs: Any, - ) -> Dict[str, Any]: - """ - Generate structured output using Ollama model. - - Args: - prompt: Input prompt - schema: JSON schema for output structure - temperature: Sampling temperature - **kwargs: Additional generation parameters - - Returns: - Structured output matching schema - """ - if not self.capabilities.supports_structured_output: - raise ValueError(f"Model {self.name} does not support structured output") - - # Create prompt with schema instructions - structured_prompt = f""" -{prompt} - -Please respond with a JSON object that matches this schema: -{json.dumps(schema, indent=2)} - -Return only the JSON object, no additional text. -""" - - try: - response = await self.generate( - structured_prompt, temperature=temperature, **kwargs - ) - - # Parse JSON response - try: - return json.loads(response) - except json.JSONDecodeError: - # Try to extract JSON from response - import re - - json_match = re.search(r"\{.*\}", response, re.DOTALL) - if json_match: - return json.loads(json_match.group()) - else: - raise ValueError("Could not parse JSON from response") - - except Exception as e: - raise RuntimeError(f"Ollama structured generation error: {str(e)}") from e - - def _get_model_expertise(self, model_name: str) -> list[str]: - """Get model expertise areas based on model name.""" - name_lower = model_name.lower() - - # Code-specialized models - if any(x in name_lower for x in ["codellama", "deepseek", "coder", "starcoder"]): - return ["code", "reasoning", "programming"] - - # Fast/compact models - elif any(x in name_lower for x in ["1b", "3b"]) or "mini" in name_lower: - return ["fast", "compact", "general"] - - # Reasoning models - elif any(x in name_lower for x in ["wizard", "orca", "vicuna", "reasoning"]): - return ["reasoning", "analysis", "general"] - - # Large capable models - elif any(x in name_lower for x in ["70b", "405b"]) or "instruct" in name_lower: - return ["reasoning", "analysis", "creative", "general"] - - # Medium models - elif any(x in name_lower for x in ["7b", "8b", "13b", "27b"]): - return ["general", "chat", "reasoning"] - - # Default - return ["general"] - - def _estimate_model_size(self, model_name: str) -> float: - """Estimate model size in billions of parameters from name.""" - from ..utils.model_utils import parse_model_size - return parse_model_size(model_name, None) - - async def health_check(self) -> bool: - """ - Check if Ollama model is available and healthy. - - Returns: - True if healthy, False otherwise - """ - try: - # Check if Ollama is running - response = await asyncio.to_thread( - requests.get, f"{self.base_url}/api/tags", timeout=5 - ) - if response.status_code != 200: - self._is_available = False - return False - - # Check if our model is available - models = response.json().get("models", []) - model_names = [model["name"] for model in models] - if self.model_name not in model_names: - self._is_available = False - return False - - # Simple test generation - await self.generate("Test", max_tokens=1, temperature=0.0) - self._is_available = True - return True - - except Exception: - # Try to start Ollama if it's not running - if await asyncio.to_thread(self._start_ollama_if_installed): - # Try health check again after starting - try: - response = await asyncio.to_thread( - requests.get, f"{self.base_url}/api/tags", timeout=5 - ) - if response.status_code == 200: - # Check if our model is available - models = response.json().get("models", []) - model_names = [model["name"] for model in models] - if self.model_name not in model_names: - # Try to pull the model - await asyncio.to_thread(self._pull_model) - # Check again - response = await asyncio.to_thread( - requests.get, f"{self.base_url}/api/tags", timeout=5 - ) - models = response.json().get("models", []) - model_names = [model["name"] for model in models] - if self.model_name not in model_names: - self._is_available = False - return False - - # Simple test generation - await self.generate("Test", max_tokens=1, temperature=0.0) - self._is_available = True - return True - except Exception: - pass - - self._is_available = False - return False - - async def estimate_cost( - self, - prompt: str, - max_tokens: Optional[int] = None, - ) -> float: - """ - Estimate cost for generation (local models are free). - - Args: - prompt: Input prompt - max_tokens: Maximum tokens to generate - - Returns: - Estimated cost in USD (0.0 for local models) - """ - return 0.0 # Local models are free - - def get_available_models(self) -> List[str]: - """Get list of available Ollama models.""" - try: - response = requests.get(f"{self.base_url}/api/tags", timeout=5) - if response.status_code == 200: - models = response.json().get("models", []) - return [model["name"] for model in models] - except Exception: - pass - - return list(self.MODEL_CONFIGS.keys()) - - def get_model_info(self) -> Dict[str, Any]: - """Get model information.""" - return { - "name": self.model_name, - "base_url": self.base_url, - "available": self._is_available, - "timeout": self.timeout, - "supported_quantizations": self.requirements.supports_quantization, - } - - def is_ollama_running(self) -> bool: - """Check if Ollama service is running.""" - try: - response = requests.get(f"{self.base_url}/api/tags", timeout=5) - return response.status_code == 200 - except Exception: - return False - - def get_ollama_models(self) -> List[Dict[str, Any]]: - """Get list of all models available in Ollama.""" - try: - response = requests.get(f"{self.base_url}/api/tags", timeout=5) - if response.status_code == 200: - return response.json().get("models", []) - except Exception: - pass - return [] - - @classmethod - def create_from_config(cls, config: Dict[str, Any]) -> "OllamaModel": - """Create Ollama model from configuration.""" - return cls(**config) - - @staticmethod - def check_ollama_installation() -> bool: - """Check if Ollama CLI is installed.""" - try: - result = subprocess.run( - ["ollama", "--version"], capture_output=True, text=True, timeout=5 - ) - return result.returncode == 0 - except (subprocess.TimeoutExpired, FileNotFoundError): - return False - - @staticmethod - def get_recommended_models() -> List[str]: - """Get list of recommended models for testing.""" - return [ - "gemma2:27b", # Best quality, high resource requirements - "gemma2:9b", # Good balance of quality and resources - "llama3.2:3b", # Good for testing, moderate resources - "llama3.2:1b", # Fastest, lowest resources - ] diff --git a/src/orchestrator/integrations/openai_model.py b/src/orchestrator/integrations/openai_model.py deleted file mode 100644 index 56134bc3..00000000 --- a/src/orchestrator/integrations/openai_model.py +++ /dev/null @@ -1,705 +0,0 @@ -"""OpenAI model integration for the orchestrator framework.""" - -from __future__ import annotations - -import logging - -import json -import os -from typing import Any, Dict, List, Optional - -try: - import openai - from openai import OpenAI - - OPENAI_AVAILABLE = True -except ImportError: - OPENAI_AVAILABLE = False - OpenAI = None - openai = None - -from orchestrator.core.model import ( - Model, - ModelCapabilities, - ModelCost, - ModelMetrics, - ModelRequirements, -) - - -logger = logging.getLogger(__name__) - -class OpenAIModel(Model): - """OpenAI model implementation.""" - - # Model configurations - MODEL_CONFIGS = { - # GPT-5 series (latest models) - "gpt-5": { - "capabilities": ModelCapabilities( - supported_tasks=[ - "generate", "analyze", "transform", "code", "reasoning", - "vision", "multimodal", "complex_reasoning", "summarize", "extract" - ], - context_window=256000, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=16384, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=0.1, - cpu_cores=1, - disk_space_gb=0.1, - min_python_version="3.8", - requires_gpu=False, - ), - "metrics": ModelMetrics( - latency_p50=1.0, - latency_p95=3.0, - throughput=20.0, - accuracy=0.98, - cost_per_token=0.00002, # Will be converted to proper cost - success_rate=0.98, - ), - "cost": ModelCost( - input_cost_per_1k_tokens=0.015, # $0.015 per 1k input tokens - output_cost_per_1k_tokens=0.060, # $0.060 per 1k output tokens - ), - }, - "gpt-5-mini": { - "capabilities": ModelCapabilities( - supported_tasks=[ - "generate", "analyze", "transform", "code", "reasoning", "summarize", "extract" - ], - context_window=128000, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=8192, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=0.1, - cpu_cores=1, - disk_space_gb=0.1, - min_python_version="3.8", - requires_gpu=False, - ), - "metrics": ModelMetrics( - latency_p50=0.8, - latency_p95=2.0, - throughput=30.0, - accuracy=0.95, - cost_per_token=0.000005, - success_rate=0.98, - ), - "cost": ModelCost( - input_cost_per_1k_tokens=0.003, # $0.003 per 1k input tokens - output_cost_per_1k_tokens=0.012, # $0.012 per 1k output tokens - ), - }, - "gpt-5-nano": { - "capabilities": ModelCapabilities( - supported_tasks=[ - "generate", "analyze", "transform", "simple_tasks", "summarize", "extract" - ], - context_window=32000, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt"], - max_tokens=4096, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=0.1, - cpu_cores=1, - disk_space_gb=0.1, - min_python_version="3.8", - requires_gpu=False, - ), - "metrics": ModelMetrics( - latency_p50=0.5, - latency_p95=1.5, - throughput=50.0, - accuracy=0.92, - cost_per_token=0.000001, - success_rate=0.98, - ), - "cost": ModelCost( - input_cost_per_1k_tokens=0.0005, # $0.0005 per 1k input tokens - output_cost_per_1k_tokens=0.002, # $0.002 per 1k output tokens - ), - }, - "gpt-4": { - "capabilities": ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "summarize", - ], - context_window=8192, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=4096, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=0.1, - cpu_cores=1, - disk_space_gb=0.1, - min_python_version="3.8", - requires_gpu=False, - ), - "metrics": ModelMetrics( - latency_p50=2.0, - latency_p95=5.0, - throughput=10.0, - accuracy=0.95, - cost_per_token=0.00003, - success_rate=0.99, - ), - "cost": ModelCost( - input_cost_per_1k_tokens=0.010, # $0.01 per 1k input tokens - output_cost_per_1k_tokens=0.030, # $0.03 per 1k output tokens - ), - }, - "gpt-4-turbo": { - "capabilities": ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "vision", - "summarize", - ], - context_window=128000, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=4096, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=0.1, - cpu_cores=1, - disk_space_gb=0.1, - min_python_version="3.8", - requires_gpu=False, - ), - "metrics": ModelMetrics( - latency_p50=1.5, - latency_p95=4.0, - throughput=15.0, - accuracy=0.96, - cost_per_token=0.00001, - success_rate=0.99, - ), - "cost": ModelCost( - input_cost_per_1k_tokens=0.001, # $0.001 per 1k input tokens - output_cost_per_1k_tokens=0.002, # $0.002 per 1k output tokens - ), - }, - "gpt-3.5-turbo": { - "capabilities": ModelCapabilities( - supported_tasks=["generate", "analyze", "transform", "code", "summarize"], - context_window=16384, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=4096, - temperature_range=(0.0, 2.0), - ), - "requirements": ModelRequirements( - memory_gb=0.1, - cpu_cores=1, - disk_space_gb=0.1, - min_python_version="3.8", - requires_gpu=False, - ), - "metrics": ModelMetrics( - latency_p50=1.0, - latency_p95=2.5, - throughput=25.0, - accuracy=0.88, - cost_per_token=0.0000015, - success_rate=0.98, - ), - "cost": ModelCost( - input_cost_per_1k_tokens=0.0005, # $0.0005 per 1k input tokens - output_cost_per_1k_tokens=0.0015, # $0.0015 per 1k output tokens - ), - }, - } - - def __init__( - self, - model_name: str = "gpt-3.5-turbo", - api_key: Optional[str] = None, - base_url: Optional[str] = None, - max_retries: int = 3, - timeout: float = 30.0, - **kwargs: Any, - ) -> None: - """ - Initialize OpenAI model. - - Args: - model_name: OpenAI model name - api_key: OpenAI API key (if not provided, will use OPENAI_API_KEY env var) - base_url: Base URL for API calls - max_retries: Maximum number of retries for failed requests - timeout: Request timeout in seconds - **kwargs: Additional arguments passed to parent class - """ - global OPENAI_AVAILABLE, openai, OpenAI - if not OPENAI_AVAILABLE: - # Try to install on demand - import subprocess - import sys - - # Installing at runtime reaches the network and mutates the - # live environment on an ordinary pipeline run, so it is - # gated behind the same explicit opt-in as utils.auto_install. - from ..utils.auto_install import ( - AUTO_INSTALL_ENV_VAR, - auto_install_enabled, - ) - - if not auto_install_enabled(): - # This is the whole story: the library is absent and we are - # not allowed to fetch it. Re-wrapping it as "Failed to - # install ..." reported one cause twice and named the wrong - # one -- nothing was attempted, so nothing failed to install. - raise ImportError( - "OpenAI library is not installed. Install it with: " - "pip install 'py-orc[openai]' " - f"(or set {AUTO_INSTALL_ENV_VAR}=1 to install automatically)." - ) - - logger.info("OpenAI library not found; installing it.") - try: - subprocess.check_call( - [sys.executable, "-m", "pip", "install", "openai"] - ) - # Re-import after installation - import openai - from openai import OpenAI - - OPENAI_AVAILABLE = True - except Exception as e: - raise ImportError( - f"Could not install the OpenAI library automatically: {e}. " - "Install it with: pip install openai" - ) from e - - # Get model configuration - config = self.MODEL_CONFIGS.get(model_name) - if not config: - # Try to find a matching base config - if model_name.startswith("gpt-4"): - config = self.MODEL_CONFIGS["gpt-4"] - elif model_name.startswith("gpt-3.5"): - config = self.MODEL_CONFIGS["gpt-3.5-turbo"] - elif ( - model_name.startswith("o1") - or model_name.startswith("o3") - or model_name.startswith("o4") - ): - # New reasoning models - use GPT-4 config as base - config = self.MODEL_CONFIGS["gpt-4"] - else: - # Default to gpt-3.5-turbo config - config = self.MODEL_CONFIGS["gpt-3.5-turbo"] - - # Use cost from config if available, otherwise create default - cost = config.get("cost", ModelCost()) - - super().__init__( - name=model_name, - provider="openai", - capabilities=config["capabilities"], - requirements=config["requirements"], - metrics=config["metrics"], - cost=cost, - **kwargs, - ) - - # Initialize OpenAI client - self.api_key = api_key or os.getenv("OPENAI_API_KEY") - if not self.api_key: - raise ValueError( - "OpenAI API key not provided. Set OPENAI_API_KEY environment variable " - "or pass api_key parameter." - ) - - self.client = OpenAI( - api_key=self.api_key, - base_url=base_url, - max_retries=max_retries, - timeout=timeout, - ) - - self.model_name = model_name - self.max_retries = max_retries - self.timeout = timeout - - # Rate limiting - self._rate_limiter = None - self._last_request_time = 0.0 - self._min_request_interval = 0.1 # 10 requests per second max - - async def generate( - self, - prompt: str, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """ - Generate text using OpenAI API. - - Args: - prompt: Input prompt - temperature: Sampling temperature (0.0 to 2.0) - max_tokens: Maximum tokens to generate - **kwargs: Additional OpenAI parameters - - Returns: - Generated text - """ - await self._rate_limit() - - # Validate temperature - temp_min, temp_max = self.capabilities.temperature_range - if not temp_min <= temperature <= temp_max: - raise ValueError( - f"Temperature {temperature} not in valid range {self.capabilities.temperature_range}" - ) - - # Set default max_tokens if not provided - if max_tokens is None: - max_tokens = self.capabilities.max_tokens - - try: - # Check if messages already passed in kwargs (e.g., from generate_multimodal) - if "messages" in kwargs: - messages = kwargs.pop("messages") - else: - messages = [{"role": "user", "content": prompt}] - - # Handle response_format conversion - if "response_format" in kwargs: - rf = kwargs.pop("response_format") - if isinstance(rf, str): - if rf == "json_object": - kwargs["response_format"] = {"type": "json_object"} - else: - # Keep as-is if it's already a dict or other format - kwargs["response_format"] = rf - else: - kwargs["response_format"] = rf - - # Build API parameters - api_params = { - "model": self.model_name, - "messages": messages, - **kwargs, - } - - # Handle model-specific parameter names - # GPT-5 models have specific requirements - if "gpt-5" in self.model_name.lower(): - # GPT-5 models require max_completion_tokens instead of max_tokens - api_params["max_completion_tokens"] = max_tokens - # GPT-5 models only support default temperature (1.0), but we should always set it - # Setting temperature to 1.0 for consistency even if different value requested - api_params["temperature"] = 1.0 - else: - api_params["max_tokens"] = max_tokens - api_params["temperature"] = temperature - - response = self.client.chat.completions.create(**api_params) - - return response.choices[0].message.content or "" - - except Exception as e: - raise RuntimeError(f"OpenAI API error: {str(e)}") from e - - async def generate_structured( - self, - prompt: str, - schema: Dict[str, Any], - temperature: float = 0.7, - **kwargs: Any, - ) -> Dict[str, Any]: - """ - Generate structured output using OpenAI API. - - Args: - prompt: Input prompt - schema: JSON schema for output structure - temperature: Sampling temperature - **kwargs: Additional OpenAI parameters - - Returns: - Structured output matching schema - """ - if not self.capabilities.supports_structured_output: - raise ValueError(f"Model {self.name} does not support structured output") - - await self._rate_limit() - - # Create prompt with schema instructions - structured_prompt = f""" - {prompt} - - Please respond with a JSON object that matches this schema: - {json.dumps(schema, indent=2)} - - Return only the JSON object, no additional text. - """ - - try: - # Prepare API parameters with GPT-5 compatibility - api_params = { - "model": self.model_name, - "messages": [{"role": "user", "content": structured_prompt}], - } - - # Handle model-specific parameters - if "gpt-5" in self.model_name.lower(): - # GPT-5 models only support default temperature (1.0) - api_params["temperature"] = 1.0 - # For max_tokens in kwargs - if "max_tokens" in kwargs: - api_params["max_completion_tokens"] = kwargs.pop("max_tokens") - else: - api_params["temperature"] = temperature - - # Add any additional kwargs - api_params.update(kwargs) - - response = self.client.chat.completions.create(**api_params) - - content = response.choices[0].message.content or "{}" - - # Parse JSON response - try: - return json.loads(content) - except json.JSONDecodeError: - # Try to extract JSON from response - import re - - json_match = re.search(r"\{.*\}", content, re.DOTALL) - if json_match: - return json.loads(json_match.group()) - else: - raise ValueError("Could not parse JSON from response") - - except Exception as e: - raise RuntimeError(f"OpenAI structured generation error: {str(e)}") from e - - async def health_check(self) -> bool: - """ - Check if OpenAI API is available and healthy. - - Returns: - True if healthy, False otherwise - """ - try: - # Run synchronous client in thread pool to avoid blocking - import asyncio - - loop = asyncio.get_event_loop() - - def _sync_health_check(): - api_params = { - "model": self.model_name, - "messages": [{"role": "user", "content": "Test"}], - "timeout": 5.0, # Add explicit timeout - } - - # Handle model-specific parameter names - if "gpt-5" in self.model_name.lower(): - api_params["max_completion_tokens"] = 1 - # GPT-5 models only support default temperature - else: - api_params["max_tokens"] = 1 - api_params["temperature"] = 0.0 - - self.client.chat.completions.create(**api_params) - return True - - # Run in executor with timeout - result = await asyncio.wait_for( - loop.run_in_executor(None, _sync_health_check), timeout=10.0 - ) - self._is_available = result - return result - - except Exception: - self._is_available = False - return False - - async def estimate_cost( - self, - prompt: str, - max_tokens: Optional[int] = None, - ) -> float: - """ - Estimate cost for generation. - - Args: - prompt: Input prompt - max_tokens: Maximum tokens to generate - - Returns: - Estimated cost in USD - """ - # Rough token estimation (1 token ≈ 4 characters) - input_tokens = len(prompt) // 4 - output_tokens = max_tokens or 100 - - total_tokens = input_tokens + output_tokens - return total_tokens * self.metrics.cost_per_token - - async def _rate_limit(self) -> None: - """Apply rate limiting to API requests.""" - import asyncio - import time - - current_time = time.time() - time_since_last = current_time - self._last_request_time - - if time_since_last < self._min_request_interval: - await asyncio.sleep(self._min_request_interval - time_since_last) - - self._last_request_time = time.time() - - def supports_streaming(self) -> bool: - """Check if model supports streaming.""" - return self.capabilities.supports_streaming - - async def generate_stream( - self, - prompt: str, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ): - """ - Generate text with streaming. - - Args: - prompt: Input prompt - temperature: Sampling temperature - max_tokens: Maximum tokens to generate - **kwargs: Additional OpenAI parameters - - Yields: - Streaming text chunks - """ - if not self.supports_streaming(): - raise ValueError(f"Model {self.name} does not support streaming") - - await self._rate_limit() - - try: - # Build API parameters - api_params = { - "model": self.model_name, - "messages": [{"role": "user", "content": prompt}], - "stream": True, - **kwargs, - } - - # Handle model-specific parameter names - if "gpt-5" in self.model_name.lower(): - api_params["max_completion_tokens"] = max_tokens - # GPT-5 models only support default temperature - if temperature == 1.0: - api_params["temperature"] = temperature - else: - api_params["max_tokens"] = max_tokens - api_params["temperature"] = temperature - - stream = self.client.chat.completions.create(**api_params) - - for chunk in stream: - if chunk.choices[0].delta.content is not None: - yield chunk.choices[0].delta.content - - except Exception as e: - raise RuntimeError(f"OpenAI streaming error: {str(e)}") from e - - async def generate_multimodal( - self, - messages: List[Dict[str, Any]], - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """ - Generate text from multimodal input using OpenAI's native vision support. - - Args: - messages: List of message dicts with role and content (can include images) - temperature: Sampling temperature - max_tokens: Maximum tokens to generate - **kwargs: Additional OpenAI parameters - - Returns: - Generated text - """ - # For OpenAI models with vision support (gpt-4-vision, etc.), - # we need to format image content properly - formatted_messages = [] - for msg in messages: - if isinstance(msg.get("content"), list): - # Convert our format to OpenAI format - content_parts = [] - for block in msg["content"]: - if block["type"] == "text": - content_parts.append({"type": "text", "text": block["text"]}) - elif block["type"] == "image": - # OpenAI expects image_url format - if block.get("source", {}).get("type") == "base64": - content_parts.append( - { - "type": "image_url", - "image_url": { - "url": f"data:{block['source'].get('media_type', 'image/png')};base64,{block['source']['data']}" - }, - } - ) - formatted_messages.append( - {"role": msg["role"], "content": content_parts} - ) - else: - formatted_messages.append(msg) - - kwargs["messages"] = formatted_messages - return await self.generate("", temperature, max_tokens, **kwargs) - - def get_available_models(self) -> List[str]: - """Get list of available OpenAI models.""" - return list(self.MODEL_CONFIGS.keys()) - - @classmethod - def create_from_config(cls, config: Dict[str, Any]) -> "OpenAIModel": - """Create OpenAI model from configuration.""" - return cls(**config) diff --git a/src/orchestrator/models/__init__.py b/src/orchestrator/models/__init__.py index a291b26f..b3737279 100644 --- a/src/orchestrator/models/__init__.py +++ b/src/orchestrator/models/__init__.py @@ -1,8 +1,15 @@ -"""Model management and selection with unified provider abstractions.""" +"""Model management and selection. + +The canonical registry is :class:`ModelRegistry` from +:mod:`orchestrator.models.model_registry`, reached through +:func:`get_model_registry`. The skills-era "unified" registry and its +provider-configuration system were retired under #430 -- they only ever +supported Anthropic, which is no longer a provider of this product. +""" -# Legacy model registry (for backwards compatibility) from .model_registry import ( ModelNotFoundError, + ModelRegistry, NoEligibleModelsError, UCBModelSelector, ) @@ -11,61 +18,25 @@ set_model_registry, reset_model_registry, ) - -# New unified provider system -from .registry import ModelRegistry as UnifiedModelRegistry -from .config import ( - RegistryConfiguration, - ModelProviderSpec, - create_default_configuration, - create_registry_from_config, - create_registry_from_env, - load_configuration_from_dict, - configuration_to_dict, - CLOUD_ONLY_CONFIG, - LOCAL_ONLY_CONFIG, - DEVELOPMENT_CONFIG, -) from .providers import ( ModelProvider, ProviderConfig, ProviderError, - AnthropicProvider, ) -# Keep legacy ModelRegistry for backwards compatibility -# TODO: Eventually migrate all usage to UnifiedModelRegistry -from .model_registry import ModelRegistry as LegacyModelRegistry +#: Kept for backwards compatibility with code that imported the legacy name. +LegacyModelRegistry = ModelRegistry __all__ = [ - # Legacy registry (backwards compatibility) + "ModelRegistry", "LegacyModelRegistry", - "UCBModelSelector", + "UCBModelSelector", "ModelNotFoundError", "NoEligibleModelsError", "get_model_registry", - "set_model_registry", + "set_model_registry", "reset_model_registry", - - # New unified provider system - "UnifiedModelRegistry", - "RegistryConfiguration", - "ModelProviderSpec", - "create_default_configuration", - "create_registry_from_config", - "create_registry_from_env", - "load_configuration_from_dict", - "configuration_to_dict", - "CLOUD_ONLY_CONFIG", - "LOCAL_ONLY_CONFIG", - "DEVELOPMENT_CONFIG", - - # Provider abstractions (Anthropic-only for Claude Skills refactor) "ModelProvider", "ProviderConfig", "ProviderError", - "AnthropicProvider", ] - -# For backwards compatibility, keep ModelRegistry pointing to legacy -ModelRegistry = LegacyModelRegistry diff --git a/src/orchestrator/models/anthropic_model.py b/src/orchestrator/models/anthropic_model.py deleted file mode 100644 index d9a409db..00000000 --- a/src/orchestrator/models/anthropic_model.py +++ /dev/null @@ -1,696 +0,0 @@ -"""Anthropic model adapter implementation with LangChain backend support.""" - -from __future__ import annotations - -import os -import asyncio -import logging -import re -from typing import TYPE_CHECKING, Any, Dict, Optional - -if TYPE_CHECKING: - # Annotations are lazy (`from __future__ import annotations`), so the SDK is - # only required when a client is constructed -- see __init__ below. Keeping - # this at module scope made the model registry unimportable without the - # `anthropic` extra. - from anthropic import AsyncAnthropic - -from ..core.model import Model, ModelCapabilities, ModelRequirements, ModelCost -from ..utils.auto_install import safe_import -from ..utils.api_keys_flexible import ensure_api_key - -logger = logging.getLogger(__name__) - - -class AnthropicModel(Model): - """Anthropic model implementation.""" - - def __init__( - self, - name: str, - api_key: Optional[str] = None, - base_url: Optional[str] = None, - capabilities: Optional[ModelCapabilities] = None, - requirements: Optional[ModelRequirements] = None, - use_langchain: bool = True, - **kwargs: Any, - ) -> None: - """ - Initialize Anthropic model with LangChain backend support. - - Args: - name: Model name (e.g., "claude-3-opus", "claude-3-sonnet") - api_key: Anthropic API key (defaults to ANTHROPIC_API_KEY env var) - base_url: Custom API base URL (optional) - capabilities: Model capabilities - requirements: Resource requirements - use_langchain: Whether to try using LangChain backend (defaults to True) - **kwargs: Additional arguments - """ - # Set default capabilities based on model - if capabilities is None: - capabilities = self._get_default_capabilities(name) - - # Set default requirements - if requirements is None: - requirements = ModelRequirements( - memory_gb=0.5, - cpu_cores=1, - requires_gpu=False, - disk_space_gb=0.1, - ) - - # Set cost information - cost = self._get_model_cost(name) - - super().__init__( - name=name, - provider="anthropic", - capabilities=capabilities, - requirements=requirements, - cost=cost, - ) - - # Get API key using existing infrastructure - self.api_key = api_key - if not self.api_key: - try: - self.api_key = ensure_api_key("anthropic") - except Exception: - self.api_key = os.getenv("ANTHROPIC_API_KEY") - if not self.api_key: - raise ValueError("Anthropic API key not provided") - - # Try to initialize LangChain model first - self.langchain_model = None - self._use_langchain = False - - if use_langchain: - try: - langchain_anthropic = safe_import("langchain_anthropic", auto_install=True) - if langchain_anthropic: - self.langchain_model = langchain_anthropic.ChatAnthropic( - model=self._normalize_model_name(name), - api_key=self.api_key, - base_url=base_url, - temperature=kwargs.get("temperature", 0.7), - max_tokens=kwargs.get("max_tokens"), - **{k: v for k, v in kwargs.items() if k not in ['temperature', 'max_tokens']} - ) - self._use_langchain = True - logger.info(f"Using LangChain backend for Anthropic model: {name}") - else: - logger.warning(f"LangChain not available, falling back to direct Anthropic for: {name}") - except Exception as e: - logger.warning(f"Failed to initialize LangChain Anthropic model: {e}, falling back to direct Anthropic") - - # Fallback: Initialize direct Anthropic client - if not self._use_langchain: - try: - from anthropic import AsyncAnthropic - except ImportError as exc: # pragma: no cover - depends on install - raise ImportError( - "AnthropicModel requires the 'anthropic' package. " - "Install it with: pip install 'py-orc[anthropic]'" - ) from exc - - self.client = AsyncAnthropic( - api_key=self.api_key, - base_url=base_url, - ) - logger.info(f"Using direct Anthropic client for model: {name}") - - # Set model-specific attributes (preserve existing functionality) - self._model_id = self._normalize_model_name(name) - self._expertise = self._get_model_expertise(name) - self._size_billions = self._estimate_model_size(name) - self._is_available = True - - #: A model id the caller has already pinned: either dated - #: (``claude-haiku-4-5-20251001``) or an Anthropic ``-latest`` alias. - _QUALIFIED_MODEL_RE = re.compile(r"^claude-.+(-\d{8}|-latest)$", re.IGNORECASE) - - #: Recognized bare family names. These are NOT mapped to hard-coded ids: - #: every attempt to hard-code them in this file has rotted (the 2024 dated - #: ids 404 today, and invented ``-latest`` aliases 404 too -- both verified - #: against the live API). They are resolved from the Models API instead; - #: see :meth:`resolve_family_alias`. - _FAMILIES = ("opus", "sonnet", "haiku") - - #: Process-wide cache of family -> concrete id, filled from the Models API. - _family_cache: Dict[str, str] = {} - - @classmethod - async def resolve_family_alias(cls, client: Any, family: str) -> str: - """Return the newest served model id for ``family``. - - Asks the API what exists rather than trusting a table in this file. - Anthropic ids sort chronologically by their trailing date, so the - lexicographic maximum is the newest release of that family. - """ - if family in cls._family_cache: - return cls._family_cache[family] - - try: - listing = await client.models.list() - except Exception as exc: # noqa: BLE001 - surfaced with guidance below - raise RuntimeError( - f"cannot resolve the bare model name {family!r}: the Anthropic " - f"Models API is unreachable ({exc}). Pass a fully-qualified id " - f"such as 'claude-haiku-4-5-20251001' instead." - ) from exc - - candidates = sorted( - model.id - for model in listing.data - if family in model.id.lower() - ) - if not candidates: - available = ", ".join(sorted(m.id for m in listing.data)) - raise RuntimeError( - f"no Anthropic model matches the family {family!r}. " - f"Available: {available}" - ) - - resolved = candidates[-1] - cls._family_cache[family] = resolved - logger.info("Resolved model family %r to %r", family, resolved) - return resolved - - def _normalize_model_name(self, name: str) -> str: - """Resolve a model name to an id to send to the API. - - A fully-qualified id is returned untouched. This matters: the previous - implementation substring-matched on the family name and rewrote *every* - id containing "haiku"/"opus"/"sonnet" to a hard-coded 2024 model, so - - AnthropicModel(name="claude-haiku-4-5-20251001") - - actually requested ``claude-3-haiku-20240307`` and the API answered - 404 not_found_error. No current Claude model was reachable at all, and - the caller's explicit choice was discarded with no warning. Caught by - the live acceptance test on its first real run. - """ - if self._QUALIFIED_MODEL_RE.match(name): - return name - - name_lower = name.lower() - - # Legacy generations keep their exact ids; they have no rolling alias. - if "claude-2.1" in name_lower: - return "claude-2.1" - if "claude-2" in name_lower: - return "claude-2.0" - if "instant" in name_lower: - return "claude-instant-1.2" - - # A bare family name is left as-is here and resolved against the - # Models API on first use (see _resolve_model_id). Resolution needs a - # network round trip, which must not happen in __init__. - # - # Unrecognized names are also passed through: an unknown id produces a - # clear 404 from the API, which is far more debuggable than silently - # substituting some near match. - return name - - async def _resolve_model_id(self) -> str: - """The id to send to the API, resolving a bare family name if needed.""" - if self._model_id.lower() in self._FAMILIES: - return await self.resolve_family_alias(self.client, self._model_id.lower()) - return self._model_id - - def _get_default_capabilities(self, name: str) -> ModelCapabilities: - """Get default capabilities based on model name.""" - name_lower = name.lower() - - # Claude 3.5 Sonnet / Sonnet 4 - if "sonnet" in name_lower and ( - "3.5" in name_lower or "sonnet-4" in name_lower or "sonnet4" in name_lower - ): - return ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "creative", - "chat", - "instruct", - "vision", - "math", - "research", - ], - context_window=200000, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=8192, - temperature_range=(0.0, 1.0), - ) - - # Claude 3 Opus - elif "opus" in name_lower: - return ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "creative", - "chat", - "instruct", - "vision", - "math", - "research", - ], - context_window=200000, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=4096, - temperature_range=(0.0, 1.0), - vision_capable=True, - code_specialized=True, - supports_tools=True, - accuracy_score=0.95, - speed_rating="medium", - ) - - # Claude 3 Sonnet - elif "sonnet" in name_lower: - return ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "creative", - "chat", - "instruct", - "vision", - ], - context_window=200000, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=4096, - temperature_range=(0.0, 1.0), - vision_capable=True, - code_specialized=True, - supports_tools=True, - accuracy_score=0.90, - speed_rating="medium", - ) - - # Claude 3 Haiku - elif "haiku" in name_lower: - return ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "chat", - "instruct", - "vision", - ], - context_window=200000, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=4096, - temperature_range=(0.0, 1.0), - vision_capable=True, - code_specialized=True, - supports_tools=True, - accuracy_score=0.85, - speed_rating="fast", - ) - - # Claude 2.x - elif "claude-2" in name_lower: - return ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "creative", - "chat", - "instruct", - ], - context_window=100000, - supports_function_calling=False, - supports_structured_output=False, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=4096, - temperature_range=(0.0, 1.0), - ) - - # Claude Instant - elif "instant" in name_lower: - return ModelCapabilities( - supported_tasks=["generate", "chat", "instruct"], - context_window=100000, - supports_function_calling=False, - supports_structured_output=False, - supports_streaming=True, - languages=["en"], - max_tokens=4096, - temperature_range=(0.0, 1.0), - ) - - # Default - return ModelCapabilities( - supported_tasks=["generate", "chat"], - context_window=100000, - supports_function_calling=False, - supports_structured_output=False, - supports_streaming=True, - languages=["en"], - max_tokens=4096, - ) - - def _get_model_expertise(self, name: str) -> list[str]: - """Get model expertise areas.""" - name_lower = name.lower() - - if "opus" in name_lower: - return [ - "general", - "reasoning", - "code", - "creative", - "analysis", - "research", - "math", - ] - elif "sonnet" in name_lower: - if "3.5" in name_lower or "sonnet-4" in name_lower: - return [ - "general", - "reasoning", - "code", - "creative", - "analysis", - "research", - ] - return ["general", "reasoning", "code", "analysis"] - elif "haiku" in name_lower: - return ["general", "chat", "code"] - elif "instant" in name_lower: - return ["general", "chat"] - - return ["general"] - - def _estimate_model_size(self, name: str) -> float: - """Estimate model size in billions of parameters.""" - name_lower = name.lower() - - if "opus" in name_lower: - return 175.0 # Estimated - elif "sonnet" in name_lower: - return 70.0 # Estimated - elif "haiku" in name_lower: - return 20.0 # Estimated - elif "instant" in name_lower: - return 10.0 # Estimated - - return 1.0 - - def _get_model_cost(self, name: str) -> ModelCost: - """Get cost information for Anthropic model.""" - name_lower = name.lower() - - # Anthropic pricing (as of 2024) - if "opus" in name_lower: - return ModelCost( - input_cost_per_1k_tokens=15.0 / 1000, # $15 per 1M input tokens = $0.015 per 1K - output_cost_per_1k_tokens=75.0 / 1000, # $75 per 1M output tokens = $0.075 per 1K - is_free=False, - ) - elif "sonnet" in name_lower: - if "3-5" in name_lower or "3.5" in name_lower: - return ModelCost( - input_cost_per_1k_tokens=3.0 / 1000, # $3 per 1M tokens - output_cost_per_1k_tokens=15.0 / 1000, # $15 per 1M tokens - is_free=False, - ) - else: - return ModelCost( - input_cost_per_1k_tokens=3.0 / 1000, # $3 per 1M tokens - output_cost_per_1k_tokens=15.0 / 1000, # $15 per 1M tokens - is_free=False, - ) - elif "haiku" in name_lower: - return ModelCost( - input_cost_per_1k_tokens=0.25 / 1000, # $0.25 per 1M tokens - output_cost_per_1k_tokens=1.25 / 1000, # $1.25 per 1M tokens - is_free=False, - ) - elif "instant" in name_lower: - return ModelCost( - input_cost_per_1k_tokens=0.8 / 1000, # $0.80 per 1M tokens - output_cost_per_1k_tokens=2.4 / 1000, # $2.40 per 1M tokens - is_free=False, - ) - else: - # Default pricing for unknown Anthropic models - return ModelCost( - input_cost_per_1k_tokens=8.0 / 1000, - output_cost_per_1k_tokens=24.0 / 1000, - is_free=False, - ) - - async def generate( - self, - prompt: str, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """ - Generate text from prompt using LangChain or direct Anthropic. - - Args: - prompt: Input prompt - temperature: Sampling temperature - max_tokens: Maximum tokens to generate - **kwargs: Additional parameters - - Returns: - Generated text - """ - # Use LangChain if available - if self._use_langchain and self.langchain_model: - try: - return await self._langchain_generate(prompt, temperature, max_tokens, **kwargs) - except Exception as e: - logger.warning(f"LangChain generation failed, falling back to direct Anthropic: {e}") - # Fall through to direct Anthropic implementation - - # Fallback to direct Anthropic implementation (preserve original functionality) - return await self._direct_anthropic_generate(prompt, temperature, max_tokens, **kwargs) - - async def _langchain_generate( - self, - prompt: str, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """Generate using LangChain backend.""" - try: - # Handle system prompt for LangChain - system_prompt = kwargs.get("system_prompt") - if system_prompt and system_prompt.strip(): - from langchain_core.messages import SystemMessage, HumanMessage - messages = [ - SystemMessage(content=system_prompt), - HumanMessage(content=prompt) - ] - - if hasattr(self.langchain_model, 'ainvoke'): - response = await self.langchain_model.ainvoke(messages) - else: - response = await asyncio.to_thread(self.langchain_model.invoke, messages) - else: - # Simple prompt - if hasattr(self.langchain_model, 'ainvoke'): - response = await self.langchain_model.ainvoke(prompt) - else: - response = await asyncio.to_thread(self.langchain_model.invoke, prompt) - - # Extract content from LangChain response - if hasattr(response, 'content'): - return response.content - else: - return str(response) - - except Exception as e: - raise RuntimeError(f"LangChain Anthropic generation failed: {str(e)}") - - async def _direct_anthropic_generate( - self, - prompt: str, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """Generate using direct Anthropic client (original implementation).""" - try: - # Prepare messages - messages = [{"role": "user", "content": prompt}] - - # Add system message if provided - system_prompt = kwargs.get("system_prompt") - - # Make API call - api_kwargs = { - "model": await self._resolve_model_id(), - "messages": messages, - "temperature": temperature, - "max_tokens": max_tokens or self.capabilities.max_tokens, - "stream": False, - } - - # Only add system prompt if it's provided and non-empty - if system_prompt and system_prompt.strip(): - api_kwargs["system"] = system_prompt - - response = await self.client.messages.create(**api_kwargs) - - # Extract response - if response.content: - # Handle different content types - if isinstance(response.content, list): - # Extract text from content blocks - text_parts = [] - for block in response.content: - if hasattr(block, "text"): - text_parts.append(block.text) - return " ".join(text_parts) - else: - return str(response.content) - - return "" - - except Exception as e: - # Log error and raise - raise RuntimeError(f"Anthropic generation failed: {str(e)}") - - async def generate_structured( - self, - prompt: str, - schema: Dict[str, Any], - temperature: float = 0.7, - **kwargs: Any, - ) -> Dict[str, Any]: - """ - Generate structured output from prompt. - - Args: - prompt: Input prompt - schema: JSON schema for output - temperature: Sampling temperature - **kwargs: Additional parameters - - Returns: - Structured output - """ - try: - # Add schema instruction to prompt - import json - schema_prompt = f"{prompt}\n\nPlease respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}" - - # Generate response using existing generate method (handles LangChain/direct Anthropic automatically) - response = await self.generate( - prompt=schema_prompt, - temperature=temperature, - **kwargs, - ) - - # Parse JSON response - try: - return json.loads(response) - except json.JSONDecodeError: - # Try to extract JSON from response - import re - json_match = re.search(r"\{.*\}", response, re.DOTALL) - if json_match: - return json.loads(json_match.group()) - raise ValueError("Could not parse JSON from response") - - except Exception as e: - raise RuntimeError(f"Anthropic structured generation failed: {str(e)}") - - async def health_check(self) -> bool: - """ - Check if model is available and healthy. - - Returns: - True if healthy - """ - try: - # Use the generate method which handles LangChain/direct Anthropic automatically - response = await self.generate("Hi", temperature=0.0, max_tokens=5) - return len(response) > 0 - except Exception: - return False - - async def estimate_cost( - self, - prompt: str, - max_tokens: Optional[int] = None, - ) -> float: - """ - Estimate cost for generation. - - Args: - prompt: Input prompt - max_tokens: Maximum tokens to generate - - Returns: - Estimated cost in USD - """ - # Estimate token counts (Anthropic uses similar tokenization to OpenAI) - prompt_tokens = len(prompt) // 4 # Rough estimate - output_tokens = max_tokens or 1000 - - # Cost per 1M tokens - model_lower = self._model_id.lower() - if "opus" in model_lower: - input_cost = 15.0 # $15 per 1M input tokens - output_cost = 75.0 # $75 per 1M output tokens - elif "sonnet" in model_lower: - if "3-5" in model_lower or "3.5" in model_lower: - input_cost = 3.0 # $3 per 1M input tokens - output_cost = 15.0 # $15 per 1M output tokens - else: - input_cost = 3.0 # $3 per 1M input tokens - output_cost = 15.0 # $15 per 1M output tokens - elif "haiku" in model_lower: - input_cost = 0.25 # $0.25 per 1M input tokens - output_cost = 1.25 # $1.25 per 1M output tokens - elif "instant" in model_lower: - input_cost = 0.8 # $0.80 per 1M input tokens - output_cost = 2.4 # $2.40 per 1M output tokens - else: - # Default pricing - input_cost = 8.0 - output_cost = 24.0 - - # Calculate total cost - total_cost = (prompt_tokens / 1_000_000 * input_cost) + ( - output_tokens / 1_000_000 * output_cost - ) - return total_cost diff --git a/src/orchestrator/models/config.py b/src/orchestrator/models/config.py deleted file mode 100644 index 89e46e0e..00000000 --- a/src/orchestrator/models/config.py +++ /dev/null @@ -1,282 +0,0 @@ -"""Model provider configuration and discovery system.""" - -from __future__ import annotations - -import os -from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional - -from .providers.base import ProviderConfig -from .registry import ModelRegistry - - -@dataclass -class ModelProviderSpec: - """Specification for a model provider.""" - - name: str - type: str # "openai", "anthropic", "local" - config: Dict[str, Any] = field(default_factory=dict) - enabled: bool = True - priority: int = 0 # Higher priority providers are checked first - - def to_provider_config(self) -> ProviderConfig: - """Convert to ProviderConfig.""" - return ProviderConfig(name=self.name, **self.config) - - -@dataclass -class RegistryConfiguration: - """Configuration for the model registry.""" - - providers: List[ModelProviderSpec] = field(default_factory=list) - auto_discover: bool = True - default_timeout: float = 30.0 - max_retries: int = 3 - cache_models: bool = True - - def add_provider( - self, - name: str, - provider_type: str, - config: Optional[Dict[str, Any]] = None, - enabled: bool = True, - priority: int = 0, - ) -> None: - """Add a provider specification.""" - spec = ModelProviderSpec( - name=name, - type=provider_type, - config=config or {}, - enabled=enabled, - priority=priority, - ) - self.providers.append(spec) - - def get_enabled_providers(self) -> List[ModelProviderSpec]: - """Get list of enabled providers sorted by priority.""" - enabled = [p for p in self.providers if p.enabled] - return sorted(enabled, key=lambda p: p.priority, reverse=True) - - -def create_default_configuration() -> RegistryConfiguration: - """Create a default registry configuration with common providers.""" - config = RegistryConfiguration() - - # OpenAI provider - config.add_provider( - name="openai", - provider_type="openai", - config={ - "api_key": None, # Will use environment variable - "organization": None, - "timeout": 30.0, - "max_retries": 3, - }, - priority=100, # High priority for cloud provider - ) - - # Anthropic provider - config.add_provider( - name="anthropic", - provider_type="anthropic", - config={ - "api_key": None, # Will use environment variable - "timeout": 30.0, - "max_retries": 3, - }, - priority=95, # High priority for cloud provider - ) - - # Local provider (Ollama) - config.add_provider( - name="local", - provider_type="local", - config={ - "base_url": "http://localhost:11434", - "timeout": 60.0, # Longer timeout for local models - "max_retries": 2, - }, - priority=50, # Lower priority than cloud providers - ) - - return config - - -def create_registry_from_config(config: RegistryConfiguration) -> ModelRegistry: - """Create and configure a model registry from configuration.""" - registry = ModelRegistry() - - for provider_spec in config.get_enabled_providers(): - try: - registry.configure_provider( - provider_name=provider_spec.name, - provider_type=provider_spec.type, - config=provider_spec.config, - ) - except Exception as e: - # Log error but continue with other providers - import logging - logger = logging.getLogger(__name__) - logger.error(f"Failed to configure provider {provider_spec.name}: {e}") - - return registry - - -def create_registry_from_env() -> ModelRegistry: - """Create a registry configured from environment variables.""" - config = RegistryConfiguration() - - # Configure OpenAI if API key is available - if os.getenv("OPENAI_API_KEY"): - config.add_provider( - name="openai", - provider_type="openai", - config={ - "api_key": os.getenv("OPENAI_API_KEY"), - "organization": os.getenv("OPENAI_ORG_ID"), - "base_url": os.getenv("OPENAI_BASE_URL"), - }, - priority=100, - ) - - # Configure Anthropic if API key is available - if os.getenv("ANTHROPIC_API_KEY"): - config.add_provider( - name="anthropic", - provider_type="anthropic", - config={ - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "base_url": os.getenv("ANTHROPIC_BASE_URL"), - }, - priority=95, - ) - - # Always add local provider (will check Ollama availability during init) - config.add_provider( - name="local", - provider_type="local", - config={ - "base_url": os.getenv("OLLAMA_BASE_URL", "http://localhost:11434"), - }, - priority=50, - ) - - return create_registry_from_config(config) - - -def load_configuration_from_dict(config_dict: Dict[str, Any]) -> RegistryConfiguration: - """Load registry configuration from a dictionary.""" - config = RegistryConfiguration() - - # Load global settings - config.auto_discover = config_dict.get("auto_discover", True) - config.default_timeout = config_dict.get("default_timeout", 30.0) - config.max_retries = config_dict.get("max_retries", 3) - config.cache_models = config_dict.get("cache_models", True) - - # Load providers - for provider_dict in config_dict.get("providers", []): - spec = ModelProviderSpec( - name=provider_dict["name"], - type=provider_dict["type"], - config=provider_dict.get("config", {}), - enabled=provider_dict.get("enabled", True), - priority=provider_dict.get("priority", 0), - ) - config.providers.append(spec) - - return config - - -def configuration_to_dict(config: RegistryConfiguration) -> Dict[str, Any]: - """Convert registry configuration to dictionary.""" - return { - "auto_discover": config.auto_discover, - "default_timeout": config.default_timeout, - "max_retries": config.max_retries, - "cache_models": config.cache_models, - "providers": [ - { - "name": spec.name, - "type": spec.type, - "config": spec.config, - "enabled": spec.enabled, - "priority": spec.priority, - } - for spec in config.providers - ], - } - - -# Example configuration presets -CLOUD_ONLY_CONFIG = { - "providers": [ - { - "name": "openai", - "type": "openai", - "enabled": True, - "priority": 100, - }, - { - "name": "anthropic", - "type": "anthropic", - "enabled": True, - "priority": 95, - }, - { - "name": "local", - "type": "local", - "enabled": False, # Disabled for cloud-only - }, - ] -} - -LOCAL_ONLY_CONFIG = { - "providers": [ - { - "name": "openai", - "type": "openai", - "enabled": False, # Disabled for local-only - }, - { - "name": "anthropic", - "type": "anthropic", - "enabled": False, # Disabled for local-only - }, - { - "name": "local", - "type": "local", - "enabled": True, - "priority": 100, # High priority for local-only setup - }, - ] -} - -DEVELOPMENT_CONFIG = { - "auto_discover": True, - "cache_models": True, - "providers": [ - { - "name": "openai", - "type": "openai", - "config": {"timeout": 60.0}, # Longer timeout for development - "enabled": True, - "priority": 100, - }, - { - "name": "anthropic", - "type": "anthropic", - "config": {"timeout": 60.0}, - "enabled": True, - "priority": 95, - }, - { - "name": "local", - "type": "local", - "config": {"timeout": 120.0}, # Very long timeout for slow local models - "enabled": True, - "priority": 50, - }, - ] -} \ No newline at end of file diff --git a/src/orchestrator/models/huggingface_credentials.py b/src/orchestrator/models/huggingface_credentials.py new file mode 100644 index 00000000..76576f81 --- /dev/null +++ b/src/orchestrator/models/huggingface_credentials.py @@ -0,0 +1,115 @@ +"""Credential resolution for the HuggingFace Inference API. + +The HuggingFace router (https://router.huggingface.co/v1) is an +OpenAI-compatible gateway to the Inference Providers network. It needs one +bearer token -- a fine-grained token with "Make calls to Inference Providers" +permission. + +Resolution order, highest priority first: + +1. ``HF_TOKEN`` in the environment (the CI path). +2. ``~/.orchestrator/.env`` -- this project's own credential store. +3. ``~/.cache/huggingface/token`` -- the HuggingFace CLI's own store, written + by ``hf auth login``. Sharing that copy beats a second one that has to be + rotated separately (the same reason the Dartmouth resolver reads the + sibling llmxive store). + +Nothing here ever logs, prints, or returns a token inside an error message. +``ResolvedCredential`` and ``mask_key`` are shared with the Dartmouth adapter +rather than copied: one secret-hygiene implementation, audited once. +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path + +from .dartmouth_credentials import ResolvedCredential, mask_key # noqa: F401 + +logger = logging.getLogger(__name__) + +__all__ = [ + "HF_TOKEN_ENV_VAR", + "HuggingFaceCredentialError", + "ResolvedCredential", + "mask_key", + "resolve_huggingface_api_key", +] + +HF_TOKEN_ENV_VAR = "HF_TOKEN" + +#: This project's own store, shared with the rest of the CLI configuration. +_ORCHESTRATOR_ENV_FILE = Path.home() / ".orchestrator" / ".env" + +#: The HuggingFace CLI's store. Same machine, same user, same token. +_HF_CLI_TOKEN_FILE = Path.home() / ".cache" / "huggingface" / "token" + + +class HuggingFaceCredentialError(RuntimeError): + """Raised when no HuggingFace API token can be found.""" + + +def _read_env_file(path: Path, variable: str) -> str | None: + """Read ``variable`` from a ``KEY=value`` file, ignoring comments.""" + if not path.is_file(): + return None + try: + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + name, _, value = line.partition("=") + if name.strip() == variable: + return value.strip().strip('"').strip("'") or None + except OSError as exc: + logger.debug("Could not read %s: %s", path, exc) + return None + + +def _read_hf_cli_token(path: Path) -> str | None: + """Read the HF CLI token store: a plain file holding the raw token.""" + if not path.is_file(): + return None + try: + value = path.read_text(encoding="utf-8").strip() + except OSError as exc: + logger.debug("Could not read %s: %s", path, exc) + return None + return value or None + + +def resolve_huggingface_api_key(*, required: bool = True) -> ResolvedCredential | None: + """Find a HuggingFace API token. + + Args: + required: Raise :class:`HuggingFaceCredentialError` when nothing is + found. Pass ``False`` to probe availability without handling an + exception. + + Returns: + The credential and its source, or ``None`` when absent and + ``required`` is ``False``. + """ + env_value = os.environ.get(HF_TOKEN_ENV_VAR) + if env_value and env_value.strip(): + return ResolvedCredential(env_value.strip(), f"${HF_TOKEN_ENV_VAR}") + + orchestrator_value = _read_env_file(_ORCHESTRATOR_ENV_FILE, HF_TOKEN_ENV_VAR) + if orchestrator_value: + return ResolvedCredential(orchestrator_value, str(_ORCHESTRATOR_ENV_FILE)) + + cli_value = _read_hf_cli_token(_HF_CLI_TOKEN_FILE) + if cli_value: + return ResolvedCredential(cli_value, str(_HF_CLI_TOKEN_FILE)) + + if not required: + return None + raise HuggingFaceCredentialError( + "No HuggingFace API token found. Set " + f"{HF_TOKEN_ENV_VAR}, or add " + f"'{HF_TOKEN_ENV_VAR}=' to {_ORCHESTRATOR_ENV_FILE}, or run " + f"'hf auth login' (which writes {_HF_CLI_TOKEN_FILE}). " + "Create a fine-grained token with 'Make calls to Inference Providers' " + "permission at https://huggingface.co/settings/tokens." + ) diff --git a/src/orchestrator/models/huggingface_model.py b/src/orchestrator/models/huggingface_model.py new file mode 100644 index 00000000..dcde7f4c --- /dev/null +++ b/src/orchestrator/models/huggingface_model.py @@ -0,0 +1,672 @@ +"""HuggingFace Inference API model adapter. + +The HuggingFace router (https://router.huggingface.co/v1) is an +OpenAI-compatible gateway onto the Inference Providers network: one token, +many backend providers, server-side routing. Deliberately implemented against +the HTTP API with ``aiohttp`` -- already a core dependency -- rather than +through ``huggingface_hub``. The wire format is a documented, stable +OpenAI-compatible contract, and adding an SDK to speak it would put a heavy +dependency on the core install path for no capability gain (ADR 0001's +dependency policy). + +Free-first is enforced here rather than left to the caller. Unlike Dartmouth +Chat, nothing on this gateway is permanently zero-cost: a model is free only +while some live provider carries an ``is_free`` promo or explicit zero +pricing, and an *unpinned* request routes ``:fastest`` -- which may be a paid +provider. So a free model is pinned to its free provider on the wire +(``model_id:provider``), and a paid model is refused unless +:data:`ALLOW_PAID_ENV_VAR` is set. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +from typing import Any, Dict, Optional +from urllib.parse import urlsplit + +from ..core.model import Model, ModelCapabilities, ModelCost, ModelRequirements +from .dartmouth_model import ALLOW_PAID_ENV_VAR, paid_models_allowed +from .huggingface_credentials import mask_key, resolve_huggingface_api_key + +logger = logging.getLogger(__name__) + +__all__ = [ + "ALLOW_PAID_ENV_VAR", + "DEFAULT_BASE_URL", + "DEFAULT_MAX_TOKENS", + "DEFAULT_REQUEST_TIMEOUT_SECONDS", + "HuggingFaceInferenceModel", + "HuggingFaceModelError", + "InsecureEndpoint", + "ModelLoading", + "ModelUnavailable", + "PaidModelRefused", + "PaymentRequired", + "RateLimited", + "ReasoningTruncated", + "ReservedRequestField", + "validate_base_url", +] + +#: Overridable for testing against a different router. +DEFAULT_BASE_URL = os.environ.get( + "HF_ROUTER_BASE_URL", "https://router.huggingface.co/v1" +) + +#: Reasoning models spend tokens on `reasoning_content` *before* emitting any +#: `content`. With a small budget the whole allowance is consumed thinking and +#: `content` comes back absent -- observed live with +#: prism-ml/Ternary-Bonsai-27B-AWQ-4bit at max_tokens=32, which returned 31 +#: reasoning tokens and no answer. This default is large enough that a +#: reasoning model can finish and still answer. +DEFAULT_MAX_TOKENS = 2048 + +#: Generation can legitimately take minutes when a model cold-starts behind +#: the router, so this is far longer than ``ProviderConfig.timeout``'s 30s +#: default. A provider built with an explicit config uses that config's value. +DEFAULT_REQUEST_TIMEOUT_SECONDS = 300 + +#: Router error bodies are echoed into exceptions and logs for diagnosis. +#: They are attacker-influenced (a prompt can be reflected back), so they are +#: truncated and stripped of control characters first -- an unescaped newline +#: or carriage return lets one line of response forge additional log lines. +_MAX_ERROR_BODY_CHARS = 500 + + +def _safe_error_body(body: str) -> str: + """Render a router error body safely for an exception or log line.""" + collapsed = " ".join(body.split()) + if len(collapsed) > _MAX_ERROR_BODY_CHARS: + collapsed = collapsed[:_MAX_ERROR_BODY_CHARS] + "... (truncated)" + return collapsed + + +class HuggingFaceModelError(RuntimeError): + """Raised when a HuggingFace Inference API request fails.""" + + +class PaidModelRefused(HuggingFaceModelError): + """Raised when a paid model is requested without an explicit opt-in.""" + + +class ModelUnavailable(HuggingFaceModelError): + """Raised when the router is up but this model's providers are not. + + Distinct from :class:`HuggingFaceModelError` because it is transient and + model-specific: the router returns 502 "provider error" when every + provider serving a model is failing. A caller seeing this should try a + different model, not give up. + """ + + +class ModelLoading(ModelUnavailable): + """Raised when the model is cold-starting behind the router (HTTP 503). + + Carries the router's ``estimated_time`` when present so a caller can + decide whether to wait or move on; :meth:`HuggingFaceProvider.generate_free` + moves on. + """ + + def __init__(self, message: str, *, estimated_seconds: Optional[float] = None): + super().__init__(message) + self.estimated_seconds = estimated_seconds + + +class RateLimited(HuggingFaceModelError): + """Raised on HTTP 429. Account-level, not model-specific. + + Walking a fallback chain of other models does not help -- every request + draws on the same account quota -- so ``generate_free`` deliberately lets + this propagate rather than hammering the API. + """ + + def __init__(self, message: str, *, retry_after: Optional[float] = None): + super().__init__(message) + self.retry_after = retry_after + + +class PaymentRequired(HuggingFaceModelError): + """Raised on HTTP 402: the account's included monthly credits are gone. + + Observed live (2026-08-21): the router answers ``402`` with "You have + depleted your monthly included credits". Account-level like + :class:`RateLimited`, but not transient on any useful timescale -- the + credits reset monthly -- so ``generate_free`` lets it propagate too. + """ + + +class ReservedRequestField(HuggingFaceModelError): + """Raised when a caller tries to override a field this adapter controls. + + ``model`` is the field the free/paid policy was checked against at + construction, so silently letting a request body override it converts an + approved free model into an unapproved paid one *after* the check. That is + a policy bypass, not a convenience, so it is refused rather than ignored. + """ + + +class InsecureEndpoint(HuggingFaceModelError): + """Raised when a router URL would send the bearer token in the clear.""" + + +class ReasoningTruncated(HuggingFaceModelError): + """Raised when a reasoning model spent its whole budget thinking. + + Separate from a generic failure because it is *recoverable two ways*: + raise ``max_tokens``, or ask a model that does not emit a reasoning + scratchpad. :meth:`HuggingFaceProvider.generate_free` uses the second. + """ + + +#: Substrings in a router error body that mean "this model's backends are +#: down" rather than "your request was wrong". +_UNAVAILABLE_MARKERS = ( + "cannot connect to host", + "model_not_loaded", + "no healthy upstream", + "provider error", + "service unavailable", + "temporarily unavailable", +) + +#: Substrings that mean the 503 is a cold start, not an outage. +_LOADING_MARKERS = ("currently loading", "model is loading") + + +def _looks_unavailable(body: str) -> bool: + lowered = body.lower() + return any(marker in lowered for marker in _UNAVAILABLE_MARKERS) + + +def _parse_estimated_seconds(body: str) -> Optional[float]: + """Pull ``estimated_time`` out of a model-loading body, when present.""" + try: + payload = json.loads(body) + except (json.JSONDecodeError, ValueError): + return None + if isinstance(payload, dict): + value = payload.get("estimated_time") + if isinstance(value, (int, float)) and value >= 0: + return float(value) + return None + + +def _parse_retry_after(headers: Any) -> Optional[float]: + """Seconds until the rate limit resets, from ``Retry-After`` when sent.""" + value = (headers or {}).get("Retry-After") + if value is None: + return None + try: + seconds = float(value) + except (TypeError, ValueError): + return None + return seconds if seconds >= 0 else None + + +#: Request fields this adapter owns. A caller may tune sampling, penalties and +#: the token budget, but not these -- see :class:`ReservedRequestField`. +#: ``stream`` is included because a streamed reply is a series of SSE events, +#: which ``_extract_text`` would misread as a malformed response. +RESERVED_REQUEST_FIELDS = frozenset({"model", "messages", "stream"}) + +#: Hosts allowed to serve the router over plaintext HTTP. Every request +#: carries the bearer token in an ``Authorization`` header, so anything that +#: leaves the machine must be TLS. Loopback is exempt so a local mock router +#: remains testable. +_PLAINTEXT_OK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"}) + + +def validate_base_url(url: str) -> str: + """Return ``url`` normalised, or raise if it would leak the credential. + + A mistyped or hostile ``base_url`` receives the bearer token on the very + first request, so the scheme is checked before any call is made rather + than trusted. + + Raises: + InsecureEndpoint: if the URL is malformed, or is plaintext HTTP to + anything other than loopback. + """ + cleaned = url.strip().rstrip("/") + parts = urlsplit(cleaned) + + if not parts.scheme or not parts.netloc: + raise InsecureEndpoint( + f"HuggingFace base_url {url!r} is not a valid absolute URL. " + f"Expected something like {DEFAULT_BASE_URL!r}." + ) + if parts.scheme == "https": + return cleaned + if parts.scheme == "http" and (parts.hostname or "") in _PLAINTEXT_OK_HOSTS: + return cleaned + raise InsecureEndpoint( + f"HuggingFace base_url {url!r} uses {parts.scheme!r}, which would send " + f"the API token unencrypted to {parts.hostname!r}. Use https:// " + f"(plaintext http:// is permitted only for " + f"{', '.join(sorted(_PLAINTEXT_OK_HOSTS))})." + ) + + +class HuggingFaceInferenceModel(Model): + """A chat model served through the HuggingFace Inference Providers router. + + Named to stay distinct from the retired local-transformers + ``HuggingFaceModel`` adapter (#430): this class speaks only to the hosted + router and never downloads weights. + """ + + def __init__( + self, + name: str, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + route: Optional[str] = None, + capabilities: Optional[ModelCapabilities] = None, + requirements: Optional[ModelRequirements] = None, + cost: Optional[ModelCost] = None, + timeout: float = DEFAULT_REQUEST_TIMEOUT_SECONDS, + max_retries: int = 3, + retry_delay: float = 1.0, + **kwargs: Any, + ) -> None: + """Initialize the adapter. + + Args: + name: A Hub model id the router serves, e.g. ``openai/gpt-oss-120b``. + api_key: Bearer token. Resolved from the environment or the local + credential stores when omitted. + base_url: Router root. Must be HTTPS unless it is loopback. + route: Provider to pin on the wire (``name:route``). The provider + attaches this for free models, where an unpinned request could + route to a paid provider and bill the account. + cost: Real pricing from the live catalog, normally supplied by + :meth:`HuggingFaceProvider.create_model`. Omitting it means + the price is **unknown**, which is treated as paid. + + Raises: + PaidModelRefused: if the model costs money, or its price is + unknown, and the paid opt-in is not set. + InsecureEndpoint: if ``base_url`` would leak the credential. + """ + # Distinguishes "the catalog says this is free" from "nobody asked the + # catalog". Only the former is safe to run without an opt-in. + self._pricing_is_known = cost is not None + + super().__init__( + name=name, + provider="huggingface", + # ModelCapabilities requires at least one task, so a bare + # ModelCapabilities() is not constructible. The provider supplies + # richer capabilities from the catalog; this is the floor for a + # model built directly. + capabilities=capabilities + or ModelCapabilities( + supported_tasks=["generate", "analyze", "transform", "summarize"], + supports_structured_output=True, + ), + requirements=requirements or ModelRequirements(), + # NOT `cost or ModelCost(is_free=True)`. Defaulting an unpriced + # model to free would let a typo'd model id skip the cost gate + # entirely and bill a real account. + cost=cost if cost is not None else ModelCost(is_free=False), + **kwargs, + ) + + credential = ( + None if api_key else resolve_huggingface_api_key(required=True) + ) + self._api_key = api_key or (credential.key if credential else "") + self._base_url = validate_base_url(base_url or DEFAULT_BASE_URL) + self.route = route + self._is_available = True + self._timeout = timeout + self._max_retries = max_retries + self._retry_delay = retry_delay + # Created on first request and reused, so a fallback chain walking + # several models does not pay a fresh TLS handshake each time. Closed + # by aclose(); see the async-context-manager support below. + self._session: Optional[Any] = None + + if credential is not None: + logger.debug( + "HuggingFace credential %s resolved from %s", + mask_key(self._api_key), + credential.source, + ) + + self._enforce_cost_policy() + + def _enforce_cost_policy(self) -> None: + """Refuse a paid model unless the operator opted in. + + Checked at construction rather than at call time so the failure lands + where the model was chosen, not deep inside a pipeline run. + + Unknown pricing is refused alongside known-paid pricing. The router + catalog is the only authority on what a model costs, so a model built + without consulting it has an unknown price -- and treating unknown as + free is precisely the assumption that spends money by accident. + """ + if paid_models_allowed(): + return + if not self._pricing_is_known: + raise PaidModelRefused( + f"{self.name!r} was constructed without pricing, so its cost " + f"is unknown and it is treated as paid. Build it through " + f"HuggingFaceProvider.create_model(), which attaches real " + f"pricing from the live catalog, or pass an explicit cost=. " + f"Set {ALLOW_PAID_ENV_VAR}=1 to permit unpriced and paid usage." + ) + if self.cost.is_free: + return + raise PaidModelRefused( + f"{self.name!r} costs money " + f"(input ${self.cost.input_cost_per_1k_tokens:.6f}/1k, " + f"output ${self.cost.output_cost_per_1k_tokens:.6f}/1k) and " + f"{ALLOW_PAID_ENV_VAR} is not set to '1'. The router catalog " + f"marks free routes -- see HuggingFaceProvider.list_free_models() " + f"-- or set {ALLOW_PAID_ENV_VAR}=1 to permit paid usage." + ) + + @property + def api_key_is_set(self) -> bool: + """Whether a credential is available. Never exposes the token itself.""" + return bool(self._api_key) + + @property + def _wire_model_id(self) -> str: + """The model id sent on the wire, with any provider pin applied.""" + return f"{self.name}:{self.route}" if self.route else self.name + + async def _get_session(self) -> Any: + """Return the shared HTTP session, opening it on first use.""" + import aiohttp + + if self._session is None or self._session.closed: + self._session = aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout(total=self._timeout), + headers={ + "Authorization": f"Bearer {self._api_key}", + "Content-Type": "application/json", + }, + ) + return self._session + + async def aclose(self) -> None: + """Close the shared HTTP session. Safe to call more than once.""" + if self._session is not None and not self._session.closed: + await self._session.close() + self._session = None + + async def __aenter__(self) -> "HuggingFaceInferenceModel": + return self + + async def __aexit__(self, *exc_info: Any) -> None: + await self.aclose() + + async def _post(self, path: str, payload: Dict[str, Any]) -> Dict[str, Any]: + """POST JSON to the router and return the decoded response. + + Transport failures (connection reset, DNS blip) are retried + ``max_retries`` times. A *model* being down, cold-starting, or the + account being rate-limited is deliberately NOT retried in place: + cold starts and outages outlast any sensible retry loop, and a 429 is + account-level, so retrying just hammers the same quota. Callers with a + fallback chain (:meth:`HuggingFaceProvider.generate_free`) move to the + next model instead. + """ + import aiohttp + + url = f"{self._base_url}/{path.lstrip('/')}" + last_error: Optional[Exception] = None + + for attempt in range(self._max_retries + 1): + try: + session = await self._get_session() + async with session.post(url, json=payload) as response: + body = await response.text() + if response.status >= 400: + # The body may echo the request but never the bearer + # token. It is still attacker-influenced, so it is + # sanitised before going into an exception or a log. + detail = ( + f"HuggingFace router returned HTTP " + f"{response.status} for model {self.name!r}: " + f"{_safe_error_body(body)}" + ) + if response.status == 429: + raise RateLimited( + detail, + retry_after=_parse_retry_after(response.headers), + ) + if response.status == 402: + raise PaymentRequired(detail) + if response.status == 503 and any( + marker in body.lower() for marker in _LOADING_MARKERS + ): + raise ModelLoading( + detail, + estimated_seconds=_parse_estimated_seconds(body), + ) + if response.status >= 500 or _looks_unavailable(body): + raise ModelUnavailable(detail) + raise HuggingFaceModelError(detail) + try: + return json.loads(body) + except json.JSONDecodeError as exc: + # A maintenance window serves an HTML redirect page. + raise HuggingFaceModelError( + f"HuggingFace router returned a non-JSON response " + f"for {self.name!r} (is the router in " + f"maintenance?): {_safe_error_body(body)}" + ) from exc + except aiohttp.ClientError as exc: + last_error = exc + if attempt < self._max_retries: + logger.warning( + "HuggingFace transport error for %s (attempt %d/%d), " + "retrying: %s", + self.name, + attempt + 1, + self._max_retries + 1, + exc, + ) + await asyncio.sleep(self._retry_delay * (attempt + 1)) + # The session may be poisoned by the failure; drop it so + # the next attempt opens a fresh connection. + await self.aclose() + + raise HuggingFaceModelError( + f"HuggingFace router request failed for {self.name!r} after " + f"{self._max_retries + 1} attempts: {last_error}" + ) from last_error + + @staticmethod + def _extract_text(response: Dict[str, Any], model_name: str) -> str: + """Pull the assistant text out of a chat-completion response. + + Reasoning models put their scratchpad in ``reasoning_content`` and the + answer in ``content``. When the token budget is exhausted while + thinking, ``content`` is absent -- which is a truncation, not an empty + answer, and must not be returned as an empty string. + """ + choices = response.get("choices") or [] + if not choices: + raise HuggingFaceModelError( + f"HuggingFace router returned no choices for {model_name!r}" + ) + choice = choices[0] + message = choice.get("message") or {} + content = message.get("content") + + if content: + return content + + finish_reason = choice.get("finish_reason") + if message.get("reasoning_content"): + raise ReasoningTruncated( + f"{model_name!r} produced only reasoning tokens and no answer " + f"(finish_reason={finish_reason!r}). This is a reasoning " + f"model: raise max_tokens (default {DEFAULT_MAX_TOKENS}) so it " + f"can finish thinking and still reply." + ) + raise HuggingFaceModelError( + f"{model_name!r} returned an empty response " + f"(finish_reason={finish_reason!r})" + ) + + async def generate( + self, + prompt: str, + temperature: float = 0.7, + max_tokens: Optional[int] = None, + **kwargs: Any, + ) -> str: + """Generate text from ``prompt``. + + Extra ``kwargs`` are forwarded to the router as request fields, so + sampling parameters work as expected. Fields in + :data:`RESERVED_REQUEST_FIELDS` are refused rather than forwarded. + + Raises: + ReservedRequestField: if ``kwargs`` would override a field this + adapter controls -- notably ``model``, which the cost policy + was checked against. + """ + reserved = sorted(RESERVED_REQUEST_FIELDS.intersection(kwargs)) + if reserved: + raise ReservedRequestField( + f"cannot override {', '.join(repr(f) for f in reserved)} on a " + f"request to {self.name!r}: these fields are set by the " + f"adapter. Overriding 'model' in particular would bypass the " + f"free/paid check already made for {self.name!r} -- construct " + f"a different model instead." + ) + + messages = [] + system_prompt = kwargs.pop("system_prompt", None) + if system_prompt and system_prompt.strip(): + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": prompt}) + + # Caller-supplied fields go in first and the controlled fields are + # written over them, so the reserved-field check above is belt and + # braces: even if it were bypassed, `model` still cannot be swapped. + payload: Dict[str, Any] = dict(kwargs) + payload.update( + { + "model": self._wire_model_id, + "messages": messages, + "temperature": temperature, + "max_tokens": max_tokens or DEFAULT_MAX_TOKENS, + } + ) + + response = await self._post("chat/completions", payload) + return self._extract_text(response, self.name) + + async def generate_structured( + self, + prompt: str, + schema: Dict[str, Any], + temperature: float = 0.7, + **kwargs: Any, + ) -> Dict[str, Any]: + """Generate JSON conforming to ``schema``. + + Only some providers behind the router honour a ``response_format`` + parameter (the catalog's ``supports_structured_output`` flag), so the + schema is stated in the prompt instead and the reply is parsed. A + reply that is not valid JSON raises rather than being silently coerced + into a string. + """ + instruction = ( + f"{prompt}\n\n" + f"Respond with JSON only -- no prose, no markdown fences -- " + f"conforming to this JSON Schema:\n{json.dumps(schema, indent=2)}" + ) + text = await self.generate(instruction, temperature=temperature, **kwargs) + + cleaned = text.strip() + if cleaned.startswith("```"): + # Models frequently fence JSON despite being told not to. + cleaned = cleaned.split("```", 2)[1] if "```" in cleaned[3:] else cleaned + cleaned = cleaned.removeprefix("json").strip().strip("`").strip() + try: + parsed = json.loads(cleaned) + except json.JSONDecodeError as exc: + raise HuggingFaceModelError( + f"{self.name!r} did not return valid JSON: {exc}. " + f"Response began: {text[:200]!r}" + ) from exc + if not isinstance(parsed, dict): + raise HuggingFaceModelError( + f"{self.name!r} returned a {type(parsed).__name__}, expected a " + f"JSON object" + ) + + # Parsing proves the reply is JSON, not that it is the JSON that was + # asked for. Small models routinely return well-formed objects with + # the wrong keys or types, which would otherwise flow downstream as if + # the schema had been honoured. + import jsonschema + + try: + jsonschema.validate(instance=parsed, schema=schema) + except jsonschema.ValidationError as exc: + raise HuggingFaceModelError( + f"{self.name!r} returned JSON that does not match the " + f"requested schema at {list(exc.absolute_path) or ''}: " + f"{exc.message}. Received: {json.dumps(parsed)[:200]}" + ) from exc + except jsonschema.SchemaError as exc: + raise HuggingFaceModelError( + f"the schema passed to generate_structured() is itself " + f"invalid: {exc.message}" + ) from exc + return parsed + + async def health_check(self) -> bool: + """Whether the router will serve this model.""" + try: + await self.generate("ping", temperature=0.0, max_tokens=DEFAULT_MAX_TOKENS) + return True + except Exception as exc: # noqa: BLE001 - health checks report, not raise + logger.warning( + "HuggingFace health check failed for %s: %s", self.name, exc + ) + return False + finally: + # A health check is a one-shot probe, so it must not leave a + # session open behind it. + await self.aclose() + + async def estimate_cost( + self, + prompt: str, + max_tokens: Optional[int] = None, + ) -> float: + """Estimate USD cost. Exactly 0.0 for the free routes. + + Raises: + HuggingFaceModelError: if the model was built without pricing. The + zero-filled default would otherwise report $0.00 for a model + that may well bill -- a confidently wrong budget number is + worse than an error. + """ + if not self._pricing_is_known: + raise HuggingFaceModelError( + f"cannot estimate cost for {self.name!r}: it was constructed " + f"without pricing. Build it through " + f"HuggingFaceProvider.create_model() to attach real pricing." + ) + if self.cost.is_free: + return 0.0 + # ~4 characters per token is the usual rough English estimate; this is + # a budgeting aid, not billing. + input_tokens = max(1, len(prompt) // 4) + output_tokens = max_tokens or DEFAULT_MAX_TOKENS + return self.cost.calculate_cost(input_tokens, output_tokens) diff --git a/src/orchestrator/models/openai_model.py b/src/orchestrator/models/openai_model.py deleted file mode 100644 index ae83e5be..00000000 --- a/src/orchestrator/models/openai_model.py +++ /dev/null @@ -1,626 +0,0 @@ -"""OpenAI model adapter implementation with LangChain backend support.""" - -from __future__ import annotations - -import os -import asyncio -import logging -from typing import Any, Dict, Optional - -from ..core.model import Model, ModelCapabilities, ModelRequirements, ModelCost -from ..utils.auto_install import safe_import -from ..utils.api_keys_flexible import ensure_api_key - -logger = logging.getLogger(__name__) - - -class OpenAIModel(Model): - """OpenAI model implementation.""" - - def __init__( - self, - name: str, - api_key: Optional[str] = None, - organization: Optional[str] = None, - base_url: Optional[str] = None, - capabilities: Optional[ModelCapabilities] = None, - requirements: Optional[ModelRequirements] = None, - use_langchain: bool = True, - **kwargs: Any, - ) -> None: - """ - Initialize OpenAI model with LangChain backend support. - - Args: - name: Model name (e.g., "gpt-4", "gpt-3.5-turbo") - api_key: OpenAI API key (defaults to OPENAI_API_KEY env var) - organization: OpenAI organization ID (optional) - base_url: Custom API base URL (optional) - capabilities: Model capabilities - requirements: Resource requirements - use_langchain: Whether to try using LangChain backend (defaults to True) - **kwargs: Additional arguments - """ - # Set default capabilities based on model - if capabilities is None: - capabilities = self._get_default_capabilities(name) - - # Set default requirements - if requirements is None: - requirements = ModelRequirements( - memory_gb=0.5, - cpu_cores=1, - requires_gpu=False, - disk_space_gb=0.1, - ) - - # Set cost information - cost = self._get_model_cost(name) - - super().__init__( - name=name, - provider="openai", - capabilities=capabilities, - requirements=requirements, - cost=cost, - ) - - # Get API key using existing infrastructure - self.api_key = api_key - if not self.api_key: - try: - self.api_key = ensure_api_key("openai") - except Exception: - self.api_key = os.getenv("OPENAI_API_KEY") - if not self.api_key: - raise ValueError("OpenAI API key not provided") - - # Try to initialize LangChain model first - self.langchain_model = None - self._use_langchain = False - - if use_langchain: - try: - langchain_openai = safe_import("langchain_openai", auto_install=True) - if langchain_openai: - # Prepare model kwargs based on model type - model_kwargs = { - "model": name, - "api_key": self.api_key, - "organization": organization, - "base_url": base_url, - "temperature": kwargs.get("temperature", 0.7), - } - - # Handle max_tokens vs max_completion_tokens for GPT-5 - max_tokens_value = kwargs.get("max_tokens") - if max_tokens_value: - if "gpt-5" in name.lower(): - model_kwargs["max_completion_tokens"] = max_tokens_value - else: - model_kwargs["max_tokens"] = max_tokens_value - - # Add remaining kwargs - for k, v in kwargs.items(): - if k not in ['temperature', 'max_tokens', 'max_completion_tokens']: - model_kwargs[k] = v - - self.langchain_model = langchain_openai.ChatOpenAI(**model_kwargs) - self._use_langchain = True - logger.info(f"Using LangChain backend for OpenAI model: {name}") - else: - logger.warning(f"LangChain not available, falling back to direct OpenAI for: {name}") - except Exception as e: - logger.warning(f"Failed to initialize LangChain OpenAI model: {e}, falling back to direct OpenAI") - - # Always initialize direct OpenAI client for image generation - # (even if using LangChain for text) - try: - from openai import AsyncOpenAI - except ImportError as exc: - raise ImportError( - "OpenAI models require the 'openai' package. " - "Install it with: pip install 'py-orc[openai]'" - ) from exc - - self.client = AsyncOpenAI( - api_key=self.api_key, - organization=organization, - base_url=base_url, - ) - - if not self._use_langchain: - logger.info(f"Using direct OpenAI client for model: {name}") - - # Set model-specific attributes (preserve existing functionality) - self._model_id = self._normalize_model_name(name) - self._expertise = self._get_model_expertise(name) - self._size_billions = self._estimate_model_size(name) - self._is_available = True - - def _normalize_model_name(self, name: str) -> str: - """Normalize model name to OpenAI format.""" - # Handle common variations - name_lower = name.lower() - - # GPT-4.1 variations - if "gpt-4.1" in name_lower or "gpt-41" in name_lower: - if "mini" in name_lower: - return "gpt-4-0125-preview" # Using latest GPT-4 as substitute - return "gpt-4-turbo-preview" - - # GPT-4 variations - if name_lower.startswith("gpt-4"): - if "turbo" in name_lower: - return "gpt-4-turbo-preview" - elif "32k" in name_lower: - return "gpt-4-32k" - return "gpt-4" - - # GPT-3.5 variations - if "gpt-3.5" in name_lower or "gpt-35" in name_lower: - if "16k" in name_lower: - return "gpt-3.5-turbo-16k" - return "gpt-3.5-turbo" - - # Default: return as-is - return name - - def _get_default_capabilities(self, name: str) -> ModelCapabilities: - """Get default capabilities based on model name.""" - name_lower = name.lower() - - # DALL-E models for image generation - if "dall-e" in name_lower: - return ModelCapabilities( - supported_tasks=[ - "image-generation", - "generate-image", - "create-image" - ], - context_window=4000, # Prompt length limit - supports_function_calling=False, - supports_structured_output=False, - supports_streaming=False, - languages=["en"], # DALL-E works best with English - max_tokens=4000, # Prompt token limit - temperature_range=(0.0, 1.0), - domains=["visual", "creative", "artistic"], - vision_capable=False, # Generates images, doesn't analyze them - code_specialized=False, - supports_tools=False - ) - - # GPT-4 models - if "gpt-4" in name_lower: - context_window = 128000 if "turbo" in name_lower else 8192 - if "32k" in name_lower: - context_window = 32768 - - # Check if it's a vision model - is_vision = "vision" in name_lower or "gpt-4-turbo" in name_lower or "gpt-4o" in name_lower - - tasks = [ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "creative", - "chat", - "instruct", - ] - - if is_vision: - tasks.extend(["vision", "image-analysis", "visual-reasoning"]) - - return ModelCapabilities( - supported_tasks=tasks, - context_window=context_window, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=4096, - temperature_range=(0.0, 2.0), - domains=["general", "technical", "creative", "business", "visual"] if is_vision else ["general", "technical", "creative", "business"], - vision_capable=is_vision, - code_specialized=True, - supports_tools=True, - supports_json_mode=True, - accuracy_score=0.95, - speed_rating="medium", - ) - - # GPT-3.5 models - elif "gpt-3.5" in name_lower: - context_window = 16385 if "16k" in name_lower else 4096 - - return ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "chat", - "instruct", - ], - context_window=context_window, - supports_function_calling=True, - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=4096, - temperature_range=(0.0, 2.0), - domains=["general", "technical"], - code_specialized=True, - supports_tools=True, - supports_json_mode=True, - accuracy_score=0.85, - speed_rating="fast", - ) - - # Default capabilities - return ModelCapabilities( - supported_tasks=["generate", "chat"], - context_window=4096, - supports_function_calling=False, - supports_structured_output=False, - supports_streaming=True, - languages=["en"], - max_tokens=2048, - ) - - def _get_model_expertise(self, name: str) -> list[str]: - """Get model expertise areas.""" - name_lower = name.lower() - - if "gpt-4" in name_lower: - return ["general", "reasoning", "code", "creative", "analysis"] - elif "gpt-3.5" in name_lower: - return ["general", "chat", "instruct"] - - return ["general"] - - def _estimate_model_size(self, name: str) -> float: - """Estimate model size in billions of parameters.""" - name_lower = name.lower() - - if "gpt-4" in name_lower: - return 1760.0 # Estimated - elif "gpt-3.5" in name_lower: - return 175.0 - - return 1.0 - - def _get_model_cost(self, name: str) -> ModelCost: - """Get cost information for model.""" - name_lower = name.lower() - - # GPT-4 pricing (as of 2024) - if "gpt-4" in name_lower: - if "turbo" in name_lower or "preview" in name_lower: - return ModelCost( - input_cost_per_1k_tokens=0.01, - output_cost_per_1k_tokens=0.03, - is_free=False, - ) - else: - return ModelCost( - input_cost_per_1k_tokens=0.03, - output_cost_per_1k_tokens=0.06, - is_free=False, - ) - - # GPT-3.5 pricing - elif "gpt-3.5" in name_lower: - return ModelCost( - input_cost_per_1k_tokens=0.0005, - output_cost_per_1k_tokens=0.0015, - is_free=False, - ) - - # Default pricing for unknown models - else: - return ModelCost( - input_cost_per_1k_tokens=0.002, - output_cost_per_1k_tokens=0.002, - is_free=False, - ) - - async def generate( - self, - prompt: str, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """ - Generate text from prompt using LangChain or direct OpenAI. - - Args: - prompt: Input prompt - temperature: Sampling temperature - max_tokens: Maximum tokens to generate - **kwargs: Additional parameters - - Returns: - Generated text - """ - # Use LangChain if available - if self._use_langchain and self.langchain_model: - try: - return await self._langchain_generate(prompt, temperature, max_tokens, **kwargs) - except Exception as e: - logger.warning(f"LangChain generation failed, falling back to direct OpenAI: {e}") - # Fall through to direct OpenAI implementation - - # Fallback to direct OpenAI implementation (preserve original functionality) - return await self._direct_openai_generate(prompt, temperature, max_tokens, **kwargs) - - async def _langchain_generate( - self, - prompt: str, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """Generate using LangChain backend.""" - try: - # Filter out non-LangChain kwargs - # LangChain only accepts specific parameters, not arbitrary config - langchain_kwargs = {} - if temperature != 0.7: - langchain_kwargs['temperature'] = temperature - if max_tokens: - langchain_kwargs['max_tokens'] = max_tokens - - # Handle system prompt for LangChain - if "system_prompt" in kwargs: - from langchain_core.messages import SystemMessage, HumanMessage - messages = [ - SystemMessage(content=kwargs["system_prompt"]), - HumanMessage(content=prompt) - ] - - if hasattr(self.langchain_model, 'ainvoke'): - response = await self.langchain_model.ainvoke(messages, **langchain_kwargs) - else: - response = await asyncio.to_thread(self.langchain_model.invoke, messages, **langchain_kwargs) - else: - # Simple prompt - if hasattr(self.langchain_model, 'ainvoke'): - response = await self.langchain_model.ainvoke(prompt, **langchain_kwargs) - else: - response = await asyncio.to_thread(self.langchain_model.invoke, prompt, **langchain_kwargs) - - # Extract content from LangChain response - if hasattr(response, 'content'): - return response.content - else: - return str(response) - - except Exception as e: - raise RuntimeError(f"LangChain OpenAI generation failed: {str(e)}") - - async def _direct_openai_generate( - self, - prompt: str, - temperature: float = 0.7, - max_tokens: Optional[int] = None, - **kwargs: Any, - ) -> str: - """Generate using direct OpenAI client (original implementation).""" - try: - # Prepare messages - messages = [{"role": "user", "content": prompt}] - - # Add system message if provided - if "system_prompt" in kwargs: - messages.insert( - 0, {"role": "system", "content": kwargs["system_prompt"]} - ) - # For GPT-5, add system message if JSON is expected and no system prompt provided - elif "gpt-5" in self._model_id.lower() and ("json" in prompt.lower() or "JSON" in prompt): - messages.insert( - 0, {"role": "system", "content": "You are a helpful assistant that provides structured responses. When asked for JSON, always return valid JSON."} - ) - - # Prepare API call parameters - api_params = { - "model": self._model_id, - "messages": messages, - "n": 1, - "stream": False, - } - - # GPT-5 only supports temperature=1 - if "gpt-5" in self._model_id.lower(): - api_params["temperature"] = 1.0 - else: - api_params["temperature"] = temperature - - # Handle max_tokens vs max_completion_tokens based on model - max_tokens_value = max_tokens or self.capabilities.max_tokens - if "gpt-5" in self._model_id.lower(): - # GPT-5 models use max_completion_tokens - api_params["max_completion_tokens"] = max_tokens_value - else: - # Older models use max_tokens - api_params["max_tokens"] = max_tokens_value - - # Make API call - response = await self.client.chat.completions.create(**api_params) - - # Extract response - if response.choices and response.choices[0].message: - return response.choices[0].message.content or "" - - return "" - - except Exception as e: - # Log error and raise - raise RuntimeError(f"OpenAI generation failed: {str(e)}") - - async def generate_image( - self, - prompt: str, - size: str = "1024x1024", - quality: str = "standard", - style: str = "vivid", - **kwargs: Any, - ) -> Dict[str, Any]: - """ - Generate image using DALL-E 3. - - Args: - prompt: Text description of the image to generate - size: Image size (1024x1024, 1792x1024, or 1024x1792) - quality: Image quality (standard or hd) - style: Style (vivid or natural) - **kwargs: Additional parameters - - Returns: - Dictionary with image URL and metadata - """ - # Check if this model supports image generation - if "dall-e" not in self._model_id.lower(): - raise ValueError(f"Model {self._model_id} doesn't support image generation. Use dall-e-3 or dall-e-2.") - - try: - # Make API call to DALL-E - response = await self.client.images.generate( - model=self._model_id, - prompt=prompt, - size=size, - quality=quality, - style=style, - n=1 # DALL-E 3 only supports n=1 - ) - - # Extract image data - if response.data and len(response.data) > 0: - image_data = response.data[0] - return { - "url": image_data.url, - "revised_prompt": getattr(image_data, 'revised_prompt', prompt), - "size": size, - "quality": quality, - "style": style, - "data": [{"url": image_data.url}] # For compatibility - } - - raise RuntimeError("No image data in response") - - except Exception as e: - logger.error(f"Image generation failed: {str(e)}") - raise RuntimeError(f"DALL-E generation failed: {str(e)}") - - async def generate_structured( - self, - prompt: str, - schema: Dict[str, Any], - temperature: float = 0.7, - **kwargs: Any, - ) -> Dict[str, Any]: - """ - Generate structured output from prompt. - - Args: - prompt: Input prompt - schema: JSON schema for output - temperature: Sampling temperature - **kwargs: Additional parameters - - Returns: - Structured output - """ - try: - # Add schema instruction to prompt - import json - schema_prompt = f"{prompt}\n\nPlease respond with valid JSON matching this schema:\n{json.dumps(schema, indent=2)}" - - # Generate response using existing generate method (handles LangChain/direct OpenAI automatically) - response = await self.generate( - prompt=schema_prompt, - temperature=temperature, - **kwargs, - ) - - # Parse JSON response - try: - return json.loads(response) - except json.JSONDecodeError: - # Try to extract JSON from response - import re - json_match = re.search(r"\{.*\}", response, re.DOTALL) - if json_match: - return json.loads(json_match.group()) - raise ValueError("Could not parse JSON from response") - - except Exception as e: - raise RuntimeError(f"OpenAI structured generation failed: {str(e)}") - - async def health_check(self) -> bool: - """ - Check if model is available and healthy. - - Returns: - True if healthy - """ - try: - # Use the generate method which handles LangChain/direct OpenAI automatically - response = await self.generate("Hi", temperature=0.0, max_tokens=5) - return len(response) > 0 - except Exception: - return False - - async def estimate_cost( - self, - prompt: str, - max_tokens: Optional[int] = None, - ) -> float: - """ - Estimate cost for generation. - - Args: - prompt: Input prompt - max_tokens: Maximum tokens to generate - - Returns: - Estimated cost in USD - """ - # Estimate token counts - import tiktoken - - try: - encoding = tiktoken.encoding_for_model(self._model_id) - prompt_tokens = len(encoding.encode(prompt)) - except Exception: - # Fallback: rough estimate - prompt_tokens = len(prompt) // 4 - - output_tokens = max_tokens or 1000 - - # Cost per 1K tokens (approximate) - model_lower = self._model_id.lower() - if "gpt-4" in model_lower: - if "turbo" in model_lower: - input_cost = 0.01 # $0.01 per 1K input tokens - output_cost = 0.03 # $0.03 per 1K output tokens - else: - input_cost = 0.03 # $0.03 per 1K input tokens - output_cost = 0.06 # $0.06 per 1K output tokens - elif "gpt-3.5" in model_lower: - input_cost = 0.0005 # $0.0005 per 1K input tokens - output_cost = 0.0015 # $0.0015 per 1K output tokens - else: - # Default pricing - input_cost = 0.002 - output_cost = 0.002 - - # Calculate total cost - total_cost = (prompt_tokens / 1000 * input_cost) + ( - output_tokens / 1000 * output_cost - ) - return total_cost diff --git a/src/orchestrator/models/providers/__init__.py b/src/orchestrator/models/providers/__init__.py index 73ad3f52..d5044595 100644 --- a/src/orchestrator/models/providers/__init__.py +++ b/src/orchestrator/models/providers/__init__.py @@ -1,15 +1,14 @@ """Provider abstractions for unified model management. Concrete providers are resolved lazily so that importing the model registry -does not require every provider SDK to be installed. ``AnthropicProvider`` -needs the ``anthropic`` extra; accessing it without that extra raises the -underlying ``ImportError`` naming the missing package. +does not require provider SDKs. ``DartmouthProvider`` needs no extra at +all -- Dartmouth Chat is an OpenAI-compatible HTTP gateway and the adapter +speaks it with ``aiohttp``, which is already a core dependency. It also +serves several models at zero cost per token, so it is the cheapest way to +run this project against real models. -``DartmouthProvider`` needs no extra at all -- Dartmouth Chat is an -OpenAI-compatible HTTP gateway and the adapter speaks it with ``aiohttp``, -which is already a core dependency. It also serves several models at zero -cost per token, so it is the cheapest way to run this project against real -models. +The supported providers are Dartmouth Chat and the HuggingFace Inference API +(#484). The Anthropic provider was retired (#430) and is deliberately absent. """ from ..._lazy import lazy_exports @@ -18,8 +17,8 @@ "ModelProvider": ".base", "ProviderConfig": ".base", "ProviderError": ".base", - "AnthropicProvider": ".anthropic_provider", "DartmouthProvider": ".dartmouth_provider", + "HuggingFaceProvider": ".huggingface_provider", } __all__ = sorted(_EXPORTS) diff --git a/src/orchestrator/models/providers/anthropic_provider.py b/src/orchestrator/models/providers/anthropic_provider.py deleted file mode 100644 index 6ee2d59b..00000000 --- a/src/orchestrator/models/providers/anthropic_provider.py +++ /dev/null @@ -1,530 +0,0 @@ -"""Anthropic model provider implementation.""" - -from __future__ import annotations - -import logging -from typing import TYPE_CHECKING, Any, List, Optional - -if TYPE_CHECKING: - # Annotations are strings (`from __future__ import annotations`), so the - # SDK is only needed when a client is actually constructed. Importing it at - # module scope made the whole model registry unimportable without the - # `anthropic` extra. - from anthropic import AsyncAnthropic - -from ...core.model import ModelCapabilities, ModelCost, ModelRequirements -from ...utils.api_keys_flexible import ensure_api_key -from ...utils.auto_install import safe_import -from ..anthropic_model import AnthropicModel -from .base import ModelProvider, ProviderConfig, ProviderInitializationError, ModelNotSupportedError - -logger = logging.getLogger(__name__) - - -class AnthropicProvider(ModelProvider): - """Provider for Anthropic models.""" - - # Latest Anthropic models (2025) - Simplified for Claude Skills refactor - KNOWN_MODELS = { - # Claude Opus 4.1 (Released August 2025) - "claude-opus-4-1-20250805": { - "context_window": 200000, - "max_tokens": 8192, - "input_cost": 15.0 / 1000, # $15 per 1M tokens (estimated) - "output_cost": 75.0 / 1000, # $75 per 1M tokens (estimated) - "vision": True, - "function_calling": True, - "role": "review_and_analysis", - "description": "Most powerful Claude model for deep analysis and review", - "released": "2025-08-05", - }, - "claude-opus-4.1": { - "context_window": 200000, - "max_tokens": 8192, - "input_cost": 15.0 / 1000, - "output_cost": 75.0 / 1000, - "vision": True, - "function_calling": True, - "role": "review_and_analysis", - "description": "Most powerful Claude model for deep analysis and review", - }, - - # Claude Sonnet 4.5 (Released September 2025) - "claude-sonnet-4-5": { - "context_window": 1000000, # 1M token context window for API customers - "max_tokens": 8192, - "input_cost": 3.0 / 1000, # $3 per 1M tokens - "output_cost": 15.0 / 1000, # $15 per 1M tokens - "vision": True, - "function_calling": True, - "role": "orchestrator", - "description": "World's best coding model, optimal for building agents", - "released": "2025-09-29", - }, - "claude-sonnet-4.5": { - "context_window": 1000000, # 1M token context window - "max_tokens": 8192, - "input_cost": 3.0 / 1000, - "output_cost": 15.0 / 1000, - "vision": True, - "function_calling": True, - "role": "orchestrator", - "description": "World's best coding model, optimal for building agents", - }, - - # Claude Haiku 4.5 (Released October 2025) - "claude-haiku-4-5": { - "context_window": 200000, - "max_tokens": 8192, - "input_cost": 1.0 / 1000, # $1 per 1M tokens - "output_cost": 5.0 / 1000, # $5 per 1M tokens - "vision": True, - "function_calling": True, - "role": "simple_tasks", - "description": "90% of Sonnet 4.5's performance at 1/3 the cost", - "released": "2025-10-15", - }, - "claude-haiku-4.5": { - "context_window": 200000, - "max_tokens": 8192, - "input_cost": 1.0 / 1000, - "output_cost": 5.0 / 1000, - "vision": True, - "function_calling": True, - "role": "simple_tasks", - "description": "90% of Sonnet 4.5's performance at 1/3 the cost", - }, - - # Legacy models kept for backwards compatibility (will be deprecated) - "claude-3-5-sonnet-20241022": { - "context_window": 200000, - "max_tokens": 8192, - "input_cost": 3.0 / 1000, - "output_cost": 15.0 / 1000, - "vision": True, - "function_calling": True, - "deprecated": True, - }, - "claude-3-haiku-20240307": { - "context_window": 200000, - "max_tokens": 4096, - "input_cost": 0.25 / 1000, - "output_cost": 1.25 / 1000, - "vision": True, - "function_calling": True, - "deprecated": True, - }, - } - - def __init__(self, config: ProviderConfig) -> None: - """Initialize Anthropic provider.""" - super().__init__(config) - self._client: Optional[AsyncAnthropic] = None - - async def initialize(self) -> None: - """Initialize Anthropic provider.""" - try: - # Get API key - prioritize config, then use ensure_api_key - if self.config.api_key: - api_key = self.config.api_key - else: - api_key = ensure_api_key("anthropic") - - # Imported here so the module stays importable without the - # `anthropic` extra; only constructing a client requires the SDK. - try: - from anthropic import AsyncAnthropic - except ImportError as exc: # pragma: no cover - depends on install - raise ProviderInitializationError( - "The Anthropic provider requires the 'anthropic' package. " - "Install it with: pip install 'py-orc[anthropic]'" - ) from exc - - # Initialize client - self._client = AsyncAnthropic( - api_key=api_key, - base_url=self.config.base_url, - timeout=self.config.timeout, - max_retries=self.config.max_retries, - ) - - # Anthropic doesn't have a public models endpoint, so use known models - self._available_models = set(self.KNOWN_MODELS.keys()) - - # Test connectivity with a simple completion - try: - # Try new model first, fall back to current model - test_models = ["claude-haiku-4.5", "claude-3-haiku-20240307"] - for test_model in test_models: - try: - await self._client.messages.create( - model=test_model, - max_tokens=1, - messages=[{"role": "user", "content": "hi"}] - ) - logger.info(f"Anthropic provider initialized with {len(self._available_models)} models (tested with {test_model})") - break - except Exception as model_error: - if "not_found" in str(model_error).lower(): - continue - raise model_error - except Exception as e: - logger.warning(f"Could not test Anthropic connectivity: {e}. Provider may still work.") - - self._initialized = True - - except Exception as e: - raise ProviderInitializationError(f"Failed to initialize Anthropic provider: {e}") - - async def create_model(self, model_name: str, **kwargs: Any) -> AnthropicModel: - """Create an Anthropic model instance.""" - if not self.supports_model(model_name): - raise ModelNotSupportedError(f"Model '{model_name}' not supported by Anthropic provider") - - # Get model specifications - capabilities = self.get_model_capabilities(model_name) - requirements = self.get_model_requirements(model_name) - cost = self.get_model_cost(model_name) - - # Create model instance - return AnthropicModel( - name=model_name, - api_key=self._client.api_key if self._client else None, - base_url=self.config.base_url, - capabilities=capabilities, - requirements=requirements, - **kwargs - ) - - async def health_check(self) -> bool: - """Check if Anthropic provider is healthy.""" - if not self._initialized or not self._client: - return False - - # Listing models proves credentials and connectivity without spending - # tokens. The previous implementation generated against hard-coded - # model ids ("claude-haiku-4.5", "claude-3-haiku-20240307"); both now - # 404, so health_check reported unhealthy against a perfectly good key. - try: - await self._client.models.list() - return True - except Exception as e: - logger.warning(f"Anthropic health check failed: {e}") - return False - - async def discover_models(self) -> List[str]: - """Discover available Anthropic models from the Models API. - - Anthropic does provide a discovery endpoint; the previous comment - saying otherwise was stale, and returning a hard-coded table meant - newly released models were invisible and retired ones were still - advertised. - """ - if not self._initialized or not self._client: - return list(self.KNOWN_MODELS.keys()) - try: - listing = await self._client.models.list() - return [model.id for model in listing.data] - except Exception as e: - logger.warning( - f"Anthropic model discovery failed ({e}); " - f"falling back to the built-in table." - ) - return list(self.KNOWN_MODELS.keys()) - - def get_model_capabilities(self, model_name: str) -> ModelCapabilities: - """Get capabilities for an Anthropic model.""" - if not self.supports_model(model_name): - raise ModelNotSupportedError(f"Model '{model_name}' not supported by Anthropic provider") - - name_lower = model_name.lower() - model_info = self.KNOWN_MODELS.get(model_name, {}) - - # Claude Sonnet 4.5 (2025) - if "sonnet-4" in name_lower or "sonnet-4.5" in name_lower: - return ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "creative", - "chat", - "instruct", - "vision", - "math", - "research", - "orchestration", - "agent_building", - ], - context_window=model_info.get("context_window", 1000000), # 1M tokens - supports_function_calling=model_info.get("function_calling", True), - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=model_info.get("max_tokens", 8192), - temperature_range=(0.0, 1.0), - domains=["general", "technical", "creative", "business", "visual", "agent"], - vision_capable=model_info.get("vision", True), - code_specialized=True, - supports_tools=True, - supports_json_mode=True, - accuracy_score=0.98, # World's best coding model - speed_rating="medium", - ) - - # Claude Opus 4.1 (2025) - elif "opus-4" in name_lower or "opus-4.1" in name_lower: - return ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "creative", - "chat", - "instruct", - "vision", - "math", - "research", - "review", - "deep_analysis", - ], - context_window=model_info.get("context_window", 200000), - supports_function_calling=model_info.get("function_calling", True), - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=model_info.get("max_tokens", 8192), - temperature_range=(0.0, 1.0), - domains=["general", "technical", "creative", "business", "visual", "academic"], - vision_capable=model_info.get("vision", True), - code_specialized=True, - supports_tools=True, - supports_json_mode=True, - accuracy_score=0.99, # Most powerful for deep analysis - speed_rating="slow", - ) - - # Claude Haiku 4.5 (2025) - elif "haiku-4" in name_lower or "haiku-4.5" in name_lower: - return ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "chat", - "instruct", - "vision", - "simple_tasks", - ], - context_window=model_info.get("context_window", 200000), - supports_function_calling=model_info.get("function_calling", True), - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=model_info.get("max_tokens", 8192), - temperature_range=(0.0, 1.0), - domains=["general", "technical", "visual"], - vision_capable=model_info.get("vision", True), - code_specialized=True, - supports_tools=True, - supports_json_mode=True, - accuracy_score=0.90, # 90% of Sonnet 4.5's performance - speed_rating="fast", - ) - - # Legacy Claude 3 Opus (deprecated) - elif "opus" in name_lower and "opus-4" not in name_lower: - return ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "creative", - "chat", - "instruct", - "vision", - "math", - "research", - ], - context_window=model_info.get("context_window", 200000), - supports_function_calling=model_info.get("function_calling", True), - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=model_info.get("max_tokens", 4096), - temperature_range=(0.0, 1.0), - domains=["general", "technical", "creative", "business", "visual"], - vision_capable=model_info.get("vision", True), - code_specialized=True, - supports_tools=True, - supports_json_mode=True, - accuracy_score=0.98, # Highest accuracy - speed_rating="slow", - ) - - # Claude 3 Sonnet - elif "sonnet" in name_lower: - return ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "creative", - "chat", - "instruct", - "vision", - "math", - "research", - ], - context_window=model_info.get("context_window", 200000), - supports_function_calling=model_info.get("function_calling", True), - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=model_info.get("max_tokens", 4096), - temperature_range=(0.0, 1.0), - domains=["general", "technical", "creative", "business", "visual"], - vision_capable=model_info.get("vision", True), - code_specialized=True, - supports_tools=True, - supports_json_mode=True, - accuracy_score=0.93, - speed_rating="medium", - ) - - # Claude 3 Haiku - elif "haiku" in name_lower: - return ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "chat", - "instruct", - "vision", - ], - context_window=model_info.get("context_window", 200000), - supports_function_calling=model_info.get("function_calling", True), - supports_structured_output=True, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=model_info.get("max_tokens", 4096), - temperature_range=(0.0, 1.0), - domains=["general", "technical", "visual"], - vision_capable=model_info.get("vision", True), - code_specialized=True, - supports_tools=True, - supports_json_mode=True, - accuracy_score=0.88, - speed_rating="fast", - ) - - # Claude 2.x models - elif "claude-2" in name_lower: - return ModelCapabilities( - supported_tasks=[ - "generate", - "analyze", - "transform", - "code", - "reasoning", - "creative", - "chat", - "instruct", - ], - context_window=model_info.get("context_window", 100000), - supports_function_calling=model_info.get("function_calling", False), - supports_structured_output=False, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=model_info.get("max_tokens", 4096), - temperature_range=(0.0, 1.0), - domains=["general", "technical", "creative", "business"], - vision_capable=False, - code_specialized=True, - supports_tools=False, - supports_json_mode=False, - accuracy_score=0.90, - speed_rating="medium", - ) - - # Claude Instant - elif "instant" in name_lower: - return ModelCapabilities( - supported_tasks=[ - "generate", - "chat", - "instruct", - "transform", - ], - context_window=model_info.get("context_window", 100000), - supports_function_calling=False, - supports_structured_output=False, - supports_streaming=True, - languages=["en", "es", "fr", "de", "it", "pt", "ru", "ja", "ko", "zh"], - max_tokens=model_info.get("max_tokens", 4096), - temperature_range=(0.0, 1.0), - domains=["general"], - vision_capable=False, - code_specialized=False, - supports_tools=False, - supports_json_mode=False, - accuracy_score=0.82, - speed_rating="fast", - ) - - # Default capabilities for unknown models - return ModelCapabilities( - supported_tasks=["generate", "chat"], - context_window=model_info.get("context_window", 100000), - supports_function_calling=model_info.get("function_calling", False), - supports_structured_output=False, - supports_streaming=True, - languages=["en"], - max_tokens=model_info.get("max_tokens", 4096), - temperature_range=(0.0, 1.0), - domains=["general"], - code_specialized=False, - supports_tools=False, - accuracy_score=0.85, - speed_rating="medium", - ) - - def get_model_requirements(self, model_name: str) -> ModelRequirements: - """Get resource requirements for an Anthropic model.""" - if not self.supports_model(model_name): - raise ModelNotSupportedError(f"Model '{model_name}' not supported by Anthropic provider") - - # Anthropic models are cloud-hosted, so minimal local requirements - return ModelRequirements( - memory_gb=0.1, # Minimal memory for API client - gpu_memory_gb=None, # No local GPU needed - cpu_cores=1, - supports_quantization=[], # Not applicable for cloud models - min_python_version="3.8", - requires_gpu=False, - disk_space_gb=0.05, # Just for cached responses - ) - - def get_model_cost(self, model_name: str) -> ModelCost: - """Get cost information for an Anthropic model.""" - if not self.supports_model(model_name): - raise ModelNotSupportedError(f"Model '{model_name}' not supported by Anthropic provider") - - model_info = self.KNOWN_MODELS.get(model_name, {}) - - return ModelCost( - input_cost_per_1k_tokens=model_info.get("input_cost", 8.0 / 1000), # Default to Claude 2 pricing - output_cost_per_1k_tokens=model_info.get("output_cost", 24.0 / 1000), - is_free=False, - ) \ No newline at end of file diff --git a/src/orchestrator/models/providers/huggingface_provider.py b/src/orchestrator/models/providers/huggingface_provider.py new file mode 100644 index 00000000..04673d4a --- /dev/null +++ b/src/orchestrator/models/providers/huggingface_provider.py @@ -0,0 +1,438 @@ +"""HuggingFace Inference API provider. + +Model ids, pricing and route availability come from the live catalog at +``https://router.huggingface.co/v1/models``. Nothing is hard-coded: which +models carry a free route changes upstream (promos start and end), and a +stale local list would either miss free models or -- much worse -- offer a +paid one as if it were free. + +Cost semantics, verified against the live catalog (2026-08-21): each catalog +entry carries a ``providers`` list whose entries report ``status`` +(``live``/``error``), ``pricing`` in USD **per million** tokens when +available, and an ``is_free`` promo flag when applicable. A model is free +only while a **live** provider marks it free or prices it at exactly zero -- +and because an unpinned request routes ``:fastest``, a free model is pinned +to its free provider on the wire (``model_id:provider``) so the router cannot +send it to a paid one. +""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, List, Optional, Tuple + +from ...core.model import ModelCapabilities, ModelCost, ModelRequirements +from ..huggingface_credentials import resolve_huggingface_api_key +from ..huggingface_model import ( + DEFAULT_BASE_URL, + DEFAULT_MAX_TOKENS, + DEFAULT_REQUEST_TIMEOUT_SECONDS, + HuggingFaceInferenceModel, + HuggingFaceModelError, + ModelUnavailable, + ReasoningTruncated, + validate_base_url, +) +from .base import ModelProvider, ProviderConfig + +logger = logging.getLogger(__name__) + +__all__ = [ + "HuggingFaceProvider", + "catalog_price_is_known", + "fetch_catalog_sync", + "free_models_from_catalog", + "free_route_from_catalog", + "model_cost_from_catalog", +] + + +def _live_providers(entry: Dict[str, Any]) -> List[Dict[str, Any]]: + """The provider routes reported as serving the model right now. + + A route in ``error`` state is not a route: counting it would let a downed + free route mark a model free, or a downed paid route inflate its budget + estimate. + """ + return [ + p + for p in (entry.get("providers") or []) + if isinstance(p, dict) and p.get("status") == "live" + ] + + +def _explicit_pricing(provider: Dict[str, Any]) -> Optional[Tuple[float, float]]: + """``(input, output)`` USD per million tokens, or None when unreported.""" + pricing = provider.get("pricing") + if not isinstance(pricing, dict): + return None + input_price = pricing.get("input") + output_price = pricing.get("output") + if input_price is None or output_price is None: + return None + return float(input_price), float(output_price) + + +def _is_free_route(provider: Dict[str, Any]) -> bool: + """Whether this live route currently costs nothing. + + The ``is_free`` promo flag wins when present; otherwise explicit zero + pricing counts. Absent pricing does NOT -- absence of a price is not + evidence of zero price, and guessing "free" here is the failure mode that + spends real money. + """ + if provider.get("is_free") is True: + return True + pricing = _explicit_pricing(provider) + return pricing is not None and pricing == (0.0, 0.0) + + +def free_route_from_catalog(entry: Dict[str, Any]) -> Optional[str]: + """The provider a free model must be pinned to, or None if it is not free. + + The pin matters: an unpinned request routes ``:fastest``, and the fastest + provider is not necessarily the free one. + """ + for provider in _live_providers(entry): + if _is_free_route(provider) and provider.get("provider"): + return str(provider["provider"]) + return None + + +def model_cost_from_catalog(entry: Dict[str, Any]) -> ModelCost: + """Build a :class:`ModelCost` from one catalog entry. + + Free when a live provider offers a free route. Otherwise the most + expensive live route sets the estimate: routing is server-side, so + budgeting on the cheapest route would understate. An entry with **no** + pricing at all is treated as paid, not free. + """ + live = _live_providers(entry) + if any(_is_free_route(p) for p in live): + return ModelCost(is_free=True) + + priced = [p for p in (_explicit_pricing(p) for p in live) if p is not None] + if not priced: + return ModelCost(is_free=False) + + # The router reports USD per million tokens; ModelCost is per 1k. + max_input = max(input_price for input_price, _ in priced) / 1000 + max_output = max(output_price for _, output_price in priced) / 1000 + return ModelCost( + input_cost_per_1k_tokens=max_input, + output_cost_per_1k_tokens=max_output, + is_free=False, + ) + + +def free_models_from_catalog(catalog: Dict[str, Dict[str, Any]]) -> Dict[str, ModelCost]: + """The subset of ``catalog`` that currently has a free route.""" + priced = {mid: model_cost_from_catalog(e) for mid, e in catalog.items()} + return {mid: cost for mid, cost in priced.items() if cost.is_free} + + +def catalog_price_is_known(entry: Dict[str, Any]) -> bool: + """Whether the catalog says anything definite about this model's price. + + A model with a free route is known-free; a model with explicit pricing on + a live route is known-paid. Anything else is *unknown* -- and unknown must + behave as paid without pretending a $0.00 estimate is real. + """ + live = _live_providers(entry) + return any( + _is_free_route(p) or _explicit_pricing(p) is not None for p in live + ) + + +def _probed_throughput(entry: Dict[str, Any]) -> Optional[float]: + """The fastest probed throughput among live routes, when reported.""" + throughputs = [ + float(p["throughput"]) + for p in _live_providers(entry) + if isinstance(p.get("throughput"), (int, float)) + ] + return max(throughputs) if throughputs else None + + +def fetch_catalog_sync( + base_url: str, api_key: str, timeout: float +) -> Dict[str, Dict[str, Any]]: + """Fetch the model catalog synchronously, keyed by model id. + + The async path is preferred everywhere else. This exists for + :func:`orchestrator.populate_model_registry`, which is synchronous and can + be reached from inside a running event loop -- where ``asyncio.run`` would + raise. Uses ``urllib`` rather than aiohttp for exactly that reason. + """ + import json + import urllib.request + + url = f"{validate_base_url(base_url)}/models" + request = urllib.request.Request( + url, headers={"Authorization": f"Bearer {api_key}"} + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + payload = json.loads(response.read().decode("utf-8")) + return {e["id"]: e for e in (payload.get("data") or []) if e.get("id")} + + +class HuggingFaceProvider(ModelProvider): + """Serves chat models from the HuggingFace Inference Providers router.""" + + def __init__(self, config: Optional[ProviderConfig] = None) -> None: + # ProviderConfig defaults to a 30s timeout, which is right for a + # metadata call and far too short for generation behind a cold-starting + # model. The default config therefore carries the generation timeout; + # an explicit config is honoured exactly as given. + super().__init__( + config + or ProviderConfig( + name="huggingface", timeout=DEFAULT_REQUEST_TIMEOUT_SECONDS + ) + ) + # The catalog request carries the bearer token too, so the endpoint is + # checked here as well -- not only in HuggingFaceInferenceModel. + self._base_url = validate_base_url(self.config.base_url or DEFAULT_BASE_URL) + self._catalog: Dict[str, Dict[str, Any]] = {} + self._costs: Dict[str, ModelCost] = {} + + async def initialize(self) -> None: + """Resolve credentials and load the model catalog.""" + if not self.config.api_key: + self.config.api_key = resolve_huggingface_api_key(required=True).key + await self._load_catalog() + self._initialized = True + + async def _load_catalog(self) -> None: + """Fetch and index the live model catalog.""" + import aiohttp + + url = f"{self._base_url}/models" + timeout = aiohttp.ClientTimeout(total=self.config.timeout) + try: + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.get( + url, headers={"Authorization": f"Bearer {self.config.api_key}"} + ) as response: + if response.status >= 400: + raise HuggingFaceModelError( + f"HuggingFace model catalog returned HTTP " + f"{response.status}: {(await response.text())[:300]}" + ) + payload = await response.json() + except aiohttp.ClientError as exc: + raise HuggingFaceModelError( + f"Could not reach the HuggingFace model catalog: {exc}" + ) from exc + + entries = payload.get("data") or [] + self._catalog = {e["id"]: e for e in entries if e.get("id")} + self._costs = { + model_id: model_cost_from_catalog(entry) + for model_id, entry in self._catalog.items() + } + free = sorted(m for m, c in self._costs.items() if c.is_free) + logger.info( + "HuggingFace catalog: %d models, %d with a free route (%s)", + len(self._catalog), + len(free), + ", ".join(free) or "none", + ) + + def _require_catalog(self) -> None: + if not self._catalog: + raise HuggingFaceModelError( + "HuggingFace catalog not loaded; call await provider.initialize()" + ) + + def list_free_models(self) -> List[str]: + """Model ids that currently have a zero-cost route.""" + self._require_catalog() + return sorted(m for m, cost in self._costs.items() if cost.is_free) + + def list_paid_models(self) -> List[str]: + """Model ids that cost money, or whose price is unknown.""" + self._require_catalog() + return sorted(m for m, cost in self._costs.items() if not cost.is_free) + + async def create_model( + self, model_name: str, **kwargs: Any + ) -> HuggingFaceInferenceModel: + """Build a model, carrying its real catalog pricing and free-route pin. + + Pricing is attached here so :class:`HuggingFaceInferenceModel` can + refuse a paid model without a second network round trip. + """ + self._require_catalog() + if model_name not in self._catalog: + available = ", ".join(sorted(self._catalog)[:10]) + raise HuggingFaceModelError( + f"{model_name!r} is not served by the HuggingFace router. " + f"Available include: {available}..." + ) + entry = self._catalog[model_name] + # An unpriced entry gets cost=None, not a zero-filled ModelCost: the + # model then treats its price as *unknown* -- refused without the + # opt-in, and estimate_cost raises rather than reporting a + # confidently wrong $0.00 for a model that may bill. + cost = self._costs[model_name] if catalog_price_is_known(entry) else None + return HuggingFaceInferenceModel( + name=model_name, + api_key=self.config.api_key, + base_url=self._base_url, + route=free_route_from_catalog(entry), + cost=cost, + capabilities=self.get_model_capabilities(model_name), + requirements=self.get_model_requirements(model_name), + # The provider's configured transport policy applies to every + # model it builds; previously these were hard-coded and the + # config was silently ignored. + timeout=self.config.timeout, + max_retries=self.config.max_retries, + retry_delay=self.config.retry_delay, + **kwargs, + ) + + def free_models_by_preference(self) -> List[str]: + """Free models ordered by probed throughput, unprobed ones last. + + Unlike Dartmouth there is no stable, known free set to hard-code a + preference list against -- promos start and end upstream. The + catalog's own probe data is the honest ordering signal. + """ + free = self.list_free_models() + + def rank(model_id: str) -> Tuple[int, float, str]: + throughput = _probed_throughput(self._catalog.get(model_id, {})) + if throughput is None: + return (1, 0.0, model_id) + return (0, -throughput, model_id) + + return sorted(free, key=rank) + + async def generate_free( + self, + prompt: str, + *, + models: Optional[List[str]] = None, + **kwargs: Any, + ) -> Tuple[str, str]: + """Generate using the first free-routed model whose backend answers. + + Free routes are promos on individual providers and they flap. A + single-model call strands the caller whenever that happens, so this + walks the preference order and only gives up when every candidate is + unavailable. A rate limit is account-level, so it is NOT walked past: + every other attempt would draw on the same quota. + + Returns: + ``(text, model_id)`` -- the reply and which model produced it, so + callers can record what actually answered. + + Raises: + HuggingFaceModelError: if no free model could serve the request. + RateLimited: if the account is throttled (propagates rather than + burning through the candidate list). + """ + candidates = models or self.free_models_by_preference() + if not candidates: + raise HuggingFaceModelError( + "no HuggingFace models with a free route are available" + ) + + skipped: List[str] = [] + for model_id in candidates: + model = await self.create_model(model_id) + try: + return await model.generate(prompt, **kwargs), model_id + except ModelUnavailable as exc: + # Includes ModelLoading: a cold-starting model is handed to + # the next candidate rather than waited on. + logger.warning( + "Free model %s is not serving, trying next: %s", model_id, exc + ) + skipped.append(f"{model_id} (unavailable)") + continue + except ReasoningTruncated as exc: + # Several candidates are reasoning models that spend the whole + # budget thinking. Rather than fail, hand the request to the + # next candidate -- a non-reasoning model answers the same + # prompt comfortably within the same budget. + logger.warning( + "Free model %s exhausted its budget reasoning, trying " + "next: %s", + model_id, + exc, + ) + skipped.append(f"{model_id} (reasoning truncated)") + continue + finally: + # Each candidate holds its own HTTP session. Walking a chain + # of downed models would otherwise leak one session per + # attempt. The success path closes too: the reply is already + # in hand by the time this runs. + await model.aclose() + + raise HuggingFaceModelError( + f"no free HuggingFace model could answer: {', '.join(skipped)}. " + f"If every entry says 'reasoning truncated', raise max_tokens " + f"(current default {DEFAULT_MAX_TOKENS})." + ) + + async def health_check(self) -> bool: + """Whether the catalog is reachable with the configured credential.""" + try: + await self._load_catalog() + return bool(self._catalog) + except Exception as exc: # noqa: BLE001 - health checks report + logger.warning("HuggingFace health check failed: %s", exc) + return False + + async def discover_models(self) -> List[str]: + """Every chat model id the router serves.""" + if not self._catalog: + await self._load_catalog() + return sorted(self._catalog) + + def get_model_capabilities(self, model_name: str) -> ModelCapabilities: + """Capabilities inferred from the catalog entry.""" + entry = self._catalog.get(model_name, {}) + tasks = ["generate", "analyze", "transform", "summarize"] + architecture = entry.get("architecture") or {} + modalities = architecture.get("input_modalities") or [] + vision = "image" in modalities + if vision: + tasks.append("vision") + live = _live_providers(entry) + context_lengths = [ + int(p["context_length"]) + for p in live + if isinstance(p.get("context_length"), (int, float)) + and p["context_length"] > 0 + ] + return ModelCapabilities( + supported_tasks=tasks, + context_window=max(context_lengths) if context_lengths else 32768, + supports_structured_output=any( + p.get("supports_structured_output") for p in live + ), + supports_tools=any(p.get("supports_tools") for p in live), + vision_capable=vision, + ) + + def get_model_requirements(self, model_name: str) -> ModelRequirements: + """Local requirements for a remotely hosted model. + + Inference runs on the provider's infrastructure, so the local cost is + just an HTTP request. ``ModelRequirements`` forbids zero, so the + minimum defaults stand in for "negligible" -- overstating slightly is + safer than a field that cannot be constructed. + """ + return ModelRequirements(requires_gpu=False) + + def get_model_cost(self, model_name: str) -> ModelCost: + """Live pricing for ``model_name``.""" + self._require_catalog() + if model_name not in self._costs: + raise HuggingFaceModelError(f"{model_name!r} is not in the catalog") + return self._costs[model_name] diff --git a/src/orchestrator/models/registry.py b/src/orchestrator/models/registry.py deleted file mode 100644 index a85d6f36..00000000 --- a/src/orchestrator/models/registry.py +++ /dev/null @@ -1,363 +0,0 @@ -"""Unified model registry with provider abstractions.""" - -from __future__ import annotations - -import asyncio -import logging -from typing import Any, Dict, List, Optional, Set - -from ..core.model import Model -from .providers.base import ModelProvider, ProviderConfig, ProviderError -from .providers.anthropic_provider import AnthropicProvider - -logger = logging.getLogger(__name__) - - -class ModelRegistry: - """ - Unified model registry that manages multiple providers. - - This registry provides a unified interface for discovering, configuring, - and accessing models from different providers (OpenAI, Anthropic, local, etc.). - """ - - def __init__(self) -> None: - """Initialize model registry.""" - self._providers: Dict[str, ModelProvider] = {} - self._model_cache: Dict[str, Model] = {} - self._initialized = False - - @property - def is_initialized(self) -> bool: - """Check if registry is initialized.""" - return self._initialized - - @property - def providers(self) -> Dict[str, ModelProvider]: - """Get all registered providers.""" - return self._providers.copy() - - @property - def available_models(self) -> Dict[str, str]: - """ - Get all available models across providers. - - Returns: - Dictionary mapping model names to provider names - """ - models = {} - for provider_name, provider in self._providers.items(): - for model_name in provider.available_models: - models[model_name] = provider_name - return models - - def register_provider(self, provider: ModelProvider) -> None: - """ - Register a model provider. - - Args: - provider: The provider to register - """ - self._providers[provider.name] = provider - logger.info(f"Registered provider: {provider.name}") - - def configure_provider( - self, - provider_name: str, - provider_type: str, - config: Dict[str, Any] - ) -> None: - """ - Configure and register a provider. - - Args: - provider_name: Name for the provider instance - provider_type: Type of provider (currently only 'anthropic' supported) - config: Provider configuration - """ - provider_config = ProviderConfig(name=provider_name, **config) - - if provider_type.lower() == "anthropic": - provider = AnthropicProvider(provider_config) - else: - raise ValueError( - f"Unknown provider type: {provider_type}. " - f"Only 'anthropic' is supported in the Claude Skills refactor." - ) - - self.register_provider(provider) - - async def initialize(self) -> None: - """Initialize all registered providers.""" - if self._initialized: - return - - initialization_tasks = [] - for provider_name, provider in self._providers.items(): - if not provider.is_initialized: - initialization_tasks.append(self._initialize_provider(provider_name, provider)) - - if initialization_tasks: - results = await asyncio.gather(*initialization_tasks, return_exceptions=True) - - # Log any initialization failures - for i, result in enumerate(results): - if isinstance(result, Exception): - provider_name = list(self._providers.keys())[i] - logger.error(f"Failed to initialize provider {provider_name}: {result}") - - self._initialized = True - logger.info(f"Registry initialized with {len(self._providers)} providers") - - async def _initialize_provider(self, provider_name: str, provider: ModelProvider) -> None: - """Initialize a single provider with error handling.""" - try: - await provider.initialize() - logger.info(f"Provider {provider_name} initialized successfully") - except Exception as e: - logger.error(f"Failed to initialize provider {provider_name}: {e}") - raise - - async def discover_all_models(self) -> Dict[str, List[str]]: - """ - Discover models from all providers. - - Returns: - Dictionary mapping provider names to lists of model names - """ - if not self._initialized: - await self.initialize() - - discovery_tasks = [] - provider_names = [] - - for provider_name, provider in self._providers.items(): - if provider.is_initialized: - discovery_tasks.append(provider.discover_models()) - provider_names.append(provider_name) - - if not discovery_tasks: - return {} - - results = await asyncio.gather(*discovery_tasks, return_exceptions=True) - - discovered_models = {} - for i, result in enumerate(results): - provider_name = provider_names[i] - if isinstance(result, Exception): - logger.error(f"Failed to discover models from {provider_name}: {result}") - discovered_models[provider_name] = [] - else: - discovered_models[provider_name] = result - - return discovered_models - - def find_model(self, model_name: str) -> Optional[str]: - """ - Find which provider supports a model. - - Args: - model_name: Name of the model to find - - Returns: - Provider name if found, None otherwise - """ - for provider_name, provider in self._providers.items(): - if provider.supports_model(model_name): - return provider_name - return None - - async def get_model(self, model_name: str, provider_name: Optional[str] = None, **kwargs: Any) -> Model: - """ - Get a model instance. - - Args: - model_name: Name of the model - provider_name: Specific provider to use (auto-detect if None) - **kwargs: Additional model parameters - - Returns: - Model instance - - Raises: - ValueError: If model not found or provider not available - """ - if not self._initialized: - await self.initialize() - - # Auto-detect provider if not specified - if provider_name is None: - provider_name = self.find_model(model_name) - if provider_name is None: - raise ValueError(f"Model '{model_name}' not found in any provider") - - # Check if provider exists - if provider_name not in self._providers: - raise ValueError(f"Provider '{provider_name}' not registered") - - provider = self._providers[provider_name] - - # Check if provider supports the model - if not provider.supports_model(model_name): - raise ValueError(f"Provider '{provider_name}' does not support model '{model_name}'") - - # Create cache key - cache_key = f"{provider_name}:{model_name}:{hash(frozenset(kwargs.items()))}" - - # Return cached model if available - if cache_key in self._model_cache: - return self._model_cache[cache_key] - - # Create new model instance - try: - model = await provider.get_model(model_name, **kwargs) - self._model_cache[cache_key] = model - return model - except Exception as e: - raise ValueError(f"Failed to create model '{model_name}' from provider '{provider_name}': {e}") - - async def health_check(self) -> Dict[str, bool]: - """ - Check health of all providers. - - Returns: - Dictionary mapping provider names to health status - """ - health_tasks = [] - provider_names = [] - - for provider_name, provider in self._providers.items(): - if provider.is_initialized: - health_tasks.append(provider.health_check()) - provider_names.append(provider_name) - - if not health_tasks: - return {} - - results = await asyncio.gather(*health_tasks, return_exceptions=True) - - health_status = {} - for i, result in enumerate(results): - provider_name = provider_names[i] - if isinstance(result, Exception): - logger.error(f"Health check failed for {provider_name}: {result}") - health_status[provider_name] = False - else: - health_status[provider_name] = result - - return health_status - - def get_registry_info(self) -> Dict[str, Any]: - """ - Get registry information summary. - - Returns: - Dictionary with registry information - """ - provider_info = {} - for provider_name, provider in self._providers.items(): - provider_info[provider_name] = provider.get_provider_info() - - return { - "initialized": self._initialized, - "provider_count": len(self._providers), - "total_models": len(self.available_models), - "cached_models": len(self._model_cache), - "providers": provider_info, - } - - async def select_model(self, requirements: Optional[Dict[str, Any]] = None) -> str: - """ - Select best model for given requirements. - - Args: - requirements: Model selection requirements (optional) - - Returns: - Model name from available models - - Raises: - ValueError: If no models available - """ - available_models = self.available_models - if not available_models: - raise ValueError("No models available for selection") - - # Simple selection - return first available model - # In the future this could use sophisticated selection logic - return list(available_models.keys())[0] - - def list_models(self, provider_name: Optional[str] = None) -> Dict[str, Any]: - """ - List available models with details. - - Args: - provider_name: Filter by provider (all providers if None) - - Returns: - Dictionary with model information - """ - models = {} - - providers_to_check = ( - {provider_name: self._providers[provider_name]} if provider_name - else self._providers - ) - - for prov_name, provider in providers_to_check.items(): - if not provider.is_initialized: - continue - - for model_name in provider.available_models: - try: - capabilities = provider.get_model_capabilities(model_name) - requirements = provider.get_model_requirements(model_name) - cost = provider.get_model_cost(model_name) - - models[model_name] = { - "provider": prov_name, - "capabilities": capabilities.to_dict(), - "requirements": requirements.to_dict(), - "cost": cost.to_dict() if hasattr(cost, 'to_dict') else { - "input_cost_per_1k_tokens": cost.input_cost_per_1k_tokens, - "output_cost_per_1k_tokens": cost.output_cost_per_1k_tokens, - "is_free": cost.is_free, - }, - } - except Exception as e: - logger.warning(f"Failed to get info for model {model_name}: {e}") - models[model_name] = { - "provider": prov_name, - "error": str(e), - } - - return models - - async def cleanup(self) -> None: - """Clean up registry and all providers.""" - cleanup_tasks = [] - for provider in self._providers.values(): - if provider.is_initialized: - cleanup_tasks.append(provider.cleanup()) - - if cleanup_tasks: - await asyncio.gather(*cleanup_tasks, return_exceptions=True) - - self._model_cache.clear() - self._providers.clear() - self._initialized = False - logger.info("Registry cleaned up") - - def __str__(self) -> str: - """String representation of registry.""" - return f"ModelRegistry(providers={len(self._providers)}, models={len(self.available_models)})" - - def __repr__(self) -> str: - """Detailed representation of registry.""" - return ( - f"ModelRegistry(" - f"providers={list(self._providers.keys())}, " - f"models={len(self.available_models)}, " - f"initialized={self._initialized}" - f")" - ) \ No newline at end of file diff --git a/src/orchestrator/tools/update_models.py b/src/orchestrator/tools/update_models.py deleted file mode 100644 index 0da5e0b9..00000000 --- a/src/orchestrator/tools/update_models.py +++ /dev/null @@ -1,549 +0,0 @@ -#!/usr/bin/env python3 -""" -Update models.yaml with the latest models from all providers. - -This tool fetches the current list of models from: -- OpenAI API -- Anthropic API -- Google Gemini documentation -- Ollama model library -- HuggingFace trending models -""" - -import os -import yaml -import asyncio -import aiohttp -from pathlib import Path -from typing import Dict, List, Any, Optional -import re -from datetime import datetime -import logging - -# Set up logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -class ModelUpdater: - """Updates the models.yaml file with latest models from all providers.""" - - def __init__(self, config_path: Optional[Path] = None): - """Initialize the model updater. - - Args: - config_path: Path to save models.yaml. Defaults to ~/.orchestrator/models.yaml - """ - if config_path is None: - config_path = Path.home() / ".orchestrator" / "models.yaml" - self.config_path = config_path - self.config_path.parent.mkdir(parents=True, exist_ok=True) - - # Load existing config if available - self.existing_config = {} - if self.config_path.exists(): - with open(self.config_path, "r") as f: - self.existing_config = yaml.safe_load(f) or {} - - async def fetch_openai_models(self) -> List[Dict[str, Any]]: - """Fetch available models from OpenAI API.""" - api_key = os.getenv("OPENAI_API_KEY") - if not api_key: - logger.warning("OPENAI_API_KEY not set, skipping OpenAI models") - return [] - - models = [] - try: - async with aiohttp.ClientSession() as session: - headers = {"Authorization": f"Bearer {api_key}"} - async with session.get( - "https://api.openai.com/v1/models", headers=headers - ) as resp: - if resp.status == 200: - data = await resp.json() - for model in data.get("data", []): - model_id = model["id"] - # Filter for completion/chat models - if any( - x in model_id - for x in [ - "gpt", - "text", - "davinci", - "curie", - "babbage", - "ada", - "o1", - "o3", - "o4", - ] - ): - models.append( - { - "id": model_id, - "provider": "openai", - "type": "openai", - "created": model.get("created"), - } - ) - except Exception as e: - logger.error(f"Error fetching OpenAI models: {e}") - - return models - - async def fetch_anthropic_models(self) -> List[Dict[str, Any]]: - """Fetch available models from Anthropic's own listing endpoint. - - This was a hardcoded list, on the stated belief that "Anthropic - doesn't have a models.list() endpoint". It does, and the list rotted - accordingly: it still advertised claude-2, claude-2.1 and - claude-instant-1.2, all long retired. The same mistake has now been - made three times in this repository, so ids are read from the API and - never guessed. Returns an empty list when the key or SDK is absent, - matching how every other provider here degrades. - """ - api_key = os.getenv("ANTHROPIC_API_KEY") - if not api_key: - logger.info("No ANTHROPIC_API_KEY set; skipping Anthropic models") - return [] - - try: - import anthropic - except ImportError: - logger.info( - "The anthropic package is not installed " - '(pip install "py-orc[anthropic]"); skipping Anthropic models' - ) - return [] - - try: - client = anthropic.AsyncAnthropic(api_key=api_key) - listing = await client.models.list() - return [ - {"id": model.id, "provider": "anthropic", "type": "anthropic"} - for model in listing.data - ] - except Exception as e: - logger.error(f"Error fetching Anthropic models: {e}") - return [] - - async def fetch_dartmouth_models(self) -> List[Dict[str, Any]]: - """Fetch models from the Dartmouth Chat gateway's live catalog. - - Only the **free** models are registered. The catalog also serves paid - ones, and writing those into models.yaml would make them selectable by - default, which is precisely the accident - ``ORCHESTRATOR_ALLOW_PAID_MODELS`` exists to prevent. - """ - from ..models.dartmouth_credentials import resolve_dartmouth_api_key - from ..models.providers.dartmouth_provider import DartmouthProvider - - credential = resolve_dartmouth_api_key(required=False) - if credential is None: - logger.info( - "No Dartmouth Chat credential found; skipping Dartmouth models" - ) - return [] - - try: - provider = DartmouthProvider() - await provider.initialize() - return [ - {"id": model_id, "provider": "dartmouth", "type": "dartmouth"} - for model_id in provider.list_free_models() - ] - except Exception as e: - logger.error(f"Error fetching Dartmouth models: {e}") - return [] - - async def fetch_google_models(self) -> List[Dict[str, Any]]: - """Fetch available models from Google Gemini.""" - # Google Gemini models - parsed from documentation - models = [ - {"id": "gemini-2.5-pro", "provider": "google", "type": "google"}, - {"id": "gemini-2.5-flash", "provider": "google", "type": "google"}, - {"id": "gemini-2.5-flash-lite", "provider": "google", "type": "google"}, - {"id": "gemini-2.0-flash", "provider": "google", "type": "google"}, - {"id": "gemini-2.0-flash-lite", "provider": "google", "type": "google"}, - {"id": "gemini-1.5-pro", "provider": "google", "type": "google"}, - {"id": "gemini-1.5-flash", "provider": "google", "type": "google"}, - {"id": "gemini-1.5-flash-8b", "provider": "google", "type": "google"}, - {"id": "gemini-1.0-pro", "provider": "google", "type": "google"}, - {"id": "gemini-pro", "provider": "google", "type": "google"}, - {"id": "gemini-pro-vision", "provider": "google", "type": "google"}, - ] - - return models - - async def fetch_ollama_models(self) -> List[Dict[str, Any]]: - """Fetch available models from Ollama library.""" - models = [] - - # Popular Ollama models - ollama_models = [ - "llama3.2:1b", - "llama3.2:3b", - "llama3.1:8b", - "llama3.1:70b", - "deepseek-r1:1.5b", - "deepseek-r1:8b", - "deepseek-r1:32b", - "deepseek-r1:70b", - "qwen2.5-coder:1.5b", - "qwen2.5-coder:7b", - "qwen2.5-coder:14b", - "qwen2.5-coder:32b", - "gemma3:1b", - "gemma3:4b", - "gemma3:12b", - "gemma3:27b", - "gemma3n:e4b", - "gemma3n:e6b", - "gemma3n:e12b", - "mistral:7b", - "mistral-nemo:12b", - "mixtral:8x7b", - "mixtral:8x22b", - "phi3:3.8b", - "phi3:14b", - "phi3.5:3.8b", - "codellama:7b", - "codellama:13b", - "codellama:34b", - "starcoder2:3b", - "starcoder2:7b", - "starcoder2:15b", - "vicuna:7b", - "vicuna:13b", - "vicuna:33b", - "orca2:7b", - "orca2:13b", - "neural-chat:7b", - "neural-chat:7b-v3.3", - "starling-lm:7b", - "starling-lm:7b-alpha", - "zephyr:7b", - "zephyr:7b-alpha", - "zephyr:7b-beta", - "openchat:7b", - "openchat:7b-v3.5", - "yarn-mistral:7b", - "yarn-llama2:7b", - "yarn-llama2:13b", - "stable-beluga:7b", - "stable-beluga:13b", - "stable-beluga:70b", - ] - - for model_id in ollama_models: - models.append( - { - "id": model_id, - "provider": "ollama", - "type": "ollama", - } - ) - - return models - - async def fetch_huggingface_models(self) -> List[Dict[str, Any]]: - """Fetch trending instruct models from HuggingFace.""" - models = [] - - # Top trending instruct models under 40B parameters - hf_models = [ - "meta-llama/Llama-3.2-11B-Vision-Instruct", - "meta-llama/Llama-3.1-8B-Instruct", - "Qwen/Qwen2.5-1.5B-Instruct", - "Qwen/Qwen2.5-7B-Instruct", - "Qwen/Qwen2.5-14B-Instruct", - "Qwen/Qwen2.5-32B-Instruct", - "Qwen/Qwen2-VL-7B-Instruct", - "Qwen/Qwen2.5-Coder-7B-Instruct", - "Qwen/Qwen2.5-Coder-32B-Instruct", - "microsoft/Phi-3.5-mini-instruct", - "microsoft/Phi-3.5-MoE-instruct", - "HuggingFaceTB/SmolLM-1.7B-Instruct", - "stabilityai/stable-code-instruct-3b", - "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", - "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", - "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", - "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", - "codellama/CodeLlama-7b-Instruct-hf", - "codellama/CodeLlama-13b-Instruct-hf", - "codellama/CodeLlama-34b-Instruct-hf", - "bigcode/starcoder2-3b", - "bigcode/starcoder2-7b", - "bigcode/starcoder2-15b", - "WizardLM/WizardCoder-Python-7B-V1.0", - "WizardLM/WizardCoder-Python-13B-V1.0", - "WizardLM/WizardCoder-Python-34B-V1.0", - "tencent/Hunyuan-A13B-Instruct", - "NousResearch/Hermes-3-Llama-3.1-8B", - "allenai/OLMo-2-1124-7B-Instruct", - "google/gemma-2-9b-it", - ] - - for model_id in hf_models: - models.append( - { - "id": model_id, - "provider": "huggingface", - "type": "huggingface", - } - ) - - return models - - def estimate_model_size(self, model_id: str) -> float: - """Estimate model size in billions of parameters from model ID.""" - # Extract number patterns like 7b, 13B, 1.5b, etc. - patterns = [ - r"(\d+(?:\.\d+)?)[bB]", # Matches 7b, 13B, 1.5b - r"(\d+)x(\d+)[bB]", # Matches 8x7b (multiply) - ] - - for pattern in patterns: - match = re.search(pattern, model_id) - if match: - if len(match.groups()) == 2: # Multiplication pattern - return float(match.group(1)) * float(match.group(2)) - else: - return float(match.group(1)) - - # Default sizes for known models - if "gpt-4" in model_id: - return 1760.0 # GPT-4 estimated - elif "gpt-3.5" in model_id: - return 175.0 - elif "claude-opus" in model_id or "opus" in model_id: - return 2000.0 - elif "claude-sonnet" in model_id or "sonnet" in model_id: - return 200.0 - elif "claude-haiku" in model_id or "haiku" in model_id: - return 20.0 - elif "gemini-2.5-pro" in model_id: - return 1500.0 - elif "gemini" in model_id and "pro" in model_id: - return 540.0 - elif "gemini" in model_id and "flash" in model_id: - return 80.0 - elif "o3" in model_id and "mini" not in model_id: - return 2000.0 - elif "o3-mini" in model_id or "o4-mini" in model_id: - return 70.0 - elif "o1" in model_id and "mini" not in model_id: - return 175.0 - elif "o1-mini" in model_id: - return 65.0 - - return 7.0 # Default fallback - - def create_model_entry(self, model: Dict[str, Any]) -> Dict[str, Any]: - """Create a model entry for models.yaml.""" - model_id = model["id"] - provider = model["provider"] - - # Estimate model size - size_b = self.estimate_model_size(model_id) - - # Base configuration - entry = { - "provider": provider, - "type": model["type"], - "size_b": size_b, - "config": {}, - } - - # Provider-specific configuration - if provider == "openai": - entry["config"] = { - "model_name": model_id, - "api_key": "${OPENAI_API_KEY}", - "max_retries": 3, - "timeout": 30.0, - } - elif provider == "anthropic": - entry["config"] = { - "model_name": model_id, - "api_key": "${ANTHROPIC_API_KEY}", - "max_retries": 3, - "timeout": 30.0, - } - elif provider == "google": - entry["config"] = { - "model_name": model_id, - "api_key": "${GOOGLE_AI_API_KEY}", - "max_retries": 3, - "timeout": 30.0, - } - elif provider == "ollama": - entry["config"] = { - "model_name": model_id, - "base_url": "http://localhost:11434", - "timeout": 30.0, - } - elif provider == "huggingface": - entry["config"] = { - "model_name": model_id, - "token": "${HUGGINGFACE_TOKEN}", - "device": "auto", - "torch_dtype": "auto", - } - - return entry - - async def update_models(self) -> Dict[str, Any]: - """Fetch all models and update the configuration.""" - logger.info("Fetching models from all providers...") - - # Fetch models from all providers concurrently - tasks = [ - self.fetch_openai_models(), - self.fetch_anthropic_models(), - self.fetch_dartmouth_models(), - self.fetch_google_models(), - self.fetch_ollama_models(), - self.fetch_huggingface_models(), - ] - - results = await asyncio.gather(*tasks) - - # Combine all models - all_models = [] - for models in results: - all_models.extend(models) - - logger.info(f"Found {len(all_models)} models total") - - # Create the new configuration - config = { - "# Model configuration for the Orchestrator Framework": None, - "# Auto-generated on": datetime.now().isoformat(), - "": None, - "models": {}, - } - - # Add all models - for model in all_models: - model_id = model["id"] - config["models"][model_id] = self.create_model_entry(model) - - # Add preference sections - config["preferences"] = { - "default": "gpt-4o-mini", - "fallback": [ - "gpt-3.5-turbo", - "claude-sonnet-4-20250514", - "ollama:llama3.2:1b", - ], - } - - # Cost-optimized selection (smaller, cheaper models) - config["cost_optimized"] = [ - "gpt-4o-mini", - "claude-sonnet-4-20250514", - "gemini-2.0-flash-lite", - "ollama:llama3.2:1b", - "ollama:gemma3:1b", - "huggingface:Qwen/Qwen2.5-1.5B-Instruct", - ] - - # Performance-optimized selection (larger, more capable models) - config["performance_optimized"] = [ - "o3", - "gpt-4.1", - "claude-opus-4-20250514", - "gemini-2.5-pro", - "ollama:deepseek-r1:70b", - "huggingface:Qwen/Qwen2.5-32B-Instruct", - ] - - return config - - async def save_models(self, config: Dict[str, Any]) -> None: - """Save the models configuration to file.""" - # Create a clean YAML structure - clean_config = {} - - # Add header comments - yaml_content = "# Model configuration for the Orchestrator Framework\n" - yaml_content += f"# Auto-generated on: {datetime.now().isoformat()}\n\n" - - # Add models section - clean_config["models"] = config["models"] - clean_config["preferences"] = config["preferences"] - clean_config["cost_optimized"] = config["cost_optimized"] - clean_config["performance_optimized"] = config["performance_optimized"] - - # Write to file - yaml_content += yaml.dump( - clean_config, default_flow_style=False, sort_keys=False - ) - - with open(self.config_path, "w") as f: - f.write(yaml_content) - - logger.info(f"Saved {len(config['models'])} models to {self.config_path}") - - async def run(self) -> None: - """Run the model update process.""" - config = await self.update_models() - await self.save_models(config) - - -async def update_models(config_path: Optional[Path] = None) -> None: - """ - Update the models.yaml file with latest models from all providers. - - Args: - config_path: Path to save models.yaml. Defaults to ~/.orchestrator/models.yaml - """ - # Ensure config_path is a Path object - if config_path is not None and not isinstance(config_path, Path): - config_path = Path(config_path) - updater = ModelUpdater(config_path) - await updater.run() - - -def main(): - """Command-line entry point.""" - import argparse - - parser = argparse.ArgumentParser( - description="Update Orchestrator models configuration" - ) - parser.add_argument( - "--output", - "-o", - type=Path, - help="Output path for models.yaml (default: ~/.orchestrator/models.yaml)", - ) - parser.add_argument( - "--config", - "-c", - type=Path, - default=None, - help=( - "Also update an additional models.yaml at this path (e.g. a " - "repository's checked-in copy). Not set by default: the packaged " - "default config must not be rewritten in an installed environment." - ), - ) - - args = parser.parse_args() - - async def run(): - # Update user config - await update_models(args.output) - - # Also update repository config if specified - if args.config: - logger.info(f"Also updating repository config at {args.config}") - await update_models(args.config) - - asyncio.run(run()) - - -if __name__ == "__main__": - main() diff --git a/tests/conftest.py b/tests/conftest.py index 4288482d..2d20fe28 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,7 +55,7 @@ def pytest_collection_modifyitems(config, items): # Live tests are not all about the same provider, so they cannot share one # credential gate. Dartmouth Chat tests need a Dartmouth key -- and they # cost nothing to run, so gating them behind a paid provider's key would - # needlessly forgo free coverage. + # needlessly forgo free coverage. HuggingFace tests need an HF token. def _have_dartmouth() -> bool: try: from orchestrator.models.dartmouth_credentials import ( @@ -68,10 +68,25 @@ def _have_dartmouth() -> bool: have_dartmouth = _have_dartmouth() + def _have_huggingface() -> bool: + try: + from orchestrator.models.huggingface_credentials import ( + resolve_huggingface_api_key, + ) + + return resolve_huggingface_api_key(required=False) is not None + except Exception: + return False + + have_huggingface = _have_huggingface() + def _credential_for(item) -> tuple[bool, str]: """Which credential a live test needs, and whether we have it.""" - if "dartmouth" in str(getattr(item, "fspath", "")).lower(): + fspath = str(getattr(item, "fspath", "")).lower() + if "dartmouth" in fspath: return have_dartmouth, "DARTMOUTH_CHAT_API_KEY" + if "huggingface" in fspath: + return have_huggingface, "HF_TOKEN" return have_anthropic, "ANTHROPIC_API_KEY" # The live CI job sets ORCHESTRATOR_REQUIRE_LIVE=1. Without this guard a @@ -79,12 +94,13 @@ def _credential_for(item) -> tuple[bool, str]: # having exercised no provider at all -- indistinguishable from having no # live coverage, which is the state this suite is meant to leave behind. if os.environ.get("ORCHESTRATOR_REQUIRE_LIVE") == "1" and not ( - have_anthropic or have_dartmouth + have_anthropic or have_dartmouth or have_huggingface ): raise pytest.UsageError( "ORCHESTRATOR_REQUIRE_LIVE=1 requires real live coverage, but " - "neither ANTHROPIC_API_KEY nor a Dartmouth Chat credential is " - "available. Provide one, or unset ORCHESTRATOR_REQUIRE_LIVE." + "neither ANTHROPIC_API_KEY, a Dartmouth Chat credential, nor an " + "HF_TOKEN is available. Provide one, or unset " + "ORCHESTRATOR_REQUIRE_LIVE." ) run_integration = os.environ.get("ORCHESTRATOR_RUN_INTEGRATION") == "1" diff --git a/tests/execution/test_model_selection.py b/tests/execution/test_model_selection.py deleted file mode 100644 index 2f1431e1..00000000 --- a/tests/execution/test_model_selection.py +++ /dev/null @@ -1,690 +0,0 @@ -""" -Comprehensive tests for runtime model selection integration. - -This module tests the integration of intelligent model selection capabilities -with the pipeline execution engine, ensuring that models are selected optimally -at runtime based on step requirements, execution context, and selection strategies. -""" - -import asyncio -import pytest -from unittest.mock import Mock, AsyncMock, patch, MagicMock -from typing import Dict, Any, List, Optional - -# Import the modules we're testing -from orchestrator.execution.model_selector import ( - ExecutionModelSelector, - RuntimeModelContext -) -from orchestrator.execution.engine import StateGraphEngine -from orchestrator.api.execution import ( - PipelineExecutor, - create_intelligent_pipeline_executor -) - -# Import required foundation components -from orchestrator.foundation._compatibility import ( - FoundationConfig, - PipelineSpecification, - PipelineStep, - PipelineResult, - StepResult -) -from orchestrator.models.registry import ModelRegistry -from orchestrator.models.model_selector import ModelSelectionCriteria -from orchestrator.core.model import Model, ModelCapabilities, ModelCost, ModelMetrics - - -class TestExecutionModelSelector: - """Test the ExecutionModelSelector class for runtime model selection.""" - - @pytest.fixture - def mock_model_registry(self): - """Create a mock model registry with test models.""" - registry = Mock(spec=ModelRegistry) - - # Create test models - fast_model = Mock(spec=Model) - fast_model.provider = "test" - fast_model.name = "fast-model" - fast_model.capabilities = Mock(spec=ModelCapabilities) - fast_model.capabilities.speed_rating = "fast" - fast_model.capabilities.accuracy_score = 0.8 - fast_model.capabilities.vision_capable = False - fast_model.capabilities.code_specialized = True - fast_model.capabilities.supports_function_calling = True - fast_model.cost = Mock(spec=ModelCost) - fast_model.cost.is_free = False - fast_model.cost.input_cost_per_1k_tokens = 0.001 - fast_model.cost.output_cost_per_1k_tokens = 0.002 - fast_model.metrics = Mock(spec=ModelMetrics) - fast_model.metrics.success_rate = 0.9 - - expensive_model = Mock(spec=Model) - expensive_model.provider = "test" - expensive_model.name = "expensive-model" - expensive_model.capabilities = Mock(spec=ModelCapabilities) - expensive_model.capabilities.speed_rating = "medium" - expensive_model.capabilities.accuracy_score = 0.95 - expensive_model.capabilities.vision_capable = True - expensive_model.capabilities.code_specialized = True - expensive_model.capabilities.supports_function_calling = True - expensive_model.cost = Mock(spec=ModelCost) - expensive_model.cost.is_free = False - expensive_model.cost.input_cost_per_1k_tokens = 0.01 - expensive_model.cost.output_cost_per_1k_tokens = 0.02 - expensive_model.metrics = Mock(spec=ModelMetrics) - expensive_model.metrics.success_rate = 0.95 - - free_model = Mock(spec=Model) - free_model.provider = "test" - free_model.name = "free-model" - free_model.capabilities = Mock(spec=ModelCapabilities) - free_model.capabilities.speed_rating = "slow" - free_model.capabilities.accuracy_score = 0.7 - free_model.capabilities.vision_capable = False - free_model.capabilities.code_specialized = False - free_model.capabilities.supports_function_calling = False - free_model.cost = Mock(spec=ModelCost) - free_model.cost.is_free = True - free_model.cost.input_cost_per_1k_tokens = 0.0 - free_model.cost.output_cost_per_1k_tokens = 0.0 - free_model.metrics = Mock(spec=ModelMetrics) - free_model.metrics.success_rate = 0.8 - - registry.models = { - "test:fast-model": fast_model, - "test:expensive-model": expensive_model, - "test:free-model": free_model - } - - # Mock registry methods - async def mock_filter_by_capabilities(requirements: Dict[str, Any]) -> List[Model]: - models = [fast_model, expensive_model, free_model] - filtered = [] - - for model in models: - # Simple filtering logic for tests - if requirements.get("code_specialized") and not model.capabilities.code_specialized: - continue - if requirements.get("vision_capable") and not model.capabilities.vision_capable: - continue - filtered.append(model) - - return filtered - - async def mock_get_model(provider: str, name: str) -> Optional[Model]: - key = f"{provider}:{name}" - return registry.models.get(key) - - async def mock_find_model_by_name(name: str) -> Optional[Model]: - for model in registry.models.values(): - if model.name == name: - return model - return None - - registry._filter_by_capabilities = mock_filter_by_capabilities - registry.get_model = mock_get_model - registry.find_model_by_name = mock_find_model_by_name - - return registry - - @pytest.fixture - def model_selector(self, mock_model_registry): - """Create an ExecutionModelSelector with mock registry.""" - return ExecutionModelSelector( - model_registry=mock_model_registry, - enable_adaptive_selection=True, - enable_expert_assignments=True, - enable_cost_optimization=True - ) - - @pytest.fixture - def sample_step(self): - """Create a sample pipeline step for testing.""" - step = Mock(spec=PipelineStep) - step.id = "test_step" - step.name = "Test Step" - step.model = "AUTO:code generation task" - step.tools = ["code_editor", "compiler"] - step.variables = {"output": "generated code"} - step.description = "Generate Python code for data processing" - step.prompt = "Create a function that processes data" - step.context_limit = 4000 - return step - - @pytest.fixture - def sample_pipeline_spec(self, sample_step): - """Create a sample pipeline specification.""" - spec = Mock(spec=PipelineSpecification) - spec.header = Mock() - spec.header.id = "test_pipeline" - spec.header.name = "Test Pipeline" - spec.steps = [sample_step] - - # Mock selection schema - spec.selection_schema = Mock() - spec.selection_schema.strategy = "balanced" - spec.selection_schema.cost_limit = 0.05 - spec.selection_schema.max_latency_ms = 5000 - - # Mock experts field - spec.experts = { - "code_editor": "test:fast-model", - "compiler": "test:expensive-model" - } - - return spec - - @pytest.fixture - def runtime_context(self, sample_step, sample_pipeline_spec): - """Create a runtime model context for testing.""" - return RuntimeModelContext( - step=sample_step, - pipeline_spec=sample_pipeline_spec, - execution_state={"variables": {"input": "test data"}}, - available_variables={"user_preference": "fast"}, - expert_assignments={"code_editor": "test:fast-model"}, - cost_constraints={"max_cost_per_request": 0.01}, - performance_requirements={"max_latency_ms": 3000} - ) - - @pytest.mark.asyncio - async def test_select_model_for_step_basic(self, model_selector, runtime_context): - """Test basic model selection for a step.""" - selected_model = await model_selector.select_model_for_step(runtime_context) - - assert selected_model is not None - assert selected_model.provider == "test" - assert selected_model.name in ["fast-model", "expensive-model", "free-model"] - - @pytest.mark.asyncio - async def test_select_model_with_explicit_specification(self, model_selector, runtime_context): - """Test model selection when step has explicit model specification.""" - # Set explicit model - runtime_context.step.model = "test:expensive-model" - - selected_model = await model_selector.select_model_for_step(runtime_context) - - assert selected_model is not None - assert selected_model.provider == "test" - assert selected_model.name == "expensive-model" - - @pytest.mark.asyncio - async def test_select_model_with_expert_assignments(self, model_selector, runtime_context): - """Test model selection using expert tool-model assignments.""" - # Expert assignment should take precedence - selected_model = await model_selector.select_model_for_step(runtime_context) - - assert selected_model is not None - # Should select the expert-assigned model for code_editor tool - assert selected_model.name == "fast-model" - - @pytest.mark.asyncio - async def test_select_model_cost_optimized_strategy(self, model_selector, runtime_context): - """Test model selection with cost-optimized strategy.""" - selected_model = await model_selector.select_model_for_step( - runtime_context, selection_strategy="cost_optimized" - ) - - assert selected_model is not None - # Should prefer cheaper models - assert selected_model.cost.input_cost_per_1k_tokens <= 0.01 - - @pytest.mark.asyncio - async def test_select_model_performance_optimized_strategy(self, model_selector, runtime_context): - """Test model selection with performance-optimized strategy.""" - selected_model = await model_selector.select_model_for_step( - runtime_context, selection_strategy="performance_optimized" - ) - - assert selected_model is not None - # Should prefer faster models - assert selected_model.capabilities.speed_rating in ["fast", "medium"] - - @pytest.mark.asyncio - async def test_evaluate_selection_quality(self, model_selector, mock_model_registry): - """Test evaluation of model selection quality.""" - fast_model = mock_model_registry.models["test:fast-model"] - - execution_result = { - "timestamp": "2024-01-01T00:00:00Z", - "status": "success", - "execution_time": 2.5, - "errors": [], - "output": {"result": "Generated code successfully"} - } - - quality_metrics = await model_selector.evaluate_selection_quality( - "test_step", fast_model, execution_result - ) - - assert quality_metrics["step_id"] == "test_step" - assert quality_metrics["model"] == "test:fast-model" - assert quality_metrics["success"] == True - assert quality_metrics["execution_time"] == 2.5 - assert "performance_score" in quality_metrics - assert "cost_efficiency" in quality_metrics - - @pytest.mark.asyncio - async def test_get_selection_recommendations(self, model_selector, sample_pipeline_spec): - """Test getting model selection recommendations for a pipeline.""" - execution_context = {"user_type": "developer", "budget": 0.1} - - recommendations = await model_selector.get_selection_recommendations( - sample_pipeline_spec, execution_context - ) - - assert "test_step" in recommendations - step_recommendations = recommendations["test_step"] - assert "step_name" in step_recommendations - assert "recommendations" in step_recommendations - assert len(step_recommendations["recommendations"]) > 0 - - # Check recommendation structure - first_rec = step_recommendations["recommendations"][0] - assert "model" in first_rec - assert "score" in first_rec - assert "rationale" in first_rec - assert "estimated_cost" in first_rec - - -class TestStateGraphEngine: - """Test StateGraphEngine integration with model selection.""" - - @pytest.fixture - def mock_model_registry(self): - """Create a mock model registry.""" - registry = Mock(spec=ModelRegistry) - - # Create a test model - test_model = Mock(spec=Model) - test_model.provider = "test" - test_model.name = "integration-model" - test_model.capabilities = Mock(spec=ModelCapabilities) - test_model.capabilities.speed_rating = "medium" - test_model.capabilities.accuracy_score = 0.85 - test_model.capabilities.code_specialized = True - test_model.cost = Mock(spec=ModelCost) - test_model.cost.is_free = False - test_model.metrics = Mock(spec=ModelMetrics) - test_model.metrics.success_rate = 0.9 - - registry.models = {"test:integration-model": test_model} - - # Mock registry methods - async def mock_filter_by_capabilities(requirements: Dict[str, Any]) -> List[Model]: - return [test_model] - - registry._filter_by_capabilities = mock_filter_by_capabilities - - return registry - - @pytest.fixture - def engine(self, mock_model_registry): - """Create StateGraphEngine with model registry.""" - config = FoundationConfig(max_concurrent_steps=5) - return StateGraphEngine(config=config, model_registry=mock_model_registry) - - @pytest.fixture - def sample_pipeline_spec(self): - """Create a pipeline specification for testing.""" - step = Mock(spec=PipelineStep) - step.id = "step1" - step.name = "Test Step" - step.model = "AUTO" - step.tools = ["test_tool"] - step.variables = {"output": "test output"} - step.dependencies = [] - step.condition = None - step.retry_count = 0 - - spec = Mock(spec=PipelineSpecification) - spec.header = Mock() - spec.header.id = "test_pipeline" - spec.header.name = "Test Pipeline" - spec.steps = [step] - - # Mock spec methods - def mock_get_step(step_id: str): - if step_id == "step1": - return step - return None - - def mock_get_execution_order(): - return [["step1"]] - - def mock_get_dependents(step_id: str): - return [] - - spec.get_step = mock_get_step - spec.get_execution_order = mock_get_execution_order - spec.get_dependents = mock_get_dependents - - return spec - - @pytest.mark.asyncio - async def test_engine_initialization_with_model_registry(self, mock_model_registry): - """Test engine initialization with model registry.""" - config = FoundationConfig() - engine = StateGraphEngine(config=config, model_registry=mock_model_registry) - - assert engine._model_registry is mock_model_registry - assert engine._model_selector is not None - assert isinstance(engine._model_selector, ExecutionModelSelector) - - @pytest.mark.asyncio - async def test_engine_model_selection_recommendations(self, engine, sample_pipeline_spec): - """Test getting model selection recommendations from engine.""" - execution_context = {"test": "context"} - - recommendations = engine.get_model_selection_recommendations( - sample_pipeline_spec, execution_context - ) - - assert isinstance(recommendations, dict) - # Should contain recommendations or error message - assert "step1" in recommendations or "error" in recommendations - - def test_engine_initialization_without_model_registry(self): - """Test engine initialization without model registry.""" - config = FoundationConfig() - engine = StateGraphEngine(config=config, model_registry=None) - - assert engine._model_registry is None - assert engine._model_selector is None - - -class TestPipelineExecutor: - """Test PipelineExecutor integration with intelligent model selection.""" - - @pytest.fixture - def mock_model_registry(self): - """Create a mock model registry.""" - registry = Mock(spec=ModelRegistry) - - test_model = Mock(spec=Model) - test_model.provider = "test" - test_model.name = "executor-model" - test_model.capabilities = Mock(spec=ModelCapabilities) - test_model.cost = Mock(spec=ModelCost) - test_model.metrics = Mock(spec=ModelMetrics) - - registry.models = {"test:executor-model": test_model} - return registry - - @pytest.fixture - def pipeline_executor(self, mock_model_registry): - """Create PipelineExecutor with intelligent selection.""" - return create_intelligent_pipeline_executor( - model_registry=mock_model_registry, - max_concurrent_executions=5 - ) - - @pytest.fixture - def sample_pipeline(self): - """Create a sample pipeline for testing.""" - from orchestrator.core.pipeline import Pipeline - - # Mock pipeline - pipeline = Mock(spec=Pipeline) - pipeline.id = "test_pipeline" - pipeline.name = "Test Pipeline" - pipeline.context = {"test_context": "value"} - - # Mock pipeline specification - pipeline.specification = Mock(spec=PipelineSpecification) - pipeline.specification.header = Mock() - pipeline.specification.header.id = "test_pipeline" - pipeline.specification.header.name = "Test Pipeline" - pipeline.specification.steps = [] - - return pipeline - - def test_executor_initialization_with_intelligent_selection(self, mock_model_registry): - """Test executor initialization with intelligent model selection.""" - executor = create_intelligent_pipeline_executor(mock_model_registry) - - assert executor.enable_intelligent_selection == True - assert executor.model_registry is mock_model_registry - assert executor._execution_engine is not None - - def test_executor_initialization_without_model_registry(self): - """Test executor initialization without model registry.""" - executor = PipelineExecutor( - model_registry=None, - enable_intelligent_selection=False - ) - - assert executor.enable_intelligent_selection == False - assert executor.model_registry is None - assert executor._execution_engine is None - - def test_get_model_selection_recommendations(self, pipeline_executor, sample_pipeline): - """Test getting model selection recommendations from executor.""" - execution_context = {"user_type": "test"} - - recommendations = pipeline_executor.get_model_selection_recommendations( - sample_pipeline, execution_context - ) - - assert isinstance(recommendations, dict) - - @pytest.mark.asyncio - async def test_execute_with_intelligent_selection(self, pipeline_executor, sample_pipeline): - """Test executing pipeline with intelligent model selection.""" - with patch.object(pipeline_executor, 'execute_with_monitoring') as mock_execute: - mock_manager = Mock() - mock_manager.execution_id = "test_exec_123" - mock_execute.return_value = mock_manager - - result = await pipeline_executor.execute_with_intelligent_selection( - pipeline=sample_pipeline, - context={"test": "context"}, - selection_strategy="cost_optimized" - ) - - assert result is mock_manager - mock_execute.assert_called_once() - - # Check that enhanced context was passed - call_args = mock_execute.call_args - enhanced_context = call_args.kwargs["context"] - assert "selection_strategy" in enhanced_context - assert enhanced_context["selection_strategy"] == "cost_optimized" - - def test_analyze_execution_efficiency(self, pipeline_executor): - """Test execution efficiency analysis.""" - # Setup mock execution metadata - execution_id = "test_exec_123" - pipeline_executor._execution_metadata[execution_id] = { - "pipeline_id": "test_pipeline", - "intelligent_selection": True, - "selection_strategy": "balanced", - "total_tasks": 3 - } - - analysis = pipeline_executor.analyze_execution_efficiency(execution_id) - - assert analysis["execution_id"] == execution_id - assert analysis["intelligent_selection_used"] == True - assert analysis["selection_strategy"] == "balanced" - assert "cost_efficiency" in analysis - assert "performance_efficiency" in analysis - - -class TestModelSelectionIntegration: - """Integration tests for the complete model selection system.""" - - @pytest.mark.asyncio - async def test_end_to_end_model_selection_flow(self): - """Test complete end-to-end model selection flow.""" - # This test would require more complex setup with real components - # For now, we verify that all components can be instantiated together - - # Create mock registry - registry = Mock(spec=ModelRegistry) - test_model = Mock(spec=Model) - test_model.provider = "test" - test_model.name = "e2e-model" - test_model.capabilities = Mock(spec=ModelCapabilities) - test_model.cost = Mock(spec=ModelCost) - test_model.metrics = Mock(spec=ModelMetrics) - registry.models = {"test:e2e-model": test_model} - - # Create components - model_selector = ExecutionModelSelector(registry) - engine = StateGraphEngine(FoundationConfig(), registry) - executor = create_intelligent_pipeline_executor(registry) - - # Verify all components are properly initialized - assert model_selector.model_registry is registry - assert engine._model_selector is not None - assert executor.enable_intelligent_selection == True - assert executor._execution_engine is not None - - @pytest.mark.asyncio - async def test_adaptive_selection_learning(self, mock_model_registry): - """Test adaptive selection learning from execution history.""" - model_selector = ExecutionModelSelector( - mock_model_registry, - enable_adaptive_selection=True - ) - - # Create sample context - step = Mock(spec=PipelineStep) - step.id = "adaptive_test" - step.name = "Adaptive Test" - step.model = "AUTO" - step.tools = ["test_tool"] - step.variables = {} - - spec = Mock(spec=PipelineSpecification) - - context = RuntimeModelContext( - step=step, - pipeline_spec=spec, - execution_state={}, - available_variables={} - ) - - # First selection - selected_model_1 = await model_selector.select_model_for_step(context) - - # Simulate execution result - execution_result = { - "status": "success", - "execution_time": 2.0, - "timestamp": "2024-01-01T00:00:00Z", - "output": {"result": "success"}, - "errors": [] - } - - # Record quality metrics - await model_selector.evaluate_selection_quality( - step.id, selected_model_1, execution_result - ) - - # Verify that selection history is recorded - assert len(model_selector._execution_history) > 0 - - def test_cost_constraint_handling(self): - """Test handling of cost constraints in model selection.""" - registry = Mock(spec=ModelRegistry) - - # Create expensive and cheap models - expensive_model = Mock(spec=Model) - expensive_model.cost = Mock(spec=ModelCost) - expensive_model.cost.is_free = False - expensive_model.cost.input_cost_per_1k_tokens = 0.1 - expensive_model.cost.output_cost_per_1k_tokens = 0.2 - - cheap_model = Mock(spec=Model) - cheap_model.cost = Mock(spec=ModelCost) - cheap_model.cost.is_free = True - cheap_model.cost.input_cost_per_1k_tokens = 0.0 - cheap_model.cost.output_cost_per_1k_tokens = 0.0 - - registry.models = { - "test:expensive": expensive_model, - "test:cheap": cheap_model - } - - model_selector = ExecutionModelSelector( - registry, - enable_cost_optimization=True - ) - - # Test cost estimation - expensive_cost = model_selector._estimate_model_cost(expensive_model, "per-task") - cheap_cost = model_selector._estimate_model_cost(cheap_model, "per-task") - - assert expensive_cost > cheap_cost - assert cheap_cost == 0.0 - - def test_expert_assignment_priority(self): - """Test that expert assignments take priority in model selection.""" - registry = Mock(spec=ModelRegistry) - - default_model = Mock(spec=Model) - default_model.provider = "test" - default_model.name = "default-model" - - expert_model = Mock(spec=Model) - expert_model.provider = "test" - expert_model.name = "expert-model" - - async def mock_resolve_explicit_model(model_spec: str, context): - if model_spec == "test:expert-model": - return expert_model - return None - - model_selector = ExecutionModelSelector(registry, enable_expert_assignments=True) - - # Mock the _resolve_explicit_model method - model_selector._resolve_explicit_model = mock_resolve_explicit_model - - # The expert assignment logic should be tested through integration - # This test verifies the component structure is correct - assert model_selector.enable_expert_assignments == True - - def test_performance_requirements_filtering(self): - """Test filtering models based on performance requirements.""" - criteria = ModelSelectionCriteria() - criteria.max_latency_ms = 1000 - criteria.min_accuracy_score = 0.8 - criteria.speed_preference = "fast" - - # Create test models with different performance characteristics - fast_model = Mock(spec=Model) - fast_model.capabilities = Mock(spec=ModelCapabilities) - fast_model.capabilities.speed_rating = "fast" - fast_model.capabilities.accuracy_score = 0.85 - - slow_model = Mock(spec=Model) - slow_model.capabilities = Mock(spec=ModelCapabilities) - slow_model.capabilities.speed_rating = "slow" - slow_model.capabilities.accuracy_score = 0.9 - - inaccurate_model = Mock(spec=Model) - inaccurate_model.capabilities = Mock(spec=ModelCapabilities) - inaccurate_model.capabilities.speed_rating = "fast" - inaccurate_model.capabilities.accuracy_score = 0.6 - - models = [fast_model, slow_model, inaccurate_model] - - # Create model selector to test filtering logic - registry = Mock(spec=ModelRegistry) - model_selector = ExecutionModelSelector(registry) - - # Test the filtering method - filtered_models = model_selector._filter_by_criteria(models, criteria) - - # Should filter out slow model and inaccurate model - assert fast_model in filtered_models - assert slow_model not in filtered_models # Too slow - assert inaccurate_model not in filtered_models # Not accurate enough - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) \ No newline at end of file diff --git a/tests/integration/test_auto_resolution_quality.py b/tests/integration/test_auto_resolution_quality.py deleted file mode 100644 index d065c475..00000000 --- a/tests/integration/test_auto_resolution_quality.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python3 -"""Test AUTO resolution quality with real models.""" - -import asyncio -import sys - -from orchestrator.compiler.ambiguity_resolver import AmbiguityResolver -from orchestrator.integrations.ollama_model import OllamaModel - - -async def test_auto_resolution(): - """Test various AUTO resolution scenarios.""" - print("🧪 TESTING AUTO RESOLUTION QUALITY") - print("=" * 50) - - # Create resolver with real model - model = OllamaModel(model_name="llama3.2:1b") - if not model._is_available: - print("❌ Ollama model not available") - return False - - print(f"✅ Using model: {model.name}") - resolver = AmbiguityResolver(model=model) - - # Test cases - test_cases = [ - # (content, context, expected_type) - ( - "Choose best sources for healthcare AI research", - "parameters.sources", - "list"), - ("Determine appropriate search depth", "parameters.depth", "string"), - ("Select analysis method for research data", "parameters.method", "string"), - ("Set relevance threshold", "parameters.threshold", "number"), - ("Choose output format", "parameters.format", "string"), - ("Determine summary length", "parameters.length", "string"), - ("Detect programming language", "parameters.language", "string"), - ("Choose appropriate scan type", "parameters.scan_type", "string"), - ("Determine load level for testing", "parameters.load", "string"), - ("Match source code language", "parameters.language", "string"), - ("Choose optimization focus", "parameters.type", "string"), - ("Select appropriate metrics for customer data", "parameters.metrics", "list"), - ("Choose segmentation method", "parameters.method", "string"), - ("Set significance threshold", "parameters.threshold", "number"), - ("Choose format for business audience", "parameters.format", "string"), - ("Choose ML framework language", "parameters.language", "string"), - ] - - print(f"\n📋 Testing {len(test_cases)} AUTO resolution scenarios:\n") - - results = [] - for content, context, expected_type in test_cases: - try: - print(f"🔍 Content: '{content}'") - print(f" Context: {context}") - print(f" Expected: {expected_type}") - - resolved = await resolver.resolve(content, context) - - print(f" ✅ Resolved: '{resolved}' (type: {type(resolved).__name__})") - - # Check if resolution makes sense - is_valid = True - issues = [] - - if expected_type == "list" and not isinstance(resolved, list): - # For lists, check if it's a comma-separated string at least - if isinstance(resolved, str) and "," not in resolved: - is_valid = False - issues.append("Expected list or comma-separated values") - - if expected_type == "number" and isinstance(resolved, str): - try: - float(resolved) - except ValueError: - is_valid = False - issues.append("Expected numeric value") - - if isinstance(resolved, str) and len(resolved) < 2: - is_valid = False - issues.append("Resolution too short") - - if not is_valid: - print(f" ⚠️ Issues: {', '.join(issues)}") - - results.append((content, resolved, is_valid)) - print() - - except Exception as e: - print(f" ❌ Error: {e}") - results.append((content, None, False)) - print() - - # Summary - print("=" * 50) - print("📊 RESOLUTION QUALITY SUMMARY") - print("=" * 50) - - valid_count = sum(1 for _, _, valid in results if valid) - total_count = len(results) - - print( - f"\n✅ Valid resolutions: {valid_count}/{total_count} ({valid_count/total_count*100:.1f}%)" - ) - - if valid_count < total_count: - print("\n❌ Failed resolutions:") - for content, resolved, valid in results: - if not valid: - print(f" - '{content}' → '{resolved}'") - - return valid_count / total_count >= 0.7 - - -async def test_specific_resolutions(): - """Test specific problematic resolutions.""" - print("\n🔧 TESTING SPECIFIC RESOLUTIONS") - print("=" * 50) - - model = OllamaModel(model_name="llama3.2:1b") - if not model._is_available: - return False - - resolver = AmbiguityResolver(model=model) - - # Test the problematic "sources" resolution - print("\n1️⃣ Testing sources resolution:") - sources_prompt = "Choose best sources for healthcare AI research" - resolved = await resolver.resolve(sources_prompt, "parameters.sources") - print(f" Prompt: '{sources_prompt}'") - print(f" Resolved: '{resolved}'") - print(f" Type: {type(resolved).__name__}") - - # Direct model test - print("\n2️⃣ Testing direct model response:") - direct_prompt = "List the best sources for healthcare AI research. Answer with comma-separated values:" - direct_result = await model.generate(direct_prompt, max_tokens=20, temperature=0.1) - print(f" Prompt: '{direct_prompt}'") - print(f" Response: '{direct_result}'") - - return True - - -async def main(): - """Run AUTO resolution quality tests.""" - print("🚀 AUTO RESOLUTION QUALITY TESTING") - print("Testing how well real models resolve AUTO tags") - print("=" * 50) - - # Run tests - test1_passed = await test_auto_resolution() - test2_passed = await test_specific_resolutions() - - # Summary - print("\n" + "=" * 50) - print("📊 FINAL RESULTS") - print("=" * 50) - - if test1_passed and test2_passed: - print("✅ AUTO resolution quality is acceptable") - print("💡 Some resolutions may need improvement") - else: - print("❌ AUTO resolution quality needs improvement") - print("💡 Consider fine-tuning prompts or using a larger model") - - return test1_passed - - -if __name__ == "__main__": - success = asyncio.run(main()) - sys.exit(0 if success else 1) diff --git a/tests/integration/test_claude_skills_refactor.py b/tests/integration/test_claude_skills_refactor.py deleted file mode 100644 index a8ed4e26..00000000 --- a/tests/integration/test_claude_skills_refactor.py +++ /dev/null @@ -1,272 +0,0 @@ -"""Integration tests for Claude Skills refactor - Phase 5.""" - -import asyncio -import os -import pytest -from pathlib import Path - -from orchestrator.models.registry import ModelRegistry -from orchestrator.models.providers.anthropic_provider import AnthropicProvider -from orchestrator.models.providers.base import ProviderConfig -from orchestrator.skills import ( - RegistryInstaller, - SkillCreator, - SkillRegistry, - RealWorldSkillTester, -) -from orchestrator.compiler import EnhancedSkillsCompiler - - -class TestClaudeSkillsRefactorIntegration: - """Integration tests for the complete Claude Skills refactor.""" - - @pytest.mark.asyncio - async def test_end_to_end_model_registry(self): - """Test end-to-end model registry functionality.""" - print("\n" + "="*70) - print("TEST 1: Model Registry (Anthropic-Only)") - print("="*70) - - # Create Anthropic-only registry - registry = ModelRegistry() - - api_key = os.getenv("ANTHROPIC_API_KEY") - if not api_key: - pytest.skip("ANTHROPIC_API_KEY not set") - - # Configure Anthropic provider - registry.configure_provider( - provider_name="anthropic_test", - provider_type="anthropic", - config={"api_key": api_key} - ) - print("✅ Configured Anthropic provider") - - # Test that OpenAI is rejected - with pytest.raises(ValueError, match="Only 'anthropic' is supported"): - registry.configure_provider( - provider_name="openai_test", - provider_type="openai", - config={} - ) - print("✅ Correctly rejected non-Anthropic provider") - - # Initialize - await registry.initialize() - assert registry.is_initialized - print(f"✅ Registry initialized with {len(registry.providers)} provider(s)") - - # Check models - models = registry.available_models - assert len(models) > 0 - print(f"✅ Found {len(models)} available models") - - # Verify 2025 models are present - model_list = list(models.keys()) - assert any("opus-4" in m or "sonnet-4" in m or "haiku-4" in m for m in model_list) - print("✅ 2025 models registered") - - # Test health check - health = await registry.health_check() - assert "anthropic_test" in health - print(f"✅ Health check: {health['anthropic_test']}") - - await registry.cleanup() - print("✅ Test 1 Complete\n") - - @pytest.mark.asyncio - async def test_registry_installation(self): - """Test registry installation to ~/.orchestrator.""" - print("\n" + "="*70) - print("TEST 2: Registry Installation") - print("="*70) - - installer = RegistryInstaller() - print(f"Registry location: {installer.home_dir}") - - # Verify installation - if not installer.is_installed(): - installer.install() - print("✅ Installed registry") - else: - print("✅ Registry already installed") - - # Verify structure - status = installer.verify_installation() - assert all(status.values()), f"Installation incomplete: {status}" - print("✅ All registry components present") - - # Load registries - skills_reg = installer.get_skills_registry() - models_reg = installer.get_models_registry() - - print(f"✅ Skills registry version: {skills_reg.get('version')}") - print(f"✅ Models registry version: {models_reg.get('version')}") - print(f"✅ Registered skills: {len(skills_reg.get('skills', {}))}") - print(f"✅ Registered models: {len(models_reg.get('models', {}))}") - print("✅ Test 2 Complete\n") - - @pytest.mark.asyncio - async def test_skill_creation_with_roma(self): - """Test skill creation using ROMA pattern with real API.""" - print("\n" + "="*70) - print("TEST 3: Skill Creation (ROMA Pattern)") - print("="*70) - - api_key = os.getenv("ANTHROPIC_API_KEY") - if not api_key: - pytest.skip("ANTHROPIC_API_KEY not set") - - creator = SkillCreator(api_key=api_key) - print("✅ Skill creator initialized") - - # Create a simple skill - try: - skill = await creator.create_skill( - capability="Extract key information from markdown text", - pipeline_context={"purpose": "documentation_processing"}, - max_iterations=2 - ) - - print(f"✅ Created skill: {skill['name']}") - print(f" Description: {skill['description'][:80]}...") - print(f" Version: {skill['version']}") - print(f" Status: {skill['status']}") - print(f" Atomic tasks: {len(skill.get('atomic_tasks', []))}") - - # Verify skill has required fields - assert "name" in skill - assert "description" in skill - assert "version" in skill - print("✅ Skill structure validated") - - except Exception as e: - print(f"⚠️ Skill creation encountered issue: {e}") - print(" (This is acceptable if API limits hit)") - - print("✅ Test 3 Complete\n") - - @pytest.mark.asyncio - async def test_skill_registry_operations(self): - """Test skill registry operations.""" - print("\n" + "="*70) - print("TEST 4: Skill Registry Operations") - print("="*70) - - registry = SkillRegistry() - print(f"Registry: {registry.registry_dir}") - - # List skills - skills = registry.list_skills() - initial_count = len(skills) - print(f"✅ Initial skills: {initial_count}") - - # Search for skills - search_results = registry.search("test") - print(f"✅ Search works: found {len(search_results)} results for 'test'") - - # Get statistics - stats = registry.get_statistics() - print(f"✅ Statistics: {stats['total_skills']} total skills") - - print("✅ Test 4 Complete\n") - - @pytest.mark.asyncio - async def test_enhanced_compiler_basic(self): - """Test enhanced compiler with basic pipeline.""" - print("\n" + "="*70) - print("TEST 5: Enhanced Compiler (Basic)") - print("="*70) - - # Simple pipeline without skill auto-creation - simple_pipeline = """ -id: test-basic -name: "Basic Test" -version: "1.0.0" - -steps: - - id: step1 - action: llm_generate - parameters: - prompt: "Say hello" - model: claude-3-haiku-20240307 - max_tokens: 50 -""" - - compiler = EnhancedSkillsCompiler( - development_mode=True, - validate_templates=False, - validate_tools=False, - validate_models=False, - validate_data_flow=False, - ) - print("✅ Compiler initialized") - - try: - pipeline = await compiler.compile( - simple_pipeline, - auto_create_missing_skills=False # Disable for basic test - ) - - print(f"✅ Pipeline compiled: {pipeline.id}") - print(f" Tasks: {len(pipeline.tasks)}") - print(f" Name: {pipeline.name}") - - assert pipeline.id == "test-basic" - assert len(pipeline.tasks) == 1 - print("✅ Compilation validated") - - except Exception as e: - print(f"⚠️ Compilation issue: {e}") - # Log but don't fail test - schema compatibility in progress - - print("✅ Test 5 Complete\n") - - def test_components_summary(self): - """Summary of all components created.""" - print("\n" + "="*70) - print("CLAUDE SKILLS REFACTOR - COMPONENTS SUMMARY") - print("="*70) - - components = [ - ("Model Registry", "Anthropic-only, with 2025 models"), - ("Provider System", "Simplified to single provider"), - ("Registry Installer", "~/.orchestrator management"), - ("Skill Creator", "ROMA pattern (Atomize/Plan/Execute/Aggregate)"), - ("Skill Tester", "Real-world testing (NO MOCKS)"), - ("Skill Registry", "Management, search, import/export"), - ("Skills Compiler", "Skill-aware compilation"), - ("Enhanced Compiler", "Skills + control flow integration"), - ("Example Pipelines", "3 working demonstrations"), - ] - - print("\n✅ Components Implemented:") - for name, description in components: - print(f" • {name}: {description}") - - print("\n📁 File Structure:") - print(" src/orchestrator/") - print(" ├── models/") - print(" │ ├── registry.py (Anthropic-only)") - print(" │ └── providers/") - print(" │ └── anthropic_provider.py (2025 models)") - print(" ├── skills/") - print(" │ ├── installer.py (registry management)") - print(" │ ├── creator.py (ROMA pattern)") - print(" │ ├── tester.py (real-world testing)") - print(" │ └── registry.py (skill management)") - print(" └── compiler/") - print(" ├── skills_compiler.py (skill-aware)") - print(" └── enhanced_skills_compiler.py (full integration)") - - print("\n📋 Registry Structure (~/.orchestrator/):") - print(" ├── skills/") - print(" │ ├── registry.yaml") - print(" │ └── [skill-name]/") - print(" │ ├── skill.yaml") - print(" │ ├── implementation.py") - print(" │ └── tests/") - print(" └── models/") - print(" └── registry.yaml") - - print("\n✅ Components Summary Complete\n") \ No newline at end of file diff --git a/tests/integration/test_quick_real_models.py b/tests/integration/test_quick_real_models.py deleted file mode 100644 index a218956d..00000000 --- a/tests/integration/test_quick_real_models.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env python3 -"""Quick test of real model integration.""" - -import asyncio -import sys - -from orchestrator.integrations.ollama_model import OllamaModel - - -async def test_ollama_model(): - """Test Ollama model integration.""" - print("🦙 Testing Ollama Model Integration") - print("=" * 50) - - try: - # Test with llama3.2:1b (fastest available model) - print("📥 Loading llama3.2:1b...") - model = OllamaModel(model_name="llama3.2:1b") - - print(f"✅ Model created: {model.name}") - print(f"🔍 Available: {model._is_available}") - - if not model._is_available: - print("❌ Model not available") - return False - - # Test simple generation - print("\n🧪 Testing simple generation...") - result = await model.generate("What is 2+2?", max_tokens=10, temperature=0.1) - print(f"✅ Generated: {result}") - - # Test AUTO-style resolution - print("\n🎯 Testing AUTO resolution scenarios...") - - prompts = [ - "Choose the best format for data output: json, csv, or xml", - "Select appropriate batch size: small, medium, or large", - "Pick suitable timeout value: 10, 30, or 60 seconds", - ] - - for prompt in prompts: - try: - result = await model.generate(prompt, max_tokens=20, temperature=0.1) - # Extract just the choice - choice = result.split()[0] if result else "unknown" - print(f"✅ '{prompt}' → '{choice}'") - except Exception as e: - print(f"❌ Failed: {e}") - return False - - print("\n🎉 All Ollama tests passed!") - return True - - except Exception as e: - print(f"❌ Ollama test failed: {e}") - return False - - -async def test_ambiguity_resolver(): - """Test the ambiguity resolver with real model.""" - print("\n🔧 Testing Ambiguity Resolver with Real Model") - print("=" * 50) - - try: - from orchestrator.compiler.ambiguity_resolver import AmbiguityResolver - - # Create resolver (should auto-detect and use Ollama model) - print("🔍 Creating ambiguity resolver...") - resolver = AmbiguityResolver() - - print(f"✅ Using model: {resolver.model.name}") - print(f"📍 Provider: {resolver.model.provider}") - - # Test AUTO resolution - test_cases = [ - ("Choose output format", "config.format"), - ("Select batch size", "settings.batch_size"), - ("Pick analysis method", "task.method"), - ] - - print("\n🎯 Testing AUTO resolution:") - for content, context in test_cases: - try: - resolved = await resolver.resolve(content, context) - print(f"✅ '{content}' → '{resolved}'") - except Exception as e: - print(f"❌ Failed to resolve '{content}': {e}") - return False - - print("\n🎉 Ambiguity resolver tests passed!") - return True - - except Exception as e: - print(f"❌ Ambiguity resolver test failed: {e}") - return False - - -async def main(): - """Run quick real model tests.""" - print("🚀 QUICK REAL MODEL TESTS") - print("Testing with available Ollama models") - print("=" * 50) - - results = [] - - # Test 1: Direct Ollama model - success = await test_ollama_model() - results.append(("Ollama Model", success)) - - # Test 2: Ambiguity resolver integration - success = await test_ambiguity_resolver() - results.append(("Ambiguity Resolver", success)) - - # Summary - print(f"\n{'='*50}") - print("📊 TEST RESULTS") - print("=" * 50) - - passed = sum(1 for _, success in results if success) - total = len(results) - - for test_name, success in results: - status = "✅ PASS" if success else "❌ FAIL" - print(f"{status} {test_name}") - - overall_success = passed == total - print(f"\n📈 Tests: {passed}/{total} passed ({passed/total*100:.1f}%)") - - if overall_success: - print("🎉 ALL TESTS PASSED!") - print("✅ Real model integration working") - else: - print("⚠️ SOME TESTS FAILED") - - return overall_success - - -if __name__ == "__main__": - success = asyncio.run(main()) - sys.exit(0 if success else 1) diff --git a/tests/local/test_ollama_local.py b/tests/local/test_ollama_local.py deleted file mode 100644 index 68ad082a..00000000 --- a/tests/local/test_ollama_local.py +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env python3 -"""Local Ollama testing - only runs when Ollama is available.""" - -import sys -import subprocess -import pytest - -# Mark all tests in this file as local-only (not run in CI) -pytestmark = pytest.mark.local - - -def check_ollama_available(): - """Check if Ollama is available and has models.""" - try: - # Check if ollama command exists - result = subprocess.run( - ["ollama", "list"], capture_output=True, text=True) - if result.returncode == 0: - # Check for specific models - output = result.stdout - available_models = [] - for line in output.split("\n")[1:]: # Skip header - if line.strip(): - model_name = line.split()[0] - available_models.append(model_name) - return available_models - except (subprocess.TimeoutExpired, FileNotFoundError): - pass - return [] - - -# Skip all tests if Ollama not available -available_models = check_ollama_available() -skip_reason = "Ollama not available or no models installed" -if not available_models: - pytestmark = pytest.mark.skip(reason=skip_reason) - - -class TestOllamaIntegration: - """Test Ollama model integration when available.""" - - @pytest.mark.asyncio - async def test_ollama_model_creation(self): - """Test creating Ollama model.""" - from orchestrator.integrations.ollama_model import OllamaModel - - # Use first available model - model_name = available_models[0] if available_models else "llama3.2:1b" - model = OllamaModel(model_name=model_name) - - assert model.name == model_name - assert model.provider == "ollama" - assert model._is_available - - @pytest.mark.asyncio - async def test_ollama_generation(self): - """Test Ollama text generation.""" - from orchestrator.integrations.ollama_model import OllamaModel - - model_name = available_models[0] if available_models else "llama3.2:1b" - model = OllamaModel(model_name=model_name) - - result = await model.generate("2+2=", max_tokens=5, temperature=0.0) - assert isinstance(result, str) - assert len(result) > 0 - - @pytest.mark.asyncio - async def test_ollama_health_check(self): - """Test Ollama health check.""" - from orchestrator.integrations.ollama_model import OllamaModel - - model_name = available_models[0] if available_models else "llama3.2:1b" - model = OllamaModel(model_name=model_name) - - healthy = await model.health_check() - assert healthy is True - - @pytest.mark.asyncio - async def test_ambiguity_resolver_with_ollama(self): - """Test ambiguity resolver with Ollama model.""" - from orchestrator.integrations.ollama_model import OllamaModel - from orchestrator.compiler.ambiguity_resolver import AmbiguityResolver - - model_name = available_models[0] if available_models else "llama3.2:1b" - model = OllamaModel(model_name=model_name) - resolver = AmbiguityResolver(model=model) - - # Test simple resolution - result = await resolver.resolve("Choose format", "test.format") - assert isinstance(result, str) - assert len(result) > 0 - - @pytest.mark.asyncio - async def test_auto_model_detection(self): - """Test automatic Ollama model detection.""" - from orchestrator.compiler.ambiguity_resolver import AmbiguityResolver - from orchestrator.integrations.ollama_model import OllamaModel - from orchestrator.models.model_registry import ModelRegistry - - # Create a model registry and register an Ollama model - registry = ModelRegistry() - model_name = available_models[0] if available_models else "llama3.2:1b" - model = OllamaModel(model_name=model_name) - registry.register_model(model) - - # Create resolver with registry - resolver = AmbiguityResolver(model_registry=registry) - assert resolver.model is None # Model selected lazily - assert resolver.model_registry is registry - - # Test resolution works - this will trigger model selection - result = await resolver.resolve("json or csv", "data.format") - assert isinstance(result, str) - assert len(result) > 0 - - # Now model should be selected - assert resolver.model is not None - assert resolver.model.provider == "ollama" - - -@pytest.mark.asyncio -async def test_performance_comparison(): - """Compare performance of different available models.""" - if len(available_models) < 2: - pytest.skip("Need at least 2 models for comparison") - - from orchestrator.integrations.ollama_model import OllamaModel - import time - - results = {} - - for model_name in available_models[:3]: # Test up to 3 models - model = OllamaModel(model_name=model_name) - - start_time = time.time() - try: - result = await model.generate("Hello", max_tokens=5, temperature=0.0) - duration = time.time() - start_time - results[model_name] = { - "duration": duration, - "success": True, - "result": result, - } - except Exception as e: - duration = time.time() - start_time - results[model_name] = { - "duration": duration, - "success": False, - "error": str(e), - } - - print("\n🏁 Performance Results:") - for model, data in results.items(): - if data["success"]: - print(f"✅ {model}: {data['duration']:.2f}s - '{data['result']}'") - else: - print(f"❌ {model}: {data['duration']:.2f}s - {data['error']}") - - # At least one model should work - assert any(data["success"] for data in results.values()) - - -if __name__ == "__main__": - if not available_models: - print("⚠️ Ollama not available - skipping tests") - print("💡 To run these tests:") - print(" 1. Install Ollama: https://ollama.ai") - print(" 2. Pull a model: ollama pull llama3.2:1b") - print(" 3. Run tests: python test_ollama_local.py") - sys.exit(0) - - print(f"🦙 Available Ollama models: {', '.join(available_models)}") - print("🧪 Running Ollama integration tests...") - - # Run tests manually - import pytest - - exit_code = pytest.main([__file__, "-v"]) - sys.exit(exit_code) diff --git a/tests/local/test_simple_ollama.py b/tests/local/test_simple_ollama.py deleted file mode 100644 index f8a78ce6..00000000 --- a/tests/local/test_simple_ollama.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -"""Simple Ollama integration test.""" - -import asyncio -import sys -import pytest - -# Mark all tests in this file as local-only (not run in CI) -pytestmark = pytest.mark.local - - -async def test_ollama_direct(): - """Test Ollama integration directly.""" - print("🦙 Testing Ollama Integration") - print("=" * 40) - - try: - from orchestrator.integrations.ollama_model import OllamaModel - - # Create model with longer timeout - print("📥 Creating Ollama model...") - model = OllamaModel(model_name="llama3.2:1b") - - print(f"✅ Model: {model.name}") - print(f"🔍 Available: {model._is_available}") - - if not model._is_available: - print("❌ Model not available") - return False - - # Test health check first - print("\n🏥 Running health check...") - healthy = await model.health_check() - print(f"✅ Health check: {'PASS' if healthy else 'FAIL'}") - - if not healthy: - return False - - # Test simple generation - print("\n🧪 Testing generation...") - result = await model.generate("2+2=", max_tokens=3, temperature=0.0) - print(f"✅ Result: '{result}'") - - return True - - except Exception as e: - print(f"❌ Error: {e}") - import traceback - - traceback.print_exc() - return False - - -async def main(): - success = await test_ollama_direct() - print(f"\n{'✅ SUCCESS' if success else '❌ FAILED'}") - return success - - -if __name__ == "__main__": - success = asyncio.run(main()) - sys.exit(0 if success else 1) diff --git a/tests/models/test_integration.py b/tests/models/test_integration.py deleted file mode 100644 index dfcf61d4..00000000 --- a/tests/models/test_integration.py +++ /dev/null @@ -1,634 +0,0 @@ -#!/usr/bin/env python3 -# SKIPPED: This test file uses removed providers (OpenAI, Local) - Issue #426 -import pytest -pytest.skip("Skipping entire module - uses removed providers", allow_module_level=True) - -""" -Pipeline integration tests for multi-model system with execution engine. - -Tests complete integration between multi-model system and pipeline execution engine -from Issue #309 to validate: -- Model selection within pipeline execution contexts -- Execution engine compatibility with model providers -- Variable management integration with model outputs -- Progress tracking for model operations -- Recovery and checkpointing with model state -- End-to-end pipeline execution with real models -""" - -import asyncio -import os -import pytest -import sys -import time -from datetime import datetime -from pathlib import Path -from unittest.mock import AsyncMock, MagicMock - -# Add src to path for imports -sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) - -# Model system imports -from orchestrator.models.registry import ModelRegistry -from orchestrator.models.selection.manager import ModelSelectionManager -from orchestrator.models.selection.strategies import SelectionCriteria -from orchestrator.models.providers.base import ModelCapability -from orchestrator.models.providers.openai_provider import OpenAIProvider -from orchestrator.models.providers.anthropic_provider import AnthropicProvider -from orchestrator.models.providers.local_provider import LocalProvider - -# Execution engine imports -from orchestrator.execution.integration import ( - ComprehensiveExecutionManager, - create_comprehensive_execution_manager -) -from orchestrator.execution.state import ExecutionContext, ExecutionStatus -from orchestrator.execution.variables import VariableManager, VariableScope, VariableType -from orchestrator.execution.progress import ProgressTracker, ProgressEventType -from orchestrator.execution.recovery import RecoveryManager, RecoveryStrategy - -# Core orchestrator imports -from orchestrator.core.task import Task -from orchestrator.orchestrator import Orchestrator -from orchestrator.compiler.yaml_compiler import IntegratedYAMLCompiler - - -@pytest.mark.integration -class TestModelExecutionIntegration: - """Test integration between model system and execution engine.""" - - @pytest.fixture - async def integrated_model_registry(self): - """Create integrated model registry.""" - registry = ModelRegistry() - - # Add all providers - registry.add_provider(OpenAIProvider()) - registry.add_provider(AnthropicProvider()) - registry.add_provider(LocalProvider()) - - return registry - - @pytest.fixture - async def execution_manager(self): - """Create execution manager for testing.""" - return create_comprehensive_execution_manager("test_exec", "model_pipeline") - - @pytest.fixture - async def model_selection_manager(self, integrated_model_registry): - """Create model selection manager.""" - return ModelSelectionManager(integrated_model_registry) - - async def test_execution_context_model_integration( - self, execution_manager, model_selection_manager - ): - """Test model operations within execution context.""" - execution_manager.start_execution(total_steps=3) - - # Step 1: Model selection - execution_manager.start_step("model_selection", "Select appropriate model") - - criteria = SelectionCriteria( - required_capabilities=[ModelCapability.TEXT_GENERATION], - strategy="balanced" - ) - - try: - selection_result = await model_selection_manager.select_model(criteria) - - if selection_result and selection_result.model: - # Store model selection in execution context - execution_manager.variable_manager.set_variable( - "selected_model", - { - "name": selection_result.model.name, - "provider": selection_result.model.provider, - "confidence": selection_result.confidence, - "reasoning": selection_result.reasoning - }, - scope=VariableScope.EXECUTION, - var_type=VariableType.MODEL_REFERENCE - ) - - execution_manager.complete_step("model_selection", success=True) - - # Verify model reference is stored - stored_model = execution_manager.variable_manager.get_variable("selected_model") - assert stored_model is not None - assert stored_model["name"] == selection_result.model.name - assert stored_model["provider"] == selection_result.model.provider - - print(f"Selected model: {stored_model['provider']}:{stored_model['name']}") - - except Exception as e: - print(f"Model selection failed: {e}") - execution_manager.complete_step("model_selection", success=False) - pytest.skip("No suitable models available for integration testing") - - async def test_model_execution_with_progress_tracking( - self, execution_manager, integrated_model_registry - ): - """Test model execution with progress tracking.""" - execution_manager.start_execution(total_steps=2) - - # Find a working model - working_model = await self._find_working_model(integrated_model_registry) - if not working_model: - pytest.skip("No working models available") - - model_info, model_instance = working_model - - # Step 1: Model execution with progress tracking - execution_manager.start_step("model_generation", "Generate content with model") - - # Update progress during execution - execution_manager.update_step_progress("model_generation", 25.0, "Starting generation") - - try: - # Track generation time - start_time = time.time() - result = await model_instance.generate( - "Explain machine learning in one paragraph", - max_tokens=100, - temperature=0.3 - ) - duration = time.time() - start_time - - execution_manager.update_step_progress("model_generation", 75.0, "Generation complete") - - # Store result in execution context - execution_manager.variable_manager.set_variable( - "generation_result", - { - "content": result, - "model": model_info.name, - "provider": model_info.provider, - "duration_ms": duration * 1000, - "timestamp": datetime.now().isoformat() - }, - scope=VariableScope.EXECUTION, - var_type=VariableType.TASK_RESULT - ) - - execution_manager.complete_step("model_generation", success=True) - - # Verify result storage - stored_result = execution_manager.variable_manager.get_variable("generation_result") - assert stored_result is not None - assert stored_result["content"] == result - assert stored_result["model"] == model_info.name - assert stored_result["duration_ms"] > 0 - - print(f"Generated {len(result)} chars in {duration:.2f}s using {model_info.name}") - - except Exception as e: - execution_manager.handle_step_error("model_generation", "Generate content", e) - execution_manager.complete_step("model_generation", success=False) - raise - - async def test_model_execution_with_recovery( - self, execution_manager, integrated_model_registry - ): - """Test model execution with error recovery.""" - execution_manager.start_execution(total_steps=1) - - # Find a working model - working_model = await self._find_working_model(integrated_model_registry) - if not working_model: - pytest.skip("No working models available") - - _, model_instance = working_model - - async def model_executor(): - """Executor that might fail and need recovery.""" - # Simulate potential network issues - result = await model_instance.generate( - "Test generation", - max_tokens=10, - temperature=0.1 - ) - return result - - # Execute with recovery support - success = await execution_manager.execute_step_with_recovery( - "model_task", "Execute model with recovery", model_executor - ) - - assert success is True - - # Check that step completed - step_progress = execution_manager.progress_tracker.get_step_progress( - "test_exec", "model_task" - ) - assert step_progress is not None - assert step_progress.progress_percentage == 100.0 - - async def test_checkpoint_with_model_state( - self, execution_manager, integrated_model_registry - ): - """Test checkpointing with model state.""" - execution_manager.start_execution(total_steps=2) - - # Find a working model - working_model = await self._find_working_model(integrated_model_registry) - if not working_model: - pytest.skip("No working models available") - - model_info, model_instance = working_model - - # Execute first step with model - execution_manager.start_step("step1", "First model operation") - - result1 = await model_instance.generate("Hello", max_tokens=5, temperature=0.1) - - # Store model result - execution_manager.variable_manager.set_variable( - "step1_result", - { - "content": result1, - "model": model_info.name, - "step": "step1" - }, - scope=VariableScope.EXECUTION - ) - - execution_manager.complete_step("step1", success=True) - - # Create checkpoint after first step - checkpoint = execution_manager.create_checkpoint("after_step1") - assert checkpoint is not None - - # Start second step - execution_manager.start_step("step2", "Second model operation") - - result2 = await model_instance.generate("World", max_tokens=5, temperature=0.1) - - execution_manager.variable_manager.set_variable( - "step2_result", - { - "content": result2, - "model": model_info.name, - "step": "step2" - }, - scope=VariableScope.EXECUTION - ) - - # Restore checkpoint - restore_success = execution_manager.restore_checkpoint(checkpoint.id) - assert restore_success is True - - # Verify state was restored - step1_result = execution_manager.variable_manager.get_variable("step1_result") - step2_result = execution_manager.variable_manager.get_variable("step2_result") - - assert step1_result is not None - assert step1_result["content"] == result1 - assert step2_result is None # Should be cleared by restore - - async def _find_working_model(self, registry): - """Find a working model for testing.""" - providers = registry.get_providers() - - for provider in providers: - try: - models = await provider.get_available_models() - for model_info in models[:1]: # Try first model - try: - model_instance = await provider.create_model(model_info.name) - if model_instance: - return (model_info, model_instance) - except Exception as e: - print(f"Failed to create {model_info.name}: {e}") - continue - except Exception as e: - print(f"Provider {provider.name} failed: {e}") - continue - - return None - - -@pytest.mark.integration -class TestPipelineExecutionWithModels: - """Test end-to-end pipeline execution with multi-model integration.""" - - @pytest.fixture - async def integrated_orchestrator(self): - """Create orchestrator with integrated model system.""" - from orchestrator.core.control_system import ControlSystem - - # Create a control system that integrates with models - class ModelAwareControlSystem(ControlSystem): - def __init__(self): - config = { - "capabilities": { - "supported_actions": ["generate", "analyze", "summarize"], - "model_integration": True - } - } - super().__init__(name="model-aware-control", config=config) - - # Initialize model system - self.model_registry = ModelRegistry() - self.model_registry.add_provider(OpenAIProvider()) - self.model_registry.add_provider(AnthropicProvider()) - self.model_registry.add_provider(LocalProvider()) - - self.model_selection_manager = ModelSelectionManager(self.model_registry) - self._task_results = {} - - async def execute_task(self, task: Task, context: dict = None): - """Execute task with model integration.""" - if task.action == "generate": - return await self._generate_content(task) - elif task.action == "analyze": - return await self._analyze_content(task) - else: - return {"status": "completed", "result": f"Executed {task.action}"} - - async def _generate_content(self, task): - """Generate content using selected model.""" - prompt = task.parameters.get("prompt", "") - - # Select appropriate model - criteria = SelectionCriteria( - required_capabilities=[ModelCapability.TEXT_GENERATION], - strategy="balanced" - ) - - try: - selection_result = await self.model_selection_manager.select_model(criteria) - - if not selection_result or not selection_result.model: - return {"status": "failed", "error": "No suitable model found"} - - # Create model instance - provider = self._get_provider(selection_result.model.provider) - model_instance = await provider.create_model(selection_result.model.name) - - # Generate content - result = await model_instance.generate( - prompt, - max_tokens=100, - temperature=0.3 - ) - - result_data = { - "status": "completed", - "content": result, - "model": { - "name": selection_result.model.name, - "provider": selection_result.model.provider - }, - "confidence": selection_result.confidence - } - - self._task_results[task.id] = result_data - return result_data - - except Exception as e: - error_result = { - "status": "failed", - "error": str(e), - "task_id": task.id - } - self._task_results[task.id] = error_result - return error_result - - async def _analyze_content(self, task): - """Analyze content using selected model.""" - content = task.parameters.get("content", "") - if isinstance(content, str) and content.startswith("$results."): - # Resolve reference - ref_task_id = content.split(".")[1] - if ref_task_id in self._task_results: - referenced_result = self._task_results[ref_task_id] - content = referenced_result.get("content", "") - - # For analysis, prefer higher quality models - criteria = SelectionCriteria( - required_capabilities=[ModelCapability.ANALYSIS], - min_quality_tier="high", - strategy="balanced" - ) - - try: - selection_result = await self.model_selection_manager.select_model(criteria) - - if not selection_result or not selection_result.model: - # Fallback to any text generation model - criteria = SelectionCriteria( - required_capabilities=[ModelCapability.TEXT_GENERATION] - ) - selection_result = await self.model_selection_manager.select_model(criteria) - - if not selection_result or not selection_result.model: - return {"status": "failed", "error": "No suitable analysis model found"} - - provider = self._get_provider(selection_result.model.provider) - model_instance = await provider.create_model(selection_result.model.name) - - analysis_prompt = f"Analyze the following content and provide key insights:\n\n{content}" - analysis = await model_instance.generate( - analysis_prompt, - max_tokens=150, - temperature=0.2 - ) - - result_data = { - "status": "completed", - "analysis": analysis, - "original_content_length": len(content), - "model": { - "name": selection_result.model.name, - "provider": selection_result.model.provider - } - } - - self._task_results[task.id] = result_data - return result_data - - except Exception as e: - error_result = {"status": "failed", "error": str(e)} - self._task_results[task.id] = error_result - return error_result - - def _get_provider(self, provider_name): - """Get provider by name.""" - for provider in self.model_registry.get_providers(): - if provider.name == provider_name: - return provider - raise ValueError(f"Provider {provider_name} not found") - - async def execute_pipeline(self, pipeline, context=None): - raise NotImplementedError("Use orchestrator for pipeline execution") - - def get_capabilities(self): - return self.config.get("capabilities", {}) - - async def health_check(self): - return {"status": "healthy", "name": self.name} - - control_system = ModelAwareControlSystem() - orchestrator = Orchestrator(control_system=control_system) - - return orchestrator - - async def test_simple_model_pipeline(self, integrated_orchestrator): - """Test simple pipeline with model operations.""" - pipeline_yaml = """ -name: "model_integration_test" -description: "Test pipeline with model integration" - -steps: - - id: generate - action: generate - parameters: - prompt: "Explain artificial intelligence in two sentences" - - - id: analyze - action: analyze - depends_on: [generate] - parameters: - content: "$results.generate.content" -""" - - print("🚀 Executing model integration pipeline...") - - try: - results = await integrated_orchestrator.execute_yaml(pipeline_yaml, context={}) - - print(f"✅ Pipeline completed with {len(results)} tasks") - - # Verify generation task - assert "generate" in results - generate_result = results["generate"] - assert generate_result["success"] == True - assert "content" in generate_result - assert len(generate_result["content"]) > 0 - - print(f"Generated content: {generate_result['content'][:100]}...") - print(f"Used model: {generate_result['model']['provider']}:{generate_result['model']['name']}") - - # Verify analysis task - if "analyze" in results: - analyze_result = results["analyze"] - if analyze_result["success"] == True: - assert "analysis" in analyze_result - assert len(analyze_result["analysis"]) > 0 - - print(f"Analysis: {analyze_result['analysis'][:100]}...") - print(f"Analysis model: {analyze_result['model']['provider']}:{analyze_result['model']['name']}") - - except Exception as e: - print(f"Pipeline execution failed: {e}") - pytest.skip("Pipeline execution failed - likely no working models") - - async def test_complex_model_pipeline(self, integrated_orchestrator): - """Test complex pipeline with multiple model operations.""" - pipeline_yaml = """ -name: "complex_model_pipeline" -description: "Complex pipeline with multiple model operations" - -steps: - - id: generate_topic - action: generate - parameters: - prompt: "Generate a technical topic related to machine learning (just the topic name)" - - - id: explain_topic - action: generate - depends_on: [generate_topic] - parameters: - prompt: "Explain this topic in detail: $results.generate_topic.content" - - - id: analyze_explanation - action: analyze - depends_on: [explain_topic] - parameters: - content: "$results.explain_topic.content" -""" - - print("🔧 Executing complex model pipeline...") - - try: - results = await integrated_orchestrator.execute_yaml(pipeline_yaml, context={}) - - print(f"✅ Complex pipeline completed with {len(results)} tasks") - - # Should have all three tasks - expected_tasks = ["generate_topic", "explain_topic", "analyze_explanation"] - for task_id in expected_tasks: - if task_id in results: - result = results[task_id] - print(f"{task_id}: {result['status']}") - - if result["success"] == True: - if "content" in result: - print(f" Content length: {len(result['content'])}") - if "analysis" in result: - print(f" Analysis length: {len(result['analysis'])}") - if "model" in result: - print(f" Model: {result['model']['provider']}:{result['model']['name']}") - - # At least the first task should complete - assert "generate_topic" in results - assert results["generate_topic"]["status"] == "completed" - - except Exception as e: - print(f"Complex pipeline failed: {e}") - pytest.skip("Complex pipeline execution failed") - - async def test_model_error_handling_in_pipeline(self, integrated_orchestrator): - """Test error handling when model operations fail.""" - pipeline_yaml = """ -name: "error_handling_test" -description: "Test pipeline error handling with models" - -steps: - - id: valid_generation - action: generate - parameters: - prompt: "Hello world" - - - id: analyze_results - action: analyze - depends_on: [valid_generation] - parameters: - content: "$results.valid_generation.content" -""" - - try: - results = await integrated_orchestrator.execute_yaml(pipeline_yaml, context={}) - - # Should handle errors gracefully - print(f"Error handling test completed with {len(results)} results") - - for task_id, result in results.items(): - print(f"{task_id}: {result['status']}") - if result["success"] == False: - print(f" Error: {result.get('error', 'Unknown error')}") - - except Exception as e: - print(f"Error handling test raised exception: {e}") - # This is acceptable as we're testing error scenarios - - -async def main(): - """Run pipeline integration tests.""" - print("🔗 PIPELINE INTEGRATION TESTS") - print("=" * 60) - - # Run pytest with this file - exit_code = pytest.main([ - __file__, - "-v", - "--tb=short", - "-m", "integration" - ]) - - return exit_code == 0 - - -if __name__ == "__main__": - success = asyncio.run(main()) - sys.exit(0 if success else 1) \ No newline at end of file diff --git a/tests/models/test_providers.py b/tests/models/test_providers.py deleted file mode 100644 index 29a26032..00000000 --- a/tests/models/test_providers.py +++ /dev/null @@ -1,441 +0,0 @@ -#!/usr/bin/env python3 -""" -Provider implementation tests for multi-model integration. - -Tests real provider implementations with actual API calls to validate: -- Provider abstractions work with real services -- Authentication and configuration handling -- Error handling and resilience -- Response parsing and standardization -""" - -import asyncio -import os -import pytest -import sys -from datetime import datetime -from pathlib import Path - -# Add src to path for imports -sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) - -from orchestrator.models.providers.base import ModelProvider, ProviderError, ProviderConfig -from orchestrator.models.providers.anthropic_provider import AnthropicProvider -from orchestrator.models.registry import ModelRegistry -from orchestrator.core.model import Model, ModelCapabilities, ModelCost, ModelRequirements - -# Note: OpenAIProvider and LocalProvider removed in Claude Skills refactor (Issue #426) -# Tests for these providers are skipped - - -class TestProviderAbstractions: - """Test base provider abstraction contracts.""" - - def test_base_provider_interface(self): - """Test that base provider defines correct interface.""" - # Check that BaseProvider has required methods - required_methods = [ - 'get_available_models', - 'create_model', - 'health_check', - 'get_capabilities' - ] - - for method in required_methods: - assert hasattr(BaseProvider, method) - assert callable(getattr(BaseProvider, method)) - - def test_model_info_structure(self): - """Test ModelInfo structure.""" - model_info = ModelInfo( - name="test-model", - provider="test", - capabilities=[ModelCapability.TEXT_GENERATION], - context_window=4096, - cost_per_token=0.001 - ) - - assert model_info.name == "test-model" - assert model_info.provider == "test" - assert ModelCapability.TEXT_GENERATION in model_info.capabilities - assert model_info.context_window == 4096 - assert model_info.cost_per_token == 0.001 - - def test_model_capability_enum(self): - """Test ModelCapability enum values.""" - capabilities = [ - ModelCapability.TEXT_GENERATION, - ModelCapability.CODE_GENERATION, - ModelCapability.ANALYSIS, - ModelCapability.SUMMARIZATION, - ModelCapability.TRANSLATION - ] - - # All capabilities should be valid - for capability in capabilities: - assert isinstance(capability, ModelCapability) - - -@pytest.mark.integration -@pytest.mark.skip(reason="OpenAI provider removed in Claude Skills refactor (Issue #426)") -class TestOpenAIProvider: - """Test OpenAI provider with real API calls.""" - - @pytest.fixture - def openai_provider(self): - """Create OpenAI provider instance.""" - pytest.skip("OpenAI provider removed") - - def test_openai_provider_initialization(self, openai_provider): - """Test OpenAI provider can be created.""" - assert openai_provider is not None - assert isinstance(openai_provider, BaseProvider) - assert openai_provider.name == "openai" - - async def test_openai_health_check(self, openai_provider): - """Test OpenAI health check.""" - if not os.getenv("OPENAI_API_KEY"): - pytest.skip("OPENAI_API_KEY not set - skipping real API test") - - health = await openai_provider.health_check() - assert "status" in health - # Should be "healthy" if API key is valid, otherwise may be "degraded" - assert health["status"] in ["healthy", "degraded", "unhealthy"] - - async def test_openai_get_available_models(self, openai_provider): - """Test getting available OpenAI models.""" - if not os.getenv("OPENAI_API_KEY"): - pytest.skip("OPENAI_API_KEY not set - skipping real API test") - - models = await openai_provider.get_available_models() - - # Should have at least some models - assert len(models) > 0 - - # Check model structure - for model in models: - assert isinstance(model, ModelInfo) - assert model.provider == "openai" - assert len(model.capabilities) > 0 - - async def test_openai_create_model(self, openai_provider): - """Test creating OpenAI model instance.""" - if not os.getenv("OPENAI_API_KEY"): - pytest.skip("OPENAI_API_KEY not set - skipping real API test") - - # Create with a known model - model = await openai_provider.create_model("gpt-4o-mini") - - assert model is not None - assert hasattr(model, 'generate') - assert hasattr(model, 'name') - - # Test simple generation - result = await model.generate("What is 2+2?", max_tokens=10, temperature=0.1) - assert isinstance(result, str) - assert len(result) > 0 - print(f"OpenAI generation: {result}") - - def test_openai_capabilities(self, openai_provider): - """Test OpenAI provider capabilities.""" - capabilities = openai_provider.get_capabilities() - - assert isinstance(capabilities, dict) - assert "supported_models" in capabilities - assert "features" in capabilities - assert "limitations" in capabilities - - -@pytest.mark.integration -class TestAnthropicProvider: - """Test Anthropic provider with real API calls.""" - - @pytest.fixture - def anthropic_provider(self): - """Create Anthropic provider instance.""" - return AnthropicProvider() - - def test_anthropic_provider_initialization(self, anthropic_provider): - """Test Anthropic provider can be created.""" - assert anthropic_provider is not None - assert isinstance(anthropic_provider, BaseProvider) - assert anthropic_provider.name == "anthropic" - - async def test_anthropic_health_check(self, anthropic_provider): - """Test Anthropic health check.""" - if not os.getenv("ANTHROPIC_API_KEY"): - pytest.skip("ANTHROPIC_API_KEY not set - skipping real API test") - - health = await anthropic_provider.health_check() - assert "status" in health - assert health["status"] in ["healthy", "degraded", "unhealthy"] - - async def test_anthropic_get_available_models(self, anthropic_provider): - """Test getting available Anthropic models.""" - if not os.getenv("ANTHROPIC_API_KEY"): - pytest.skip("ANTHROPIC_API_KEY not set - skipping real API test") - - models = await anthropic_provider.get_available_models() - - # Should have Claude models - assert len(models) > 0 - - # Check for known Claude models - model_names = [m.name for m in models] - claude_models = [name for name in model_names if "claude" in name.lower()] - assert len(claude_models) > 0 - - # Check model structure - for model in models: - assert isinstance(model, ModelInfo) - assert model.provider == "anthropic" - - async def test_anthropic_create_model(self, anthropic_provider): - """Test creating Anthropic model instance.""" - if not os.getenv("ANTHROPIC_API_KEY"): - pytest.skip("ANTHROPIC_API_KEY not set - skipping real API test") - - # Create with Claude model (adjust based on what's available) - model = await anthropic_provider.create_model("claude-3-haiku-20240307") - - assert model is not None - assert hasattr(model, 'generate') - assert hasattr(model, 'name') - - # Test simple generation - result = await model.generate("What is 3+3?", max_tokens=10, temperature=0.1) - assert isinstance(result, str) - assert len(result) > 0 - print(f"Anthropic generation: {result}") - - -@pytest.mark.integration -@pytest.mark.skip(reason="Local provider removed in Claude Skills refactor (Issue #426)") -class TestLocalProvider: - """Test local provider with Ollama and HuggingFace models.""" - - @pytest.fixture - def local_provider(self): - """Create local provider instance.""" - pytest.skip("Local provider removed") - - def test_local_provider_initialization(self, local_provider): - """Test local provider can be created.""" - assert local_provider is not None - assert isinstance(local_provider, BaseProvider) - assert local_provider.name == "local" - - async def test_local_health_check(self, local_provider): - """Test local provider health check.""" - health = await local_provider.health_check() - assert "status" in health - assert health["status"] in ["healthy", "degraded", "unhealthy"] - - # Should include details about available local services - assert "ollama_available" in health - assert "huggingface_available" in health - - async def test_local_get_available_models(self, local_provider): - """Test getting available local models.""" - models = await local_provider.get_available_models() - - # May have no models if nothing is installed locally - assert isinstance(models, list) - - # If we have models, check their structure - for model in models: - assert isinstance(model, ModelInfo) - assert model.provider == "local" - - async def test_ollama_integration(self, local_provider): - """Test Ollama integration through local provider.""" - # Check if Ollama is available - if not OllamaModel.check_ollama_installation(): - pytest.skip("Ollama not installed - skipping Ollama tests") - - try: - # Try to create a small Ollama model - model = await local_provider.create_model("llama3.2:1b") - if model is None: - pytest.skip("llama3.2:1b not available - skipping test") - - # Test generation - result = await model.generate("Hello", max_tokens=5, temperature=0.1) - assert isinstance(result, str) - assert len(result) > 0 - print(f"Ollama generation: {result}") - - except Exception as e: - pytest.skip(f"Ollama model creation failed: {e}") - - async def test_huggingface_integration(self, local_provider): - """Test HuggingFace integration through local provider.""" - try: - # Try to create a small HuggingFace model - model = await local_provider.create_model("TinyLlama/TinyLlama-1.1B-Chat-v1.0") - if model is None: - pytest.skip("HuggingFace model not available - skipping test") - - # Test generation - result = await model.generate("Hi", max_tokens=5, temperature=0.1) - assert isinstance(result, str) - assert len(result) > 0 - print(f"HuggingFace generation: {result}") - - except Exception as e: - pytest.skip(f"HuggingFace model creation failed: {e}") - - -@pytest.mark.integration -@pytest.mark.skip(reason="Multi-provider registry removed in Claude Skills refactor (Issue #426)") -class TestProviderRegistry: - """Test provider registry integration.""" - - @pytest.fixture - def model_registry(self): - """Create model registry with all providers.""" - pytest.skip("Multi-provider registry not applicable") - - async def test_registry_provider_enumeration(self, model_registry): - """Test enumerating all providers in registry.""" - providers = model_registry.get_providers() - - assert len(providers) == 3 - provider_names = [p.name for p in providers] - assert "openai" in provider_names - assert "anthropic" in provider_names - assert "local" in provider_names - - async def test_registry_model_discovery(self, model_registry): - """Test discovering models across all providers.""" - all_models = [] - - # Discover models from each provider - providers = model_registry.get_providers() - for provider in providers: - try: - models = await provider.get_available_models() - all_models.extend(models) - except Exception as e: - # Some providers may fail if not configured - print(f"Provider {provider.name} failed model discovery: {e}") - continue - - # Should have found at least some models - print(f"Total models discovered: {len(all_models)}") - - # Check model diversity - providers_with_models = set(model.provider for model in all_models) - print(f"Providers with models: {providers_with_models}") - - async def test_registry_model_creation(self, model_registry): - """Test creating models through registry.""" - # Try to create any available model - providers = model_registry.get_providers() - - model_created = False - for provider in providers: - try: - models = await provider.get_available_models() - if not models: - continue - - # Try first model - test_model = models[0] - model_instance = await provider.create_model(test_model.name) - - if model_instance: - # Test basic functionality - result = await model_instance.generate( - "Test", max_tokens=3, temperature=0.1 - ) - assert isinstance(result, str) - print(f"Successfully created {provider.name}:{test_model.name}") - model_created = True - break - - except Exception as e: - print(f"Failed to test {provider.name}: {e}") - continue - - # Should have created at least one working model - if not model_created: - pytest.skip("No working models available for testing") - - -@pytest.mark.integration -@pytest.mark.skip(reason="Multi-provider resilience tests not applicable (Issue #426)") -class TestProviderResilience: - """Test provider error handling and resilience.""" - - def test_invalid_api_key_handling(self): - """Test handling of invalid API keys.""" - pytest.skip("OpenAI provider removed") - - # Should not raise exception on creation - assert provider is not None - - # Health check should detect the issue - asyncio.run(self._test_degraded_health(provider)) - - async def _test_degraded_health(self, provider): - """Helper to test degraded health status.""" - health = await provider.health_check() - # Should be degraded or unhealthy with invalid key - assert health["status"] in ["degraded", "unhealthy"] - - async def test_network_error_handling(self): - """Test handling of network errors.""" - # This would require mocking network calls - # For now, we'll test timeout scenarios - pytest.skip("Network error handling test requires mock setup") - - async def test_rate_limit_handling(self): - """Test handling of rate limits.""" - # This would require triggering actual rate limits - pytest.skip("Rate limit testing requires controlled load generation") - - -@pytest.mark.skip(reason="Multi-provider compatibility tests not applicable (Issue #426)") -class TestProviderCompatibility: - """Test compatibility across different provider versions.""" - - def test_provider_version_compatibility(self): - """Test provider compatibility with different API versions.""" - pytest.skip("Multi-provider tests not applicable") - - for provider in providers: - capabilities = provider.get_capabilities() - assert "version" in capabilities or "api_version" in capabilities - print(f"{provider.name} capabilities: {capabilities}") - - def test_provider_feature_parity(self): - """Test feature parity across providers.""" - pytest.skip("Multi-provider tests not applicable") - - # All providers should support basic text generation - for provider in providers: - capabilities = provider.get_capabilities() - assert "text_generation" in str(capabilities).lower() or \ - "generation" in str(capabilities).lower() - - -async def main(): - """Run provider integration tests.""" - print("🚀 PROVIDER INTEGRATION TESTS") - print("=" * 60) - - # Run pytest with this file - exit_code = pytest.main([ - __file__, - "-v", - "--tb=short", - "-m", "integration" - ]) - - return exit_code == 0 - - -if __name__ == "__main__": - success = asyncio.run(main()) - sys.exit(0 if success else 1) \ No newline at end of file diff --git a/tests/models/validate_integration.py b/tests/models/validate_integration.py deleted file mode 100644 index 0ac4ecbf..00000000 --- a/tests/models/validate_integration.py +++ /dev/null @@ -1,489 +0,0 @@ -#!/usr/bin/env python3 -# SKIPPED: This test file uses removed providers (OpenAI, Local) - Issue #426 -import pytest -pytest.skip("Skipping entire module - uses removed providers", allow_module_level=True) - -""" -Comprehensive validation script for multi-model integration with execution engine. - -This script validates that all components work together: -- Model providers are accessible and functional -- Selection strategies work with real models -- Pipeline execution integrates correctly with model system -- Performance optimizations are effective -- All integration points work as expected - -This serves as the comprehensive test for Issue #311 Stream C. -""" - -import asyncio -import os -import sys -import time -import traceback -from pathlib import Path -from typing import Dict, List, Any, Optional - -# Add src to path for imports -sys.path.insert(0, str(Path(__file__).parent.parent.parent / "src")) - -from orchestrator.models.registry import ModelRegistry -from orchestrator.models.providers.openai_provider import OpenAIProvider -from orchestrator.models.providers.anthropic_provider import AnthropicProvider -from orchestrator.models.providers.local_provider import LocalProvider -from orchestrator.models.selection.strategies import TaskRequirements, CostOptimizedStrategy -from orchestrator.models.selection.manager import ModelSelectionManager -from orchestrator.execution.integration import create_comprehensive_execution_manager -from orchestrator.execution.variables import VariableScope, VariableType -from orchestrator.core.model import ModelCapabilities, ModelCost -from orchestrator.orchestrator import Orchestrator -from orchestrator.compiler.yaml_compiler import IntegratedYAMLCompiler - - -class IntegrationValidator: - """Validates complete multi-model system integration.""" - - def __init__(self): - self.results = [] - self.registry = None - self.execution_manager = None - self.working_models = [] - - def log_result(self, test_name: str, success: bool, details: str = "", error: str = ""): - """Log test result.""" - status = "✅ PASS" if success else "❌ FAIL" - print(f"{status} {test_name}") - if details: - print(f" {details}") - if error: - print(f" Error: {error}") - - self.results.append({ - "test": test_name, - "success": success, - "details": details, - "error": error - }) - - async def validate_model_registry(self): - """Validate model registry functionality.""" - print("\n🔧 VALIDATING MODEL REGISTRY") - print("-" * 40) - - try: - # Create registry - self.registry = ModelRegistry() - - # Register providers - self.registry.register_provider(OpenAIProvider()) - self.registry.register_provider(AnthropicProvider()) - self.registry.register_provider(LocalProvider()) - - self.log_result("Registry Creation", True, f"Created with {len(self.registry.providers)} providers") - - # Initialize registry - await self.registry.initialize() - self.log_result("Registry Initialization", self.registry.is_initialized, - f"Initialized: {self.registry.is_initialized}") - - # Check health - health_status = await self.registry.health_check() - healthy_providers = sum(1 for status in health_status.values() if status) - self.log_result("Provider Health Check", healthy_providers > 0, - f"{healthy_providers}/{len(health_status)} providers healthy") - - # Get registry info - info = self.registry.get_registry_info() - self.log_result("Registry Info", True, - f"{info['provider_count']} providers, {info['total_models']} models") - - return True - - except Exception as e: - self.log_result("Registry Validation", False, error=str(e)) - return False - - async def validate_model_discovery(self): - """Validate model discovery across providers.""" - print("\n🔍 VALIDATING MODEL DISCOVERY") - print("-" * 40) - - if not self.registry: - self.log_result("Model Discovery", False, error="Registry not available") - return False - - try: - # Discover all models - discovered = await self.registry.discover_all_models() - - total_models = sum(len(models) for models in discovered.values()) - self.log_result("Model Discovery", total_models > 0, - f"Found {total_models} models across {len(discovered)} providers") - - # Test getting a specific model - working_model_found = False - for provider_name, models in discovered.items(): - for model_name in models[:1]: # Test first model from each provider - try: - model = await self.registry.get_model(model_name, provider_name) - if model: - self.working_models.append((provider_name, model_name, model)) - working_model_found = True - self.log_result(f"Model Creation ({provider_name})", True, - f"Successfully created {model_name}") - break - except Exception as e: - self.log_result(f"Model Creation ({provider_name})", False, - f"Failed to create {model_name}", str(e)) - continue - - if working_model_found: - break - - return working_model_found - - except Exception as e: - self.log_result("Model Discovery", False, error=str(e)) - return False - - async def validate_model_selection(self): - """Validate model selection strategies.""" - print("\n🎯 VALIDATING MODEL SELECTION") - print("-" * 40) - - if not self.registry: - self.log_result("Model Selection", False, error="Registry not available") - return False - - try: - # Create selection manager - selection_manager = ModelSelectionManager(self.registry) - - # Test different selection strategies - strategies_to_test = [ - ("cost_optimized", TaskRequirements( - task_type="text_generation", - max_cost_per_1k_tokens=0.01, - prefer_local=True - )), - ("performance_optimized", TaskRequirements( - task_type="text_generation", - max_latency_ms=5000, - required_capabilities=["text_generation"] - )), - ("balanced", TaskRequirements( - task_type="text_generation", - max_cost_per_1k_tokens=0.02, - accuracy_threshold=0.8 - )) - ] - - successful_selections = 0 - for strategy_name, requirements in strategies_to_test: - try: - selected_model = await selection_manager.select_model(requirements) - if selected_model: - successful_selections += 1 - self.log_result(f"Selection Strategy ({strategy_name})", True, - f"Selected: {selected_model}") - else: - self.log_result(f"Selection Strategy ({strategy_name})", False, - "No suitable model found") - except Exception as e: - self.log_result(f"Selection Strategy ({strategy_name})", False, - error=str(e)) - - return successful_selections > 0 - - except Exception as e: - self.log_result("Model Selection", False, error=str(e)) - return False - - async def validate_execution_integration(self): - """Validate integration with execution engine.""" - print("\n⚙️ VALIDATING EXECUTION ENGINE INTEGRATION") - print("-" * 40) - - try: - # Create execution manager - self.execution_manager = create_comprehensive_execution_manager( - "integration_test", "model_pipeline" - ) - - self.log_result("Execution Manager Creation", True, - "Created comprehensive execution manager") - - # Start execution - self.execution_manager.start_execution(total_steps=3) - - # Step 1: Model selection within execution context - self.execution_manager.start_step("model_selection", "Select model for task") - - # Store model selection in execution variables - if self.working_models: - provider_name, model_name, model = self.working_models[0] - - self.execution_manager.variable_manager.set_variable( - "selected_model", - { - "name": model_name, - "provider": provider_name, - "capabilities": ["text_generation"] - }, - scope=VariableScope.EXECUTION, - var_type=VariableType.MODEL_REFERENCE - ) - - self.execution_manager.complete_step("model_selection", success=True) - self.log_result("Model Selection in Execution Context", True, - f"Selected {provider_name}:{model_name}") - - # Step 2: Model execution within execution context - self.execution_manager.start_step("model_execution", "Execute model task") - - try: - # Test actual model execution - result = await model.generate("Hello", max_tokens=5) - - # Store result in execution context - self.execution_manager.variable_manager.set_variable( - "execution_result", - { - "content": result, - "model": model_name, - "timestamp": time.time() - }, - scope=VariableScope.EXECUTION, - var_type=VariableType.TASK_RESULT - ) - - self.execution_manager.complete_step("model_execution", success=True) - self.log_result("Model Execution in Context", True, - f"Generated: {result[:50]}...") - - # Step 3: Variable integration - stored_model = self.execution_manager.variable_manager.get_variable("selected_model") - stored_result = self.execution_manager.variable_manager.get_variable("execution_result") - - variables_ok = (stored_model is not None and stored_result is not None) - self.log_result("Variable Integration", variables_ok, - f"Stored model and result successfully") - - return True - - except Exception as e: - self.execution_manager.complete_step("model_execution", success=False) - self.log_result("Model Execution in Context", False, error=str(e)) - return False - else: - self.log_result("Model Selection in Execution Context", False, - error="No working models available") - return False - - except Exception as e: - self.log_result("Execution Engine Integration", False, error=str(e)) - return False - - async def validate_pipeline_execution(self): - """Validate complete pipeline execution with models.""" - print("\n🚀 VALIDATING PIPELINE EXECUTION") - print("-" * 40) - - if not self.working_models: - self.log_result("Pipeline Execution", False, error="No working models available") - return False - - try: - # Create a simple control system for testing - from orchestrator.core.control_system import ControlSystem - from orchestrator.core.task import Task - - class TestModelControlSystem(ControlSystem): - def __init__(self, model_registry): - config = {"capabilities": {"model_integration": True}} - super().__init__(name="test-model-control", config=config) - self.model_registry = model_registry - self._results = {} - - async def execute_task(self, task: Task, context: dict = None): - if task.action == "generate_text": - return await self._generate_text(task) - return {"status": "completed", "result": f"Executed {task.action}"} - - async def _generate_text(self, task): - prompt = task.parameters.get("prompt", "Hello") - - # Use first working model - if self.parent.working_models: - _, model_name, model = self.parent.working_models[0] - - try: - result = await model.generate(prompt, max_tokens=20) - return { - "status": "completed", - "content": result, - "model": model_name - } - except Exception as e: - return {"status": "failed", "error": str(e)} - - return {"status": "failed", "error": "No working models"} - - async def execute_pipeline(self, pipeline, context=None): - raise NotImplementedError("Use orchestrator") - - def get_capabilities(self): - return self.config.get("capabilities", {}) - - async def health_check(self): - return {"status": "healthy"} - - # Set up test control system - control_system = TestModelControlSystem(self.registry) - control_system.parent = self # Give access to working_models - orchestrator = Orchestrator(control_system=control_system) - - # Test simple pipeline - pipeline_yaml = """ -name: "model_integration_test" -description: "Test model integration in pipeline" - -steps: - - id: generate - action: generate_text - parameters: - prompt: "What is AI?" -""" - - print("Executing test pipeline...") - results = await orchestrator.execute_yaml(pipeline_yaml, context={}) - - if "generate" in results: - result = results["generate"] - if result["success"] == True: - self.log_result("Pipeline Execution", True, - f"Generated content: {result['content'][:50]}...") - return True - else: - self.log_result("Pipeline Execution", False, - f"Task failed: {result.get('error', 'Unknown error')}") - return False - else: - self.log_result("Pipeline Execution", False, - "No results returned from pipeline") - return False - - except Exception as e: - self.log_result("Pipeline Execution", False, error=str(e)) - traceback.print_exc() - return False - - async def validate_performance_features(self): - """Validate performance optimization features.""" - print("\n⚡ VALIDATING PERFORMANCE FEATURES") - print("-" * 40) - - try: - # Test caching - from orchestrator.models.optimization.caching import ModelCache, CacheConfig - - cache = ModelCache(CacheConfig(max_size=10, ttl_seconds=300)) - - # Test cache operations - cache.set("test_key", {"content": "test_content"}) - cached_result = cache.get("test_key") - - cache_works = cached_result is not None and cached_result["content"] == "test_content" - self.log_result("Model Caching", cache_works, "Cache set/get operations working") - - # Test connection pooling - from orchestrator.models.optimization.pooling import ConnectionPool, PoolConfig - - pool = ConnectionPool("test_provider", PoolConfig(min_connections=1, max_connections=3)) - - # Basic pool operations - conn = await pool.get_connection() - await pool.return_connection(conn) - - self.log_result("Connection Pooling", True, "Pool operations working") - - return True - - except Exception as e: - self.log_result("Performance Features", False, error=str(e)) - return False - - async def run_validation(self): - """Run complete validation suite.""" - print("🔗 MULTI-MODEL INTEGRATION VALIDATION") - print("=" * 60) - print("Validating Issue #311 Stream C: Integration & Testing") - print() - - # Run all validation steps - validations = [ - ("Model Registry", self.validate_model_registry()), - ("Model Discovery", self.validate_model_discovery()), - ("Model Selection", self.validate_model_selection()), - ("Execution Integration", self.validate_execution_integration()), - ("Pipeline Execution", self.validate_pipeline_execution()), - ("Performance Features", self.validate_performance_features()) - ] - - passed_count = 0 - total_count = len(validations) - - for validation_name, validation_coro in validations: - try: - success = await validation_coro - if success: - passed_count += 1 - except Exception as e: - print(f"❌ {validation_name} validation failed with exception: {e}") - traceback.print_exc() - - # Final summary - print(f"\n{'=' * 60}") - print("📊 VALIDATION SUMMARY") - print("=" * 60) - - success_rate = passed_count / total_count - - for result in self.results: - status = "✅" if result["success"] else "❌" - print(f"{status} {result['test']}") - - print(f"\n📈 Overall Success Rate: {passed_count}/{total_count} ({success_rate*100:.1f}%)") - - if success_rate >= 0.8: - print("\n🎉 INTEGRATION VALIDATION PASSED!") - print("✅ Multi-model system successfully integrated with execution engine") - print("✅ Pipeline execution working with model operations") - print("✅ Performance optimizations functional") - print("✅ Issue #311 Stream C objectives completed") - return True - else: - print("\n⚠️ INTEGRATION VALIDATION NEEDS ATTENTION") - print(f"❌ Only {success_rate*100:.1f}% of validations passed") - print("🔧 Review failed validations and fix issues") - return False - - -async def main(): - """Main validation entry point.""" - validator = IntegrationValidator() - success = await validator.run_validation() - return success - - -if __name__ == "__main__": - try: - success = asyncio.run(main()) - sys.exit(0 if success else 1) - except KeyboardInterrupt: - print("\n🛑 Validation interrupted by user") - sys.exit(1) - except Exception as e: - print(f"\n💥 Validation failed with unexpected error: {e}") - traceback.print_exc() - sys.exit(1) \ No newline at end of file diff --git a/tests/orchestrator/api/test_errors.py b/tests/orchestrator/api/test_errors.py deleted file mode 100644 index c1bb72a9..00000000 --- a/tests/orchestrator/api/test_errors.py +++ /dev/null @@ -1,535 +0,0 @@ -""" -Comprehensive tests for API error handling system. - -Tests all error classes, recovery mechanisms, error handling integration, -and error context management for the orchestrator API framework. -""" - -import pytest -import uuid -from datetime import datetime -from unittest.mock import Mock, patch - -from orchestrator.api.errors import ( - OrchestratorAPIError, - PipelineCompilationError, - YAMLValidationError, - TemplateProcessingError, - PipelineExecutionError, - ExecutionTimeoutError, - StepExecutionError, - APIConfigurationError, - ModelRegistryError, - ResourceError, - NetworkError, - UserInputError, - APIErrorHandler, - APIErrorCategory, - APIErrorContext, - RecoveryGuidance, - create_api_error_handler, - handle_api_exception, -) -from orchestrator.execution import ( - ErrorSeverity, - RecoveryStrategy, - ErrorCategory, - RecoveryManager, -) - - -class TestAPIErrorContext: - """Test API error context functionality.""" - - def test_default_context_creation(self): - """Test creating default error context.""" - context = APIErrorContext() - - assert context.error_id is not None - assert len(context.error_id) == 8 # UUID short form - assert isinstance(context.timestamp, datetime) - assert context.operation is None - assert context.pipeline_id is None - assert context.metadata == {} - assert context.related_errors == [] - - def test_context_with_values(self): - """Test creating context with specific values.""" - context = APIErrorContext( - operation="test_operation", - pipeline_id="test_pipeline", - execution_id="test_execution", - step_name="test_step", - metadata={"key": "value"} - ) - - assert context.operation == "test_operation" - assert context.pipeline_id == "test_pipeline" - assert context.execution_id == "test_execution" - assert context.step_name == "test_step" - assert context.metadata == {"key": "value"} - - def test_context_to_dict(self): - """Test converting context to dictionary.""" - context = APIErrorContext( - operation="test_op", - pipeline_id="test_pipeline", - metadata={"test": "data"} - ) - - result = context.to_dict() - - assert result["error_id"] == context.error_id - assert result["operation"] == "test_op" - assert result["pipeline_id"] == "test_pipeline" - assert result["metadata"] == {"test": "data"} - assert "timestamp" in result - - # Check None values are excluded - assert "execution_id" not in result - - -class TestRecoveryGuidance: - """Test recovery guidance functionality.""" - - def test_recovery_guidance_creation(self): - """Test creating recovery guidance.""" - guidance = RecoveryGuidance( - strategy=RecoveryStrategy.RETRY, - automatic_recovery=True, - user_actions=["Check input", "Retry operation"], - recovery_steps=["1. Validate", "2. Retry"], - confidence_level=0.8 - ) - - assert guidance.strategy == RecoveryStrategy.RETRY - assert guidance.automatic_recovery is True - assert guidance.user_actions == ["Check input", "Retry operation"] - assert guidance.recovery_steps == ["1. Validate", "2. Retry"] - assert guidance.confidence_level == 0.8 - - def test_recovery_guidance_to_dict(self): - """Test converting recovery guidance to dictionary.""" - guidance = RecoveryGuidance( - strategy=RecoveryStrategy.RETRY_WITH_BACKOFF, - user_actions=["Action 1"], - system_actions=["System action"], - estimated_recovery_time=30 - ) - - result = guidance.to_dict() - - assert result["strategy"] == "retry_with_backoff" - assert result["user_actions"] == ["Action 1"] - assert result["system_actions"] == ["System action"] - assert result["estimated_recovery_time"] == 30 - - -class TestOrchestratorAPIError: - """Test base OrchestratorAPIError functionality.""" - - def test_basic_error_creation(self): - """Test creating basic API error.""" - error = OrchestratorAPIError( - message="Test error", - category=APIErrorCategory.VALIDATION, - severity=ErrorSeverity.MEDIUM - ) - - assert str(error) == "Test error" - assert error.message == "Test error" - assert error.category == APIErrorCategory.VALIDATION - assert error.severity == ErrorSeverity.MEDIUM - assert error.context is not None - assert isinstance(error.context, APIErrorContext) - - def test_error_with_context_and_recovery(self): - """Test error with custom context and recovery guidance.""" - context = APIErrorContext(operation="test_op") - guidance = RecoveryGuidance( - strategy=RecoveryStrategy.MANUAL_INTERVENTION, - user_actions=["Fix the issue"] - ) - - error = OrchestratorAPIError( - message="Test error with context", - category=APIErrorCategory.EXECUTION, - context=context, - recovery_guidance=guidance - ) - - assert error.context.operation == "test_op" - assert error.recovery_guidance.strategy == RecoveryStrategy.MANUAL_INTERVENTION - - def test_error_with_original_exception(self): - """Test error wrapping original exception.""" - original = ValueError("Original error") - - error = OrchestratorAPIError( - message="Wrapped error", - category=APIErrorCategory.USER_CONFIGURATION, - original_exception=original - ) - - assert error.original_exception == original - assert error.traceback_info is not None - - def test_error_to_error_info(self): - """Test converting API error to foundation ErrorInfo.""" - error = OrchestratorAPIError( - message="Test conversion", - category=APIErrorCategory.COMPILATION, - severity=ErrorSeverity.HIGH - ) - - error_info = error.to_error_info() - - assert error_info.message == "Test conversion" - assert error_info.category == ErrorCategory.VALIDATION # Mapped from COMPILATION - assert error_info.severity == ErrorSeverity.HIGH - assert error_info.error_id == error.context.error_id - - def test_error_to_dict(self): - """Test converting error to dictionary.""" - error = OrchestratorAPIError( - message="Dict test", - category=APIErrorCategory.NETWORK, - severity=ErrorSeverity.MEDIUM - ) - - result = error.to_dict() - - assert result["error_type"] == "OrchestratorAPIError" - assert result["message"] == "Dict test" - assert result["category"] == "network" - assert result["severity"] == "medium" - assert "context" in result - - -class TestSpecificErrorTypes: - """Test specific error type implementations.""" - - def test_pipeline_compilation_error(self): - """Test pipeline compilation error.""" - error = PipelineCompilationError( - message="Compilation failed", - yaml_content="steps:\n- invalid", - context_variables={"var1": "value1"}, - validation_errors=["Invalid step format"] - ) - - assert error.category == APIErrorCategory.COMPILATION - assert error.severity == ErrorSeverity.HIGH - assert error.context.operation == "pipeline_compilation" - assert "yaml_length" in error.context.metadata - assert "validation_errors" in error.context.metadata - assert error.recovery_guidance.strategy == RecoveryStrategy.MANUAL_INTERVENTION - - def test_yaml_validation_error(self): - """Test YAML validation error.""" - error = YAMLValidationError( - message="YAML syntax error", - yaml_line=5, - yaml_column=10 - ) - - assert error.context.operation == "yaml_validation" - assert error.context.metadata["yaml_line"] == 5 - assert error.context.metadata["yaml_column"] == 10 - assert "Fix YAML syntax errors" in error.recovery_guidance.user_actions - - def test_template_processing_error(self): - """Test template processing error.""" - error = TemplateProcessingError( - message="Missing variables", - template_variables=["var1", "var2"], - missing_variables=["var2"] - ) - - assert error.context.operation == "template_processing" - assert error.context.metadata["missing_variables"] == ["var2"] - assert any("missing variables" in action.lower() - for action in error.recovery_guidance.user_actions) - - def test_pipeline_execution_error(self): - """Test pipeline execution error.""" - error = PipelineExecutionError( - message="Execution failed", - pipeline_id="test_pipeline", - execution_id="test_execution", - failed_step="step1" - ) - - assert error.category == APIErrorCategory.EXECUTION - assert error.context.pipeline_id == "test_pipeline" - assert error.context.execution_id == "test_execution" - assert error.context.step_name == "step1" - assert error.recovery_guidance.automatic_recovery is True - - def test_execution_timeout_error(self): - """Test execution timeout error.""" - error = ExecutionTimeoutError( - message="Execution timed out", - timeout_seconds=300, - elapsed_seconds=350 - ) - - assert error.context.operation == "execution_timeout" - assert error.context.metadata["timeout_seconds"] == 300 - assert error.context.metadata["elapsed_seconds"] == 350 - assert error.recovery_guidance.strategy == RecoveryStrategy.RETRY_WITH_BACKOFF - - def test_step_execution_error(self): - """Test step execution error.""" - error = StepExecutionError( - message="Step failed", - step_id="step_123", - step_type="text_generation", - step_config={"model": "test"} - ) - - assert error.context.step_id == "step_123" - assert error.context.metadata["step_type"] == "text_generation" - assert error.context.metadata["step_config"] == {"model": "test"} - - def test_api_configuration_error(self): - """Test API configuration error.""" - error = APIConfigurationError( - message="Invalid config", - config_key="model_registry", - config_value="invalid_value" - ) - - assert error.category == APIErrorCategory.CONFIGURATION - assert error.context.metadata["config_key"] == "model_registry" - assert error.context.metadata["config_value"] == "invalid_value" - - def test_model_registry_error(self): - """Test model registry error.""" - error = ModelRegistryError( - message="Model not found", - model_name="test_model", - registry_type="local" - ) - - assert error.context.metadata["model_name"] == "test_model" - assert error.context.metadata["registry_type"] == "local" - - def test_resource_error(self): - """Test resource error.""" - error = ResourceError( - message="Resource unavailable", - resource_type="memory", - resource_id="mem_pool_1" - ) - - assert error.category == APIErrorCategory.RESOURCE_MANAGEMENT - assert error.severity == ErrorSeverity.MEDIUM - assert error.recovery_guidance.automatic_recovery is True - - def test_network_error(self): - """Test network error.""" - error = NetworkError( - message="Connection failed", - endpoint="https://api.example.com", - status_code=503 - ) - - assert error.category == APIErrorCategory.NETWORK - assert error.context.metadata["endpoint"] == "https://api.example.com" - assert error.context.metadata["status_code"] == 503 - assert error.recovery_guidance.estimated_recovery_time == 30 - - def test_user_input_error(self): - """Test user input error.""" - error = UserInputError( - message="Invalid input", - input_field="yaml_content", - expected_type="string", - provided_value=None - ) - - assert error.category == APIErrorCategory.INPUT_VALIDATION - assert error.severity == ErrorSeverity.LOW - assert error.context.metadata["input_field"] == "yaml_content" - assert error.context.metadata["expected_type"] == "string" - - -class TestAPIErrorHandler: - """Test API error handler functionality.""" - - def test_error_handler_creation(self): - """Test creating error handler.""" - handler = APIErrorHandler() - - assert handler.recovery_manager is None - assert len(handler._error_handlers) > 0 # Default handlers registered - assert handler._error_history == [] - - def test_error_handler_with_recovery_manager(self): - """Test creating error handler with recovery manager.""" - recovery_manager = Mock(spec=RecoveryManager) - handler = APIErrorHandler(recovery_manager=recovery_manager) - - assert handler.recovery_manager == recovery_manager - - def test_handle_value_error(self): - """Test handling ValueError.""" - handler = APIErrorHandler() - original_error = ValueError("Invalid value provided") - - api_error = handler.handle_error(original_error, operation="test_op") - - assert isinstance(api_error, UserInputError) - assert api_error.original_exception == original_error - assert api_error.context.operation == "test_op" - assert "Invalid input value" in api_error.message - - def test_handle_file_not_found_error(self): - """Test handling FileNotFoundError.""" - handler = APIErrorHandler() - original_error = FileNotFoundError("File not found: test.yaml") - - api_error = handler.handle_error(original_error) - - assert isinstance(api_error, APIConfigurationError) - assert "Required file not found" in api_error.message - - def test_handle_timeout_error(self): - """Test handling TimeoutError.""" - handler = APIErrorHandler() - original_error = TimeoutError("Operation timed out") - - api_error = handler.handle_error(original_error) - - assert isinstance(api_error, ExecutionTimeoutError) - assert "Operation timed out" in api_error.message - - def test_handle_connection_error(self): - """Test handling ConnectionError.""" - handler = APIErrorHandler() - original_error = ConnectionError("Connection refused") - - api_error = handler.handle_error(original_error) - - assert isinstance(api_error, NetworkError) - assert "Network error" in api_error.message - - def test_handle_unknown_error(self): - """Test handling unknown error type.""" - handler = APIErrorHandler() - original_error = RuntimeError("Unknown error") - - api_error = handler.handle_error(original_error) - - assert isinstance(api_error, OrchestratorAPIError) - assert "Unexpected error" in api_error.message - assert api_error.category == APIErrorCategory.VALIDATION - - def test_error_history_tracking(self): - """Test error history tracking.""" - handler = APIErrorHandler() - - error1 = ValueError("Error 1") - error2 = TypeError("Error 2") - - api_error1 = handler.handle_error(error1) - api_error2 = handler.handle_error(error2) - - history = handler.get_error_history() - assert len(history) == 2 - assert history[0] == api_error1 - assert history[1] == api_error2 - - handler.clear_error_history() - assert handler.get_error_history() == [] - - @patch('orchestrator.api.errors.logger') - def test_error_logging(self, mock_logger): - """Test error logging functionality.""" - handler = APIErrorHandler() - original_error = ValueError("Test error for logging") - - handler.handle_error(original_error, operation="test_logging") - - # Should log the error - mock_logger.error.assert_called() - - # Check log call - log_call = mock_logger.error.call_args - assert "UserInputError" in log_call[0][0] - assert "Invalid input value" in log_call[0][0] - - -class TestConvenienceFunctions: - """Test convenience functions.""" - - def test_create_api_error_handler(self): - """Test creating error handler via convenience function.""" - handler = create_api_error_handler() - - assert isinstance(handler, APIErrorHandler) - assert handler.recovery_manager is None - - def test_create_api_error_handler_with_recovery(self): - """Test creating error handler with recovery manager.""" - recovery_manager = Mock(spec=RecoveryManager) - handler = create_api_error_handler(recovery_manager=recovery_manager) - - assert handler.recovery_manager == recovery_manager - - def test_handle_api_exception(self): - """Test handle_api_exception convenience function.""" - original_error = ValueError("Test exception") - - api_error = handle_api_exception( - original_error, - operation="test_operation", - context={"test": "data"} - ) - - assert isinstance(api_error, UserInputError) - assert api_error.context.operation == "test_operation" - assert api_error.context.metadata["test"] == "data" - - -class TestErrorIntegration: - """Test error handling integration with foundation components.""" - - def test_error_info_conversion(self): - """Test conversion to foundation ErrorInfo.""" - error = PipelineExecutionError( - message="Integration test", - pipeline_id="test_pipeline" - ) - - error_info = error.to_error_info() - - assert error_info.message == "Integration test" - assert error_info.category == ErrorCategory.EXECUTION - assert error_info.severity == ErrorSeverity.HIGH - assert error_info.error_id == error.context.error_id - - def test_category_mapping(self): - """Test API category to foundation category mapping.""" - test_cases = [ - (APIErrorCategory.COMPILATION, ErrorCategory.VALIDATION), - (APIErrorCategory.EXECUTION, ErrorCategory.EXECUTION), - (APIErrorCategory.NETWORK, ErrorCategory.NETWORK), - (APIErrorCategory.AUTHENTICATION, ErrorCategory.AUTHENTICATION), - (APIErrorCategory.RESOURCE_MANAGEMENT, ErrorCategory.RESOURCE), - (APIErrorCategory.DEPENDENCY_RESOLUTION, ErrorCategory.DEPENDENCY), - ] - - for api_category, expected_foundation_category in test_cases: - error = OrchestratorAPIError( - message="Test mapping", - category=api_category - ) - - error_info = error.to_error_info() - assert error_info.category == expected_foundation_category - - -if __name__ == "__main__": - pytest.main([__file__]) \ No newline at end of file diff --git a/tests/orchestrator/api/test_types.py b/tests/orchestrator/api/test_types.py deleted file mode 100644 index 9c16e64c..00000000 --- a/tests/orchestrator/api/test_types.py +++ /dev/null @@ -1,813 +0,0 @@ -""" -Comprehensive tests for API type definitions and validation. - -Tests all type definitions, serialization, validation, and integration -for the orchestrator API framework type system. -""" - -import pytest -import uuid -from datetime import datetime, timedelta -from pathlib import Path -from typing import Dict, Any, Optional -from unittest.mock import Mock - -from orchestrator.api.types import ( - # Enums - APIOperation, - ValidationLevel, - CompilationMode, - ExecutionMode, - - # Request types - CompilationRequest, - ExecutionRequest, - - # Response types - APIResponse, - CompilationResult, - ExecutionResult, - ExecutionStatusInfo, - ProgressUpdate, - - # Configuration types - APIConfiguration, - - # Protocol types - PipelineCompilerProtocol, - ExecutionManagerProtocol, - ProgressMonitorProtocol, - - # TypedDict types - PipelineCompilationDict, - PipelineExecutionDict, - ExecutionStatusDict, - ValidationResult, - ResourceUsage, - StepSummary, - - # Documentation types - APIEndpoint, - API_DOCUMENTATION, - - # Response type aliases - PipelineCompilationResponse, - PipelineExecutionResponse, - ExecutionStatusResponse, - ProgressUpdateResponse, -) -from orchestrator.execution import ( - ExecutionStatus, - ExecutionMetrics, - ProgressEventType, - StepStatus, -) -from orchestrator.core.pipeline import Pipeline - - -class TestEnums: - """Test enum type definitions.""" - - def test_api_operation_enum(self): - """Test APIOperation enum values.""" - assert APIOperation.COMPILE_PIPELINE.value == "compile_pipeline" - assert APIOperation.EXECUTE_PIPELINE.value == "execute_pipeline" - assert APIOperation.GET_STATUS.value == "get_execution_status" - assert APIOperation.MONITOR_EXECUTION.value == "monitor_execution" - - # Ensure all expected operations are present - expected_ops = [ - "compile_pipeline", "execute_pipeline", "validate_yaml", - "get_execution_status", "stop_execution", "list_active_executions", - "cleanup_execution", "get_compilation_report", "get_template_variables", - "monitor_execution", "control_execution" - ] - - actual_ops = [op.value for op in APIOperation] - for expected in expected_ops: - assert expected in actual_ops - - def test_validation_level_enum(self): - """Test ValidationLevel enum values.""" - assert ValidationLevel.STRICT.value == "strict" - assert ValidationLevel.PERMISSIVE.value == "permissive" - assert ValidationLevel.DEVELOPMENT.value == "development" - assert ValidationLevel.DISABLED.value == "disabled" - - def test_compilation_mode_enum(self): - """Test CompilationMode enum values.""" - assert CompilationMode.STANDARD.value == "standard" - assert CompilationMode.FAST.value == "fast" - assert CompilationMode.SAFE.value == "safe" - assert CompilationMode.DEBUG.value == "debug" - - def test_execution_mode_enum(self): - """Test ExecutionMode enum values.""" - assert ExecutionMode.NORMAL.value == "normal" - assert ExecutionMode.DRY_RUN.value == "dry_run" - assert ExecutionMode.STEP_BY_STEP.value == "step_by_step" - assert ExecutionMode.PARALLEL.value == "parallel" - assert ExecutionMode.RECOVERY.value == "recovery" - - -class TestRequestTypes: - """Test request type definitions.""" - - def test_compilation_request_creation(self): - """Test creating compilation request.""" - request = CompilationRequest( - yaml_content="steps:\n - name: test", - context={"var": "value"}, - validation_level=ValidationLevel.STRICT - ) - - assert request.yaml_content == "steps:\n - name: test" - assert request.context == {"var": "value"} - assert request.validation_level == ValidationLevel.STRICT - assert request.resolve_ambiguities is True # Default - assert request.validate is True # Default - assert isinstance(request.request_id, str) - assert isinstance(request.timestamp, datetime) - - def test_compilation_request_defaults(self): - """Test compilation request default values.""" - request = CompilationRequest(yaml_content="test content") - - assert request.context is None - assert request.resolve_ambiguities is True - assert request.validate is True - assert request.validation_level == ValidationLevel.STRICT - assert request.compilation_mode == CompilationMode.STANDARD - assert request.enable_preprocessing is True - assert request.template_strict_mode is True - assert request.cache_result is True - assert request.include_metadata is False - assert request.debug_mode is False - assert request.user_id is None - - def test_compilation_request_with_path(self): - """Test compilation request with file path.""" - path = Path("/test/pipeline.yaml") - request = CompilationRequest(yaml_content=path) - - assert request.yaml_content == path - - def test_compilation_request_to_dict(self): - """Test converting compilation request to dictionary.""" - path = Path("/test/pipeline.yaml") - request = CompilationRequest( - yaml_content=path, - context={"test": "value"}, - validation_level=ValidationLevel.DEVELOPMENT, - user_id="test_user" - ) - - result = request.to_dict() - - assert result["yaml_content"] == str(path) - assert result["context"] == {"test": "value"} - assert result["validation_level"] == "development" - assert result["user_id"] == "test_user" - assert result["request_id"] == request.request_id - assert "timestamp" in result - - def test_execution_request_creation(self): - """Test creating execution request.""" - mock_pipeline = Mock(spec=Pipeline) - mock_pipeline.id = "test_pipeline" - - request = ExecutionRequest( - pipeline=mock_pipeline, - context={"input": "data"}, - execution_mode=ExecutionMode.NORMAL, - timeout=3600 - ) - - assert request.pipeline == mock_pipeline - assert request.context == {"input": "data"} - assert request.execution_mode == ExecutionMode.NORMAL - assert request.timeout == 3600 - assert isinstance(request.request_id, str) - assert isinstance(request.timestamp, datetime) - - def test_execution_request_defaults(self): - """Test execution request default values.""" - request = ExecutionRequest(pipeline="pipeline_content") - - assert request.pipeline == "pipeline_content" - assert request.context is None - assert request.execution_id is None - assert request.execution_mode == ExecutionMode.NORMAL - assert request.timeout is None - assert request.max_retries == 3 - assert request.enable_recovery is True - assert request.enable_checkpointing is True - assert request.enable_monitoring is True - assert request.progress_callback is None - assert request.status_callback is None - assert request.resource_limits is None - assert request.environment_vars is None - assert request.debug_mode is False - assert request.user_id is None - - def test_execution_request_to_dict(self): - """Test converting execution request to dictionary.""" - mock_pipeline = Mock(spec=Pipeline) - mock_pipeline.id = "test_pipeline_123" - - request = ExecutionRequest( - pipeline=mock_pipeline, - execution_id="exec_123", - timeout=1800, - debug_mode=True - ) - - result = request.to_dict() - - assert result["pipeline"] == "test_pipeline_123" - assert result["execution_id"] == "exec_123" - assert result["timeout"] == 1800 - assert result["debug_mode"] is True - assert result["execution_mode"] == "normal" - - -class TestResponseTypes: - """Test response type definitions.""" - - def test_api_response_success(self): - """Test successful API response.""" - response = APIResponse[str]( - success=True, - data="test_data", - request_id="req_123", - duration_ms=150.5 - ) - - assert response.success is True - assert response.data == "test_data" - assert response.request_id == "req_123" - assert response.duration_ms == 150.5 - assert isinstance(response.timestamp, datetime) - assert response.error_code is None - assert response.error_message is None - assert response.warnings == [] - assert response.metadata == {} - - def test_api_response_error(self): - """Test error API response.""" - response = APIResponse[None]( - success=False, - error_code="VALIDATION_ERROR", - error_message="Invalid input provided", - error_details={"field": "yaml_content", "issue": "malformed"}, - warnings=["Deprecated feature used"] - ) - - assert response.success is False - assert response.data is None - assert response.error_code == "VALIDATION_ERROR" - assert response.error_message == "Invalid input provided" - assert response.error_details == {"field": "yaml_content", "issue": "malformed"} - assert response.warnings == ["Deprecated feature used"] - - def test_api_response_to_dict(self): - """Test converting API response to dictionary.""" - # Test with data object that has to_dict method - mock_data = Mock() - mock_data.to_dict.return_value = {"mock": "data"} - - response = APIResponse[Any]( - success=True, - data=mock_data, - request_id="req_456", - warnings=["Warning message"] - ) - - result = response.to_dict() - - assert result["success"] is True - assert result["data"] == {"mock": "data"} - assert result["request_id"] == "req_456" - assert result["warnings"] == ["Warning message"] - assert "timestamp" in result - - # Test with simple data - simple_response = APIResponse[str]( - success=True, - data="simple_string" - ) - - simple_result = simple_response.to_dict() - assert simple_result["data"] == "simple_string" - - def test_compilation_result_creation(self): - """Test creating compilation result.""" - mock_pipeline = Mock(spec=Pipeline) - mock_pipeline.id = "compiled_pipeline" - - result = CompilationResult( - pipeline=mock_pipeline, - compilation_time=timedelta(milliseconds=250), - validation_passed=True, - template_variables=["var1", "var2"], - validation_warnings=["Minor issue"] - ) - - assert result.pipeline == mock_pipeline - assert result.compilation_time == timedelta(milliseconds=250) - assert result.validation_passed is True - assert result.template_variables == ["var1", "var2"] - assert result.validation_warnings == ["Minor issue"] - assert isinstance(result.compilation_id, str) - assert isinstance(result.compiled_at, datetime) - assert result.compiler_version == "2.0.0" - - def test_compilation_result_to_dict(self): - """Test converting compilation result to dictionary.""" - mock_pipeline = Mock(spec=Pipeline) - mock_pipeline.id = "test_pipeline" - mock_pipeline.name = "Test Pipeline" - - result = CompilationResult( - pipeline=mock_pipeline, - compilation_time=timedelta(seconds=1.5), - validation_report={"status": "passed"}, - validation_errors=["Error 1"] - ) - - result_dict = result.to_dict() - - assert result_dict["pipeline_id"] == "test_pipeline" - assert result_dict["pipeline_name"] == "Test Pipeline" - assert result_dict["compilation_time_ms"] == 1500.0 - assert result_dict["validation_report"] == {"status": "passed"} - assert result_dict["validation_errors"] == ["Error 1"] - assert result_dict["compilation_id"] == result.compilation_id - assert "compiled_at" in result_dict - - def test_execution_result_creation(self): - """Test creating execution result.""" - start_time = datetime.now() - - result = ExecutionResult( - execution_id="exec_789", - pipeline_id="pipeline_456", - status=ExecutionStatus.RUNNING, - started_at=start_time, - total_steps=5, - monitoring_enabled=True - ) - - assert result.execution_id == "exec_789" - assert result.pipeline_id == "pipeline_456" - assert result.status == ExecutionStatus.RUNNING - assert result.started_at == start_time - assert result.total_steps == 5 - assert result.monitoring_enabled is True - assert result.execution_mode == ExecutionMode.NORMAL # Default - - def test_execution_result_to_dict(self): - """Test converting execution result to dictionary.""" - start_time = datetime.now() - estimated_duration = timedelta(minutes=5) - - result = ExecutionResult( - execution_id="exec_xyz", - pipeline_id="pipeline_abc", - status=ExecutionStatus.PENDING, - started_at=start_time, - estimated_duration=estimated_duration, - total_steps=10, - progress_url="/progress", - execution_mode=ExecutionMode.DEBUG - ) - - result_dict = result.to_dict() - - assert result_dict["execution_id"] == "exec_xyz" - assert result_dict["pipeline_id"] == "pipeline_abc" - assert result_dict["status"] == "pending" - assert result_dict["started_at"] == start_time.isoformat() - assert result_dict["estimated_duration_seconds"] == 300.0 - assert result_dict["total_steps"] == 10 - assert result_dict["progress_url"] == "/progress" - assert result_dict["execution_mode"] == "debug" - - -class TestStatusTypes: - """Test status and progress type definitions.""" - - def test_execution_status_info_creation(self): - """Test creating execution status info.""" - start_time = datetime.now() - update_time = datetime.now() - - status = ExecutionStatusInfo( - execution_id="exec_status_test", - pipeline_id="pipeline_status", - status=ExecutionStatus.RUNNING, - started_at=start_time, - updated_at=update_time, - current_step="step_2", - steps_completed=2, - steps_total=5, - progress_percentage=40.0 - ) - - assert status.execution_id == "exec_status_test" - assert status.pipeline_id == "pipeline_status" - assert status.status == ExecutionStatus.RUNNING - assert status.started_at == start_time - assert status.updated_at == update_time - assert status.current_step == "step_2" - assert status.steps_completed == 2 - assert status.steps_total == 5 - assert status.progress_percentage == 40.0 - assert status.completed_at is None - assert status.duration is None - - def test_execution_status_info_to_dict(self): - """Test converting execution status info to dictionary.""" - start_time = datetime.now() - update_time = datetime.now() - completed_time = datetime.now() - duration = timedelta(minutes=2) - - mock_metrics = Mock(spec=ExecutionMetrics) - mock_metrics.__dict__ = {"steps_completed": 3} - - status = ExecutionStatusInfo( - execution_id="exec_dict_test", - pipeline_id="pipeline_dict", - status=ExecutionStatus.COMPLETED, - started_at=start_time, - updated_at=update_time, - completed_at=completed_time, - duration=duration, - metrics=mock_metrics, - step_statuses={"step1": StepStatus.COMPLETED, "step2": StepStatus.RUNNING}, - error_count=1, - variables={"var1": "value1"} - ) - - result = status.to_dict() - - assert result["execution_id"] == "exec_dict_test" - assert result["success"] == True - assert result["started_at"] == start_time.isoformat() - assert result["completed_at"] == completed_time.isoformat() - assert result["duration_seconds"] == 120.0 - assert result["metrics"] == {"steps_completed": 3} - assert result["step_statuses"] == {"step1": "completed", "step2": "running"} - assert result["error_count"] == 1 - assert result["variables"] == {"var1": "value1"} - - def test_progress_update_creation(self): - """Test creating progress update.""" - timestamp = datetime.now() - - update = ProgressUpdate( - execution_id="exec_progress", - timestamp=timestamp, - event_type=ProgressEventType.STEP_COMPLETED, - step_id="step_123", - step_name="Test Step", - message="Step completed successfully", - step_progress=100.0, - overall_progress=60.0, - steps_completed=3, - steps_total=5 - ) - - assert update.execution_id == "exec_progress" - assert update.timestamp == timestamp - assert update.event_type == ProgressEventType.STEP_COMPLETED - assert update.step_id == "step_123" - assert update.step_name == "Test Step" - assert update.message == "Step completed successfully" - assert update.step_progress == 100.0 - assert update.overall_progress == 60.0 - assert update.steps_completed == 3 - assert update.steps_total == 5 - - def test_progress_update_to_dict(self): - """Test converting progress update to dictionary.""" - timestamp = datetime.now() - - update = ProgressUpdate( - execution_id="exec_update_dict", - timestamp=timestamp, - event_type=ProgressEventType.STEP_STARTED, - step_name="Processing Step", - data={"processed_items": 10}, - metadata={"source": "executor"} - ) - - result = update.to_dict() - - assert result["execution_id"] == "exec_update_dict" - assert result["timestamp"] == timestamp.isoformat() - assert result["event_type"] == "step_started" - assert result["step_name"] == "Processing Step" - assert result["data"] == {"processed_items": 10} - assert result["metadata"] == {"source": "executor"} - - -class TestConfigurationTypes: - """Test configuration type definitions.""" - - def test_api_configuration_defaults(self): - """Test API configuration default values.""" - config = APIConfiguration() - - assert config.model_registry_config is None - assert config.auto_model_selection is True - assert config.default_validation_level == ValidationLevel.STRICT - assert config.enable_validation_caching is True - assert config.validation_timeout == 30 - assert config.default_execution_timeout == 3600 - assert config.max_concurrent_executions == 10 - assert config.enable_execution_recovery is True - assert config.enable_execution_checkpointing is True - assert config.enable_compilation_caching is True - assert config.cache_size_limit == 1000 - assert config.memory_limit_mb is None - assert config.enable_detailed_monitoring is True - assert config.progress_update_interval == 5 - assert config.status_cleanup_interval == 3600 - assert config.log_level == "INFO" - assert config.log_format == "structured" - assert config.enable_audit_logging is True - assert config.enable_authentication is False - assert config.api_key_required is False - assert config.rate_limiting is None - - def test_api_configuration_custom(self): - """Test API configuration with custom values.""" - config = APIConfiguration( - default_validation_level=ValidationLevel.DEVELOPMENT, - max_concurrent_executions=5, - memory_limit_mb=2048, - log_level="DEBUG", - enable_authentication=True, - rate_limiting={"requests_per_minute": 100} - ) - - assert config.default_validation_level == ValidationLevel.DEVELOPMENT - assert config.max_concurrent_executions == 5 - assert config.memory_limit_mb == 2048 - assert config.log_level == "DEBUG" - assert config.enable_authentication is True - assert config.rate_limiting == {"requests_per_minute": 100} - - def test_api_configuration_to_dict(self): - """Test converting API configuration to dictionary.""" - config = APIConfiguration( - model_registry_config={"type": "local"}, - default_validation_level=ValidationLevel.PERMISSIVE, - memory_limit_mb=4096 - ) - - result = config.to_dict() - - assert result["model_registry_config"] == {"type": "local"} - assert result["default_validation_level"] == "permissive" - assert result["memory_limit_mb"] == 4096 - assert result["auto_model_selection"] is True # Default - - -class TestProtocolTypes: - """Test protocol type definitions.""" - - def test_pipeline_compiler_protocol(self): - """Test pipeline compiler protocol structure.""" - # Verify protocol methods exist - protocol_methods = dir(PipelineCompilerProtocol) - - assert "compile" in protocol_methods - assert "validate_yaml" in protocol_methods - assert "get_template_variables" in protocol_methods - - def test_execution_manager_protocol(self): - """Test execution manager protocol structure.""" - protocol_methods = dir(ExecutionManagerProtocol) - - assert "get_execution_status" in protocol_methods - assert "start_execution" in protocol_methods - assert "complete_execution" in protocol_methods - assert "cleanup" in protocol_methods - - def test_progress_monitor_protocol(self): - """Test progress monitor protocol structure.""" - protocol_methods = dir(ProgressMonitorProtocol) - - assert "start_monitoring" in protocol_methods - assert "get_progress_updates" in protocol_methods - assert "stop_monitoring" in protocol_methods - - -class TestTypedDictTypes: - """Test TypedDict type definitions.""" - - def test_pipeline_compilation_dict_structure(self): - """Test PipelineCompilationDict structure.""" - # Create a valid compilation dict - compilation_dict: PipelineCompilationDict = { - "yaml_content": "steps:\n - name: test", - "context": {"var": "value"}, - "resolve_ambiguities": True, - "validate": True, - "validation_level": "strict", - "compilation_mode": "standard", - "request_id": "req_123" - } - - assert compilation_dict["yaml_content"] == "steps:\n - name: test" - assert compilation_dict["context"] == {"var": "value"} - assert compilation_dict["validation_level"] == "strict" - - def test_execution_status_dict_structure(self): - """Test ExecutionStatusDict structure.""" - status_dict: ExecutionStatusDict = { - "execution_id": "exec_123", - "pipeline_id": "pipeline_456", - "status": "running", - "started_at": "2023-01-01T00:00:00", - "updated_at": "2023-01-01T00:05:00", - "progress_percentage": 50.0, - "steps_completed": 2, - "steps_total": 4, - "step_statuses": {"step1": "completed", "step2": "running"}, - "error_count": 0, - "variables": {"output": "result"} - } - - assert status_dict["execution_id"] == "exec_123" - assert status_dict["status"] == "running" - assert status_dict["progress_percentage"] == 50.0 - assert status_dict["variables"] == {"output": "result"} - - def test_validation_result_structure(self): - """Test ValidationResult structure.""" - validation_result: ValidationResult = { - "valid": True, - "errors": [], - "warnings": ["Minor issue"], - "info": ["Validation completed"] - } - - assert validation_result["valid"] is True - assert validation_result["errors"] == [] - assert validation_result["warnings"] == ["Minor issue"] - assert validation_result["info"] == ["Validation completed"] - - def test_resource_usage_structure(self): - """Test ResourceUsage structure.""" - resource_usage: ResourceUsage = { - "memory_mb": 512.5, - "cpu_percent": 25.0, - "disk_mb": 100.0, - "network_kb": 50.2, - "execution_time_seconds": 120.5 - } - - assert resource_usage["memory_mb"] == 512.5 - assert resource_usage["cpu_percent"] == 25.0 - assert resource_usage["execution_time_seconds"] == 120.5 - - def test_step_summary_structure(self): - """Test StepSummary structure.""" - step_summary: StepSummary = { - "step_id": "step_abc", - "step_name": "Process Data", - "step_type": "data_processor", - "status": "completed", - "progress": 100.0, - "duration_seconds": 45.2, - "error_message": None, - "resource_usage": { - "memory_mb": 256.0, - "cpu_percent": 15.0, - "disk_mb": 20.0, - "network_kb": 10.5, - "execution_time_seconds": 45.2 - } - } - - assert step_summary["step_id"] == "step_abc" - assert step_summary["step_name"] == "Process Data" - assert step_summary["status"] == "completed" - assert step_summary["progress"] == 100.0 - assert step_summary["error_message"] is None - - -class TestDocumentationTypes: - """Test documentation type definitions.""" - - def test_api_endpoint_creation(self): - """Test creating API endpoint documentation.""" - endpoint = APIEndpoint( - name="test_endpoint", - method="POST", - path="/api/test", - description="Test endpoint for validation", - request_type=CompilationRequest, - response_type=CompilationResult, - parameters=["param1: string", "param2: optional int"], - error_codes=["VALIDATION_ERROR", "TIMEOUT_ERROR"], - examples=[{"test": "example"}] - ) - - assert endpoint.name == "test_endpoint" - assert endpoint.method == "POST" - assert endpoint.path == "/api/test" - assert endpoint.description == "Test endpoint for validation" - assert endpoint.request_type == CompilationRequest - assert endpoint.response_type == CompilationResult - assert endpoint.parameters == ["param1: string", "param2: optional int"] - assert endpoint.error_codes == ["VALIDATION_ERROR", "TIMEOUT_ERROR"] - assert endpoint.examples == [{"test": "example"}] - - def test_api_endpoint_to_dict(self): - """Test converting API endpoint to dictionary.""" - endpoint = APIEndpoint( - name="dict_test", - method="GET", - path="/test/dict", - description="Dictionary conversion test", - request_schema={"field": "string"}, - response_example={"result": "success"} - ) - - result = endpoint.to_dict() - - assert result["name"] == "dict_test" - assert result["method"] == "GET" - assert result["path"] == "/test/dict" - assert result["description"] == "Dictionary conversion test" - assert result["request_schema"] == {"field": "string"} - assert result["response_example"] == {"result": "success"} - - def test_api_documentation_structure(self): - """Test API_DOCUMENTATION structure.""" - assert "title" in API_DOCUMENTATION - assert "version" in API_DOCUMENTATION - assert "description" in API_DOCUMENTATION - assert "endpoints" in API_DOCUMENTATION - - assert API_DOCUMENTATION["title"] == "Orchestrator API Framework" - assert API_DOCUMENTATION["version"] == "2.0.0" - assert isinstance(API_DOCUMENTATION["endpoints"], list) - assert len(API_DOCUMENTATION["endpoints"]) > 0 - - # Check first endpoint structure - first_endpoint = API_DOCUMENTATION["endpoints"][0] - assert isinstance(first_endpoint, APIEndpoint) - assert hasattr(first_endpoint, "name") - assert hasattr(first_endpoint, "method") - assert hasattr(first_endpoint, "path") - assert hasattr(first_endpoint, "description") - - -class TestResponseTypeAliases: - """Test response type aliases.""" - - def test_pipeline_compilation_response_alias(self): - """Test PipelineCompilationResponse type alias.""" - mock_pipeline = Mock(spec=Pipeline) - compilation_result = CompilationResult( - pipeline=mock_pipeline, - compilation_time=timedelta(seconds=1) - ) - - response: PipelineCompilationResponse = APIResponse[CompilationResult]( - success=True, - data=compilation_result - ) - - assert response.success is True - assert response.data == compilation_result - assert isinstance(response, APIResponse) - - def test_execution_status_response_alias(self): - """Test ExecutionStatusResponse type alias.""" - status_info = ExecutionStatusInfo( - execution_id="test_exec", - pipeline_id="test_pipeline", - status=ExecutionStatus.RUNNING, - started_at=datetime.now(), - updated_at=datetime.now() - ) - - response: ExecutionStatusResponse = APIResponse[ExecutionStatusInfo]( - success=True, - data=status_info - ) - - assert response.success is True - assert response.data == status_info - assert isinstance(response, APIResponse) - - -if __name__ == "__main__": - pytest.main([__file__]) \ No newline at end of file diff --git a/tests/orchestrator/models/optimization/test_caching.py b/tests/orchestrator/models/optimization/test_caching.py deleted file mode 100644 index 17ffc334..00000000 --- a/tests/orchestrator/models/optimization/test_caching.py +++ /dev/null @@ -1,415 +0,0 @@ -"""Tests for model response caching system.""" - -import asyncio -import pytest -import time -from unittest.mock import AsyncMock - -from orchestrator.models.optimization.caching import ( - ModelResponseCache, - CacheStats, - CacheEntry, -) -from orchestrator.models.selection.strategies import SelectionResult -from orchestrator.core.model import Model, ModelCapabilities, ModelCost - - -class MockModel(Model): - """Mock model for testing.""" - - def __init__(self, name: str, provider: str): - super().__init__( - name=name, - provider=provider, - capabilities=ModelCapabilities(), - cost=ModelCost(is_free=True), - ) - - async def generate(self, prompt: str, temperature: float = 0.7, max_tokens: int = None, **kwargs): - return f"Response from {self.name}" - - async def generate_structured(self, prompt: str, schema: dict, temperature: float = 0.7, **kwargs): - return {"result": f"Structured from {self.name}"} - - async def health_check(self) -> bool: - return True - - async def estimate_cost(self, prompt: str, max_tokens: int = None) -> float: - return 0.001 - - -class TestCacheEntry: - """Test CacheEntry functionality.""" - - def test_initialization(self): - """Test cache entry initialization.""" - entry = CacheEntry( - value="test_value", - timestamp=1234567890.0, - ttl=3600.0, - ) - - assert entry.value == "test_value" - assert entry.timestamp == 1234567890.0 - assert entry.ttl == 3600.0 - assert entry.access_count == 0 - - def test_expiration_check(self): - """Test TTL expiration checking.""" - # Create entry that should be expired - old_time = time.time() - 7200 # 2 hours ago - entry = CacheEntry( - value="test_value", - timestamp=old_time, - ttl=3600.0, # 1 hour TTL - ) - - assert entry.is_expired() is True - - # Create entry that should not be expired - recent_time = time.time() - 1800 # 30 minutes ago - entry_not_expired = CacheEntry( - value="test_value", - timestamp=recent_time, - ttl=3600.0, # 1 hour TTL - ) - - assert entry_not_expired.is_expired() is False - - def test_no_ttl_expiration(self): - """Test entries with no TTL never expire.""" - entry = CacheEntry( - value="test_value", - timestamp=time.time() - 86400, # 1 day ago - ttl=None, - ) - - assert entry.is_expired() is False - - def test_access_tracking(self): - """Test access count and timestamp tracking.""" - entry = CacheEntry(value="test_value", timestamp=time.time()) - - initial_last_accessed = entry.last_accessed - initial_count = entry.access_count - - # Wait a bit to ensure timestamp difference - time.sleep(0.01) - entry.access() - - assert entry.access_count == initial_count + 1 - assert entry.last_accessed > initial_last_accessed - - -class TestCacheStats: - """Test CacheStats functionality.""" - - def test_hit_rate_calculation(self): - """Test hit rate calculation.""" - stats = CacheStats(hits=8, misses=2) - assert stats.hit_rate == 0.8 - - # Test with no requests - empty_stats = CacheStats() - assert empty_stats.hit_rate == 0.0 - - def test_fill_rate_calculation(self): - """Test cache fill rate calculation.""" - stats = CacheStats(total_size=75, max_size=100) - assert stats.fill_rate == 0.75 - - # Test with max_size 0 - stats_no_max = CacheStats(total_size=50, max_size=0) - assert stats_no_max.fill_rate == 0.0 - - def test_to_dict(self): - """Test conversion to dictionary.""" - stats = CacheStats( - hits=10, - misses=5, - evictions=2, - total_size=50, - max_size=100, - ) - - result = stats.to_dict() - - expected_keys = ["hits", "misses", "evictions", "total_size", "max_size", "hit_rate", "fill_rate"] - for key in expected_keys: - assert key in result - - assert result["hit_rate"] == 10 / 15 # 10 hits out of 15 total - assert result["fill_rate"] == 0.5 # 50 out of 100 - - -class TestModelResponseCache: - """Test ModelResponseCache functionality.""" - - @pytest.fixture - def cache(self): - """Create cache instance for testing.""" - return ModelResponseCache( - max_size=10, - default_ttl=3600.0, # 1 hour - max_memory_mb=1, # 1MB limit - ) - - def test_initialization(self, cache): - """Test cache initialization.""" - assert cache.max_size == 10 - assert cache.default_ttl == 3600.0 - assert cache.max_memory_bytes == 1024 * 1024 - assert len(cache._cache) == 0 - - def test_cache_key_generation(self, cache): - """Test cache key generation.""" - key1 = cache.generate_cache_key( - prompt="Hello world", - temperature=0.7, - max_tokens=100, - ) - - key2 = cache.generate_cache_key( - prompt="Hello world", - temperature=0.7, - max_tokens=100, - ) - - key3 = cache.generate_cache_key( - prompt="Different prompt", - temperature=0.7, - max_tokens=100, - ) - - # Same parameters should generate same key - assert key1 == key2 - - # Different parameters should generate different key - assert key1 != key3 - - # Keys should be reasonable length (16 chars in current implementation) - assert len(key1) == 16 - - @pytest.mark.asyncio - async def test_basic_caching(self, cache): - """Test basic cache put/get operations.""" - key = "test_key" - value = "test_value" - - # Should be empty initially - result = await cache.get(key) - assert result is None - - # Put value in cache - await cache.put(key, value) - - # Should now retrieve value - result = await cache.get(key) - assert result == value - - # Check stats - stats = await cache.get_stats() - assert stats.hits == 1 - assert stats.misses == 1 - - @pytest.mark.asyncio - async def test_ttl_expiration(self): - """Test TTL-based expiration.""" - cache = ModelResponseCache(max_size=10, default_ttl=0.05) # 50ms TTL - - key = "test_key" - value = "test_value" - - await cache.put(key, value) - - # Should retrieve immediately - result = await cache.get(key) - assert result == value - - # Wait for expiration - await asyncio.sleep(0.1) - - # Should be expired now - result = await cache.get(key) - assert result is None - - @pytest.mark.asyncio - async def test_lru_eviction(self, cache): - """Test LRU eviction when cache is full.""" - # Fill cache to capacity - for i in range(cache.max_size): - await cache.put(f"key_{i}", f"value_{i}") - - # All keys should be retrievable - for i in range(cache.max_size): - result = await cache.get(f"key_{i}") - assert result == f"value_{i}" - - # Add one more item, should evict oldest - await cache.put("new_key", "new_value") - - # First key should be evicted - result = await cache.get("key_0") - assert result is None - - # New key should be available - result = await cache.get("new_key") - assert result == "new_value" - - # Check stats - stats = await cache.get_stats() - assert stats.evictions >= 1 - - @pytest.mark.asyncio - async def test_cache_invalidation(self, cache): - """Test cache invalidation.""" - # Add some items - await cache.put("key1", "value1") - await cache.put("key2", "value2") - await cache.put("test_key3", "value3") - - # Verify items are cached - assert await cache.get("key1") == "value1" - assert await cache.get("key2") == "value2" - assert await cache.get("test_key3") == "value3" - - # Pattern-based invalidation - count = await cache.invalidate("test_") - assert count == 1 - assert await cache.get("test_key3") is None - assert await cache.get("key1") == "value1" # Should still exist - - # Full invalidation - count = await cache.invalidate() - assert count >= 2 # At least key1 and key2 - assert await cache.get("key1") is None - assert await cache.get("key2") is None - - @pytest.mark.asyncio - async def test_selection_result_caching(self, cache): - """Test caching of model selection results.""" - model = MockModel("test_model", "test_provider") - selection_result = SelectionResult( - model=model, - provider="test_provider", - confidence_score=0.95, - selection_reason="Test selection", - ) - - requirements_key = "test_requirements" - - # Should be empty initially - result = await cache.get_cached_selection(requirements_key) - assert result is None - - # Cache selection result - await cache.cache_selection(requirements_key, selection_result) - - # Should retrieve cached result - result = await cache.get_cached_selection(requirements_key) - assert result is not None - assert result.model.name == "test_model" - assert result.confidence_score == 0.95 - - @pytest.mark.asyncio - async def test_memory_management(self): - """Test memory-based eviction.""" - # Create cache with very small memory limit - cache = ModelResponseCache(max_size=100, max_memory_mb=0.001) # ~1KB limit - - # Add large values that should trigger memory eviction - large_value = "x" * 512 # 512 bytes - - await cache.put("key1", large_value) - await cache.put("key2", large_value) - await cache.put("key3", large_value) # This should trigger evictions - - # First key might be evicted due to memory pressure - stats = await cache.get_stats() - assert stats.evictions > 0 - - @pytest.mark.asyncio - async def test_size_estimation(self, cache): - """Test size estimation for different value types.""" - # Test with different types - test_cases = [ - ("string", "hello world"), - ("int", 12345), - ("float", 123.45), - ("dict", {"key": "value", "number": 42}), - ("list", [1, 2, 3, "test"]), - ] - - for name, value in test_cases: - size = cache._estimate_size(value) - assert size > 0, f"Size estimation failed for {name}" - assert isinstance(size, int), f"Size should be integer for {name}" - - @pytest.mark.asyncio - async def test_cleanup_expired_entries(self): - """Test cleanup of expired entries.""" - cache = ModelResponseCache(max_size=10, default_ttl=0.05) # 50ms TTL - - # Add entries - await cache.put("key1", "value1") - await cache.put("key2", "value2") - - # Wait for expiration - await asyncio.sleep(0.1) - - # Add new entry, which should trigger cleanup - await cache.put("key3", "value3") - - # Expired entries should be gone - assert await cache.get("key1") is None - assert await cache.get("key2") is None - assert await cache.get("key3") == "value3" - - @pytest.mark.asyncio - async def test_cache_info(self, cache): - """Test cache information retrieval.""" - # Add some items - await cache.put("key1", "value1") - await cache.put("key2", "value2") - - info = cache.get_cache_info() - - required_keys = [ - "max_size", "current_size", "max_memory_mb", - "current_memory_mb", "default_ttl", "stats" - ] - - for key in required_keys: - assert key in info, f"Missing key: {key}" - - assert info["max_size"] == 10 - assert info["current_size"] == 2 - assert isinstance(info["stats"], dict) - - @pytest.mark.asyncio - async def test_cleanup(self, cache): - """Test cache cleanup.""" - # Add some entries - await cache.put("key1", "value1") - await cache.put("key2", "value2") - - # Verify entries exist - assert await cache.get("key1") == "value1" - assert await cache.get("key2") == "value2" - - # Cleanup - await cache.cleanup() - - # All entries should be cleared - assert await cache.get("key1") is None - assert await cache.get("key2") is None - - stats = await cache.get_stats() - assert stats.total_size == 0 - - def test_string_representation(self, cache): - """Test string representation of cache.""" - str_repr = str(cache) - assert "ModelResponseCache" in str_repr - assert "0/10" in str_repr # 0 out of 10 max size - assert "hit_rate=" in str_repr \ No newline at end of file diff --git a/tests/orchestrator/models/selection/test_manager.py b/tests/orchestrator/models/selection/test_manager.py deleted file mode 100644 index 888a29e8..00000000 --- a/tests/orchestrator/models/selection/test_manager.py +++ /dev/null @@ -1,393 +0,0 @@ -"""Tests for model manager.""" - -import asyncio -import pytest -from unittest.mock import AsyncMock, MagicMock, patch - -from orchestrator.core.model import Model, ModelCapabilities, ModelCost -from orchestrator.models.registry import ModelRegistry -from orchestrator.models.selection.manager import ModelManager, ModelUsageStats -from orchestrator.models.selection.strategies import TaskRequirements, TaskBasedStrategy -from orchestrator.models.optimization.caching import ModelResponseCache - - -class MockModel(Model): - """Mock model for testing.""" - - def __init__(self, name: str, provider: str, capabilities: ModelCapabilities = None): - super().__init__( - name=name, - provider=provider, - capabilities=capabilities or ModelCapabilities(supported_tasks=["text_generation"]), - cost=ModelCost(is_free=True), - ) - self._is_available = True - - async def generate(self, prompt: str, temperature: float = 0.7, max_tokens: int = None, **kwargs): - await asyncio.sleep(0.01) # Simulate some latency - return f"Generated response from {self.name}: {prompt[:50]}..." - - async def generate_structured(self, prompt: str, schema: dict, temperature: float = 0.7, **kwargs): - await asyncio.sleep(0.01) - return {"response": f"Structured from {self.name}", "prompt": prompt} - - async def health_check(self) -> bool: - return True - - async def estimate_cost(self, prompt: str, max_tokens: int = None) -> float: - return 0.001 - - -@pytest.fixture -def mock_registry(): - """Create mock registry.""" - registry = MagicMock(spec=ModelRegistry) - registry.is_initialized = True - - # Create sample models - models = [ - MockModel("gpt-3.5-turbo", "openai"), - MockModel("claude-3-haiku", "anthropic"), - MockModel("llama2-7b", "local"), - ] - - # Mock list_models - model_info = { - model.name: { - "provider": model.provider, - "capabilities": model.capabilities.to_dict(), - "cost": model.cost.to_dict(), - } - for model in models - } - registry.list_models.return_value = model_info - - # Mock get_model - async def get_model(name, provider): - for model in models: - if model.name == name and model.provider == provider: - return model - raise ValueError(f"Model {name} not found") - - registry.get_model.side_effect = get_model - registry.initialize = AsyncMock() - registry.health_check = AsyncMock(return_value={"openai": True, "anthropic": True, "local": True}) - - return registry - - -@pytest.fixture -def model_manager(mock_registry): - """Create model manager for testing.""" - return ModelManager( - registry=mock_registry, - selection_strategy=TaskBasedStrategy(), - enable_caching=True, - enable_pooling=False, # Disable pooling for simpler tests - max_cache_size=100, - ) - - -class TestModelManager: - """Test ModelManager functionality.""" - - @pytest.mark.asyncio - async def test_initialization(self, model_manager): - """Test manager initialization.""" - assert model_manager.registry is not None - assert model_manager.selection_strategy is not None - assert model_manager.enable_caching is True - assert model_manager._cache is not None - - @pytest.mark.asyncio - async def test_select_model(self, model_manager): - """Test model selection.""" - requirements = TaskRequirements( - task_type="text_generation", - context_window=4096, - ) - - result = await model_manager.select_model(requirements) - - assert result.model is not None - assert result.provider is not None - assert result.confidence_score > 0 - assert result.selection_reason is not None - - @pytest.mark.asyncio - async def test_generate_with_model(self, model_manager): - """Test text generation with model.""" - requirements = TaskRequirements(task_type="text_generation") - selection_result = await model_manager.select_model(requirements) - - response, metadata = await model_manager.generate_with_model( - model=selection_result.model, - provider=selection_result.provider, - prompt="Hello, world!", - temperature=0.7, - ) - - assert isinstance(response, str) - assert "Hello, world!" in response or "Generated response" in response - assert "latency" in metadata - assert "cost" in metadata - assert "model" in metadata - assert metadata["cached"] is False # First call shouldn't be cached - - @pytest.mark.asyncio - async def test_generate_structured_with_model(self, model_manager): - """Test structured generation with model.""" - requirements = TaskRequirements(task_type="text_generation") - selection_result = await model_manager.select_model(requirements) - - schema = { - "type": "object", - "properties": { - "response": {"type": "string"}, - "sentiment": {"type": "string"}, - }, - } - - response, metadata = await model_manager.generate_structured_with_model( - model=selection_result.model, - provider=selection_result.provider, - prompt="Analyze this text", - schema=schema, - temperature=0.7, - ) - - assert isinstance(response, dict) - assert "response" in response - assert "latency" in metadata - assert "cost" in metadata - - @pytest.mark.asyncio - async def test_caching_behavior(self, model_manager): - """Test response caching.""" - requirements = TaskRequirements(task_type="text_generation") - selection_result = await model_manager.select_model(requirements) - - prompt = "Test prompt for caching" - - # First call - response1, metadata1 = await model_manager.generate_with_model( - model=selection_result.model, - provider=selection_result.provider, - prompt=prompt, - temperature=0.7, - use_cache=True, - ) - - # Second call with same parameters - response2, metadata2 = await model_manager.generate_with_model( - model=selection_result.model, - provider=selection_result.provider, - prompt=prompt, - temperature=0.7, - use_cache=True, - ) - - assert metadata1["cached"] is False - assert metadata2["cached"] is True - assert response1 == response2 - - @pytest.mark.asyncio - async def test_get_best_model(self, model_manager): - """Test getting best model instance.""" - requirements = TaskRequirements(task_type="text_generation") - - model, provider = await model_manager.get_best_model(requirements) - - assert isinstance(model, Model) - assert isinstance(provider, str) - assert model.capabilities.supports_task("text_generation") - - @pytest.mark.asyncio - async def test_model_stats_tracking(self, model_manager): - """Test model usage statistics tracking.""" - requirements = TaskRequirements(task_type="text_generation") - selection_result = await model_manager.select_model(requirements) - - # Make a few requests - for i in range(3): - await model_manager.generate_with_model( - model=selection_result.model, - provider=selection_result.provider, - prompt=f"Test prompt {i}", - ) - - # Check stats - stats = await model_manager.get_model_stats( - model_name=selection_result.model.name, - provider=selection_result.provider, - ) - - model_key = f"{selection_result.provider}:{selection_result.model.name}" - assert model_key in stats - assert stats[model_key]["total_requests"] == 3 - assert stats[model_key]["successful_requests"] == 3 - assert stats[model_key]["success_rate"] == 1.0 - - @pytest.mark.asyncio - async def test_health_check(self, model_manager): - """Test health check functionality.""" - # Make some requests first to have active models - requirements = TaskRequirements(task_type="text_generation") - selection_result = await model_manager.select_model(requirements) - - await model_manager.generate_with_model( - model=selection_result.model, - provider=selection_result.provider, - prompt="Health check test", - ) - - # Perform health check - health_result = await model_manager.health_check(force=True) - - assert "status" in health_result - assert "timestamp" in health_result - assert "total_models" in health_result - assert health_result["success"] == True - - @pytest.mark.asyncio - async def test_optimize_performance(self, model_manager): - """Test performance optimization.""" - # Make some requests to generate stats - requirements = TaskRequirements(task_type="text_generation") - selection_result = await model_manager.select_model(requirements) - - for i in range(5): - await model_manager.generate_with_model( - model=selection_result.model, - provider=selection_result.provider, - prompt=f"Optimization test {i}", - ) - - # Run optimization - optimization_result = await model_manager.optimize_performance() - - assert "timestamp" in optimization_result - assert "optimizations" in optimization_result - assert isinstance(optimization_result["optimizations"], list) - - @pytest.mark.asyncio - async def test_failure_tracking(self, model_manager): - """Test failure tracking and model health management.""" - requirements = TaskRequirements(task_type="text_generation") - selection_result = await model_manager.select_model(requirements) - - # Mock model to fail - original_generate = selection_result.model.generate - selection_result.model.generate = AsyncMock(side_effect=Exception("Model failed")) - - # Make failing requests - with pytest.raises(Exception): - await model_manager.generate_with_model( - model=selection_result.model, - provider=selection_result.provider, - prompt="This will fail", - ) - - # Check that failure is tracked - stats = await model_manager.get_model_stats( - model_name=selection_result.model.name, - provider=selection_result.provider, - ) - - model_key = f"{selection_result.provider}:{selection_result.model.name}" - assert stats[model_key]["failed_requests"] == 1 - assert stats[model_key]["success_rate"] < 1.0 - - # Restore original method - selection_result.model.generate = original_generate - - @pytest.mark.asyncio - async def test_cleanup(self, model_manager): - """Test manager cleanup.""" - # Make some requests to create state - requirements = TaskRequirements(task_type="text_generation") - selection_result = await model_manager.select_model(requirements) - - await model_manager.generate_with_model( - model=selection_result.model, - provider=selection_result.provider, - prompt="Cleanup test", - ) - - # Verify we have some state - stats = await model_manager.get_model_stats() - assert len(stats) > 0 - - # Clean up - await model_manager.cleanup() - - # Verify state is cleared (this might depend on implementation details) - manager_info = model_manager.get_manager_info() - # At minimum, verify cleanup was called without errors - assert isinstance(manager_info, dict) - - def test_manager_info(self, model_manager): - """Test manager info retrieval.""" - info = model_manager.get_manager_info() - - assert "strategy" in info - assert "caching_enabled" in info - assert "pooling_enabled" in info - assert info["caching_enabled"] is True - assert info["pooling_enabled"] is False - - -class TestModelUsageStats: - """Test ModelUsageStats data class.""" - - def test_success_rate_calculation(self): - """Test success rate calculation.""" - stats = ModelUsageStats( - total_requests=10, - successful_requests=8, - failed_requests=2, - ) - - assert stats.success_rate == 0.8 - - def test_success_rate_zero_requests(self): - """Test success rate with zero requests.""" - stats = ModelUsageStats() - assert stats.success_rate == 1.0 - - def test_average_latency_calculation(self): - """Test average latency calculation.""" - stats = ModelUsageStats( - successful_requests=5, - total_latency=2.5, # 2.5 seconds total - ) - - assert stats.average_latency == 0.5 # 0.5 seconds average - - def test_average_latency_zero_requests(self): - """Test average latency with zero successful requests.""" - stats = ModelUsageStats() - assert stats.average_latency == 0.0 - - def test_to_dict(self): - """Test conversion to dictionary.""" - stats = ModelUsageStats( - total_requests=10, - successful_requests=8, - failed_requests=2, - total_latency=4.0, - total_cost=0.05, - last_used=1234567890.0, - error_messages=["Error 1", "Error 2", "Error 3"], - ) - - result = stats.to_dict() - - assert result["total_requests"] == 10 - assert result["successful_requests"] == 8 - assert result["failed_requests"] == 2 - assert result["success_rate"] == 0.8 - assert result["average_latency"] == 0.5 - assert result["total_cost"] == 0.05 - assert result["last_used"] == 1234567890.0 - assert len(result["recent_errors"]) == 3 # All errors since < 5 \ No newline at end of file diff --git a/tests/orchestrator/models/selection/test_strategies.py b/tests/orchestrator/models/selection/test_strategies.py deleted file mode 100644 index 5f092e01..00000000 --- a/tests/orchestrator/models/selection/test_strategies.py +++ /dev/null @@ -1,446 +0,0 @@ -"""Tests for model selection strategies.""" - -import pytest -from unittest.mock import AsyncMock, MagicMock - -from orchestrator.core.model import Model, ModelCapabilities, ModelCost, ModelRequirements -from orchestrator.models.registry import ModelRegistry -from orchestrator.models.selection.strategies import ( - TaskRequirements, - SelectionResult, - TaskBasedStrategy, - CostAwareStrategy, - PerformanceBasedStrategy, - WeightedStrategy, - FallbackStrategy, -) - - -class MockModel(Model): - """Mock model for testing.""" - - def __init__( - self, - name: str, - provider: str, - capabilities: ModelCapabilities, - cost: ModelCost, - requirements: ModelRequirements = None, - ): - super().__init__(name, provider, capabilities, requirements, cost=cost) - self._is_available = True - - async def generate(self, prompt: str, temperature: float = 0.7, max_tokens: int = None, **kwargs): - return f"Generated response from {self.name}" - - async def generate_structured(self, prompt: str, schema: dict, temperature: float = 0.7, **kwargs): - return {"response": f"Structured response from {self.name}"} - - async def health_check(self) -> bool: - return True - - async def estimate_cost(self, prompt: str, max_tokens: int = None) -> float: - return 0.001 # $0.001 - - -@pytest.fixture -def sample_models(): - """Create sample models for testing.""" - models = [] - - # Fast, accurate, expensive model - models.append(MockModel( - name="gpt-4", - provider="openai", - capabilities=ModelCapabilities( - supported_tasks=["text_generation", "analysis", "code_generation"], - context_window=8192, - supports_function_calling=True, - accuracy_score=0.95, - speed_rating="medium", - ), - cost=ModelCost( - input_cost_per_1k_tokens=0.03, - output_cost_per_1k_tokens=0.06, - ), - )) - - # Fast, moderate accuracy, cheap model - models.append(MockModel( - name="gpt-3.5-turbo", - provider="openai", - capabilities=ModelCapabilities( - supported_tasks=["text_generation", "analysis"], - context_window=4096, - accuracy_score=0.85, - speed_rating="fast", - ), - cost=ModelCost( - input_cost_per_1k_tokens=0.001, - output_cost_per_1k_tokens=0.002, - ), - )) - - # High accuracy, slow, expensive model - models.append(MockModel( - name="claude-3-opus", - provider="anthropic", - capabilities=ModelCapabilities( - supported_tasks=["text_generation", "analysis", "creative_writing"], - context_window=200000, - accuracy_score=0.98, - speed_rating="slow", - domains=["creative", "analysis"], - ), - cost=ModelCost( - input_cost_per_1k_tokens=0.015, - output_cost_per_1k_tokens=0.075, - ), - )) - - # Free local model - models.append(MockModel( - name="llama2-7b", - provider="local", - capabilities=ModelCapabilities( - supported_tasks=["text_generation"], - context_window=4096, - accuracy_score=0.75, - speed_rating="medium", - code_specialized=True, - ), - cost=ModelCost(is_free=True), - )) - - return models - - -@pytest.fixture -def mock_registry(sample_models): - """Create mock registry with sample models.""" - registry = MagicMock(spec=ModelRegistry) - registry.is_initialized = True - - # Mock list_models to return model info - model_info = {} - for model in sample_models: - model_info[model.name] = { - "provider": model.provider, - "capabilities": model.capabilities.to_dict(), - "cost": model.cost.to_dict(), - } - registry.list_models.return_value = model_info - - # Mock get_model to return the actual models - async def get_model(name, provider): - for model in sample_models: - if model.name == name and model.provider == provider: - return model - raise ValueError(f"Model {name} not found") - - registry.get_model.side_effect = get_model - - return registry - - -class TestTaskBasedStrategy: - """Test TaskBasedStrategy.""" - - @pytest.mark.asyncio - async def test_text_generation_selection(self, mock_registry, sample_models): - """Test selection for text generation task.""" - strategy = TaskBasedStrategy() - requirements = TaskRequirements( - task_type="text_generation", - context_window=4096, - ) - - result = await strategy.select_model(mock_registry, requirements, sample_models) - - assert isinstance(result, SelectionResult) - assert result.model.capabilities.supports_task("text_generation") - assert result.confidence_score > 0 - assert len(result.alternatives) <= 4 - - @pytest.mark.asyncio - async def test_code_generation_selection(self, mock_registry, sample_models): - """Test selection for code generation task.""" - strategy = TaskBasedStrategy() - requirements = TaskRequirements( - task_type="code_generation", - required_capabilities=["code_specialized"], - ) - - result = await strategy.select_model(mock_registry, requirements, sample_models) - - # Should prefer the local model with code specialization - assert result.model.capabilities.code_specialized or result.model.capabilities.supports_task("code_generation") - - @pytest.mark.asyncio - async def test_large_context_requirement(self, mock_registry, sample_models): - """Test selection with large context requirement.""" - strategy = TaskBasedStrategy() - requirements = TaskRequirements( - task_type="text_generation", - context_window=100000, # Very large context - ) - - result = await strategy.select_model(mock_registry, requirements, sample_models) - - # Should select claude-3-opus with 200k context - assert result.model.capabilities.context_window >= 100000 - - def test_score_model(self, sample_models): - """Test model scoring.""" - strategy = TaskBasedStrategy() - requirements = TaskRequirements(task_type="text_generation") - - for model in sample_models: - score = strategy.score_model(model, requirements) - assert 0 <= score <= 1 - - def test_compatibility_check(self, sample_models): - """Test compatibility checking.""" - strategy = TaskBasedStrategy() - - # Test with exclude providers - requirements = TaskRequirements( - task_type="text_generation", - exclude_providers={"openai"}, - ) - - openai_model = next(m for m in sample_models if m.provider == "openai") - assert not strategy._is_compatible(openai_model, requirements) - - anthropic_model = next(m for m in sample_models if m.provider == "anthropic") - assert strategy._is_compatible(anthropic_model, requirements) - - -class TestCostAwareStrategy: - """Test CostAwareStrategy.""" - - @pytest.mark.asyncio - async def test_cost_optimization(self, mock_registry, sample_models): - """Test cost-aware selection.""" - strategy = CostAwareStrategy(cost_weight=0.8) # Heavy cost emphasis - requirements = TaskRequirements( - task_type="text_generation", - budget_limit=0.005, # Low budget - ) - - result = await strategy.select_model(mock_registry, requirements, sample_models) - - # Should prefer free or cheap models - assert result.model.cost.is_free or result.model.cost.input_cost_per_1k_tokens <= 0.005 - - @pytest.mark.asyncio - async def test_budget_constraint(self, mock_registry, sample_models): - """Test budget constraint filtering.""" - strategy = CostAwareStrategy() - requirements = TaskRequirements( - task_type="text_generation", - budget_limit=0.001, # Very tight budget - budget_period="per-task", - ) - - result = await strategy.select_model(mock_registry, requirements, sample_models) - - # Should select a model within budget - estimated_cost = result.model.cost.estimate_cost_for_budget_period("per-task") - assert estimated_cost <= 0.001 or result.model.cost.is_free - - def test_score_model_cost_emphasis(self, sample_models): - """Test cost-emphasized scoring.""" - strategy = CostAwareStrategy(cost_weight=0.9) - requirements = TaskRequirements(task_type="text_generation") - - free_model = next(m for m in sample_models if m.cost.is_free) - expensive_model = next(m for m in sample_models if m.cost.input_cost_per_1k_tokens > 0.01) - - free_score = strategy.score_model(free_model, requirements) - expensive_score = strategy.score_model(expensive_model, requirements) - - # Free model should score higher due to cost emphasis - assert free_score > expensive_score - - -class TestPerformanceBasedStrategy: - """Test PerformanceBasedStrategy.""" - - @pytest.mark.asyncio - async def test_performance_selection(self, mock_registry, sample_models): - """Test performance-based selection.""" - strategy = PerformanceBasedStrategy(accuracy_weight=0.8, speed_weight=0.2) - requirements = TaskRequirements(task_type="analysis") - - result = await strategy.select_model(mock_registry, requirements, sample_models) - - # Should prefer high-accuracy models - assert result.model.capabilities.accuracy_score >= 0.8 - - @pytest.mark.asyncio - async def test_speed_emphasis(self, mock_registry, sample_models): - """Test speed-emphasized selection.""" - strategy = PerformanceBasedStrategy(accuracy_weight=0.2, speed_weight=0.8) - requirements = TaskRequirements( - task_type="text_generation", - max_latency_ms=1000, # Low latency requirement - ) - - result = await strategy.select_model(mock_registry, requirements, sample_models) - - # Should prefer fast models - assert result.model.capabilities.speed_rating in ["fast", "medium"] - - def test_score_model_performance(self, sample_models): - """Test performance scoring.""" - strategy = PerformanceBasedStrategy() - requirements = TaskRequirements(task_type="text_generation") - - high_accuracy_model = max(sample_models, key=lambda m: m.capabilities.accuracy_score) - low_accuracy_model = min(sample_models, key=lambda m: m.capabilities.accuracy_score) - - high_score = strategy.score_model(high_accuracy_model, requirements) - low_score = strategy.score_model(low_accuracy_model, requirements) - - assert high_score > low_score - - -class TestWeightedStrategy: - """Test WeightedStrategy.""" - - @pytest.mark.asyncio - async def test_balanced_selection(self, mock_registry, sample_models): - """Test balanced weighted selection.""" - strategy = WeightedStrategy( - task_weight=0.25, - cost_weight=0.25, - performance_weight=0.25, - capability_weight=0.25, - ) - requirements = TaskRequirements(task_type="text_generation") - - result = await strategy.select_model(mock_registry, requirements, sample_models) - - assert isinstance(result, SelectionResult) - assert result.confidence_score > 0 - - @pytest.mark.asyncio - async def test_cost_heavy_weighting(self, mock_registry, sample_models): - """Test cost-heavy weighted selection.""" - strategy = WeightedStrategy( - task_weight=0.1, - cost_weight=0.7, - performance_weight=0.1, - capability_weight=0.1, - ) - requirements = TaskRequirements(task_type="text_generation") - - result = await strategy.select_model(mock_registry, requirements, sample_models) - - # Should prefer cost-effective models - assert result.model.cost.is_free or result.model.cost.get_cost_efficiency_score() > 50 - - def test_weight_normalization(self): - """Test that weights are properly normalized.""" - strategy = WeightedStrategy( - task_weight=2.0, - cost_weight=2.0, - performance_weight=2.0, - capability_weight=2.0, - ) - - # All weights should sum to 1.0 after normalization - total_weight = ( - strategy.task_weight + - strategy.cost_weight + - strategy.performance_weight + - strategy.capability_weight - ) - assert abs(total_weight - 1.0) < 1e-6 - - -class TestFallbackStrategy: - """Test FallbackStrategy.""" - - @pytest.mark.asyncio - async def test_successful_fallback(self, mock_registry, sample_models): - """Test successful fallback to working strategy.""" - # Create strategy with one failing strategy - failing_strategy = MagicMock() - failing_strategy.select_model = AsyncMock(side_effect=Exception("Strategy failed")) - failing_strategy.name = "failing" - - working_strategy = TaskBasedStrategy() - - fallback = FallbackStrategy(strategies=[failing_strategy, working_strategy]) - requirements = TaskRequirements(task_type="text_generation") - - result = await fallback.select_model(mock_registry, requirements, sample_models) - - assert isinstance(result, SelectionResult) - assert "[task_based]" in result.selection_reason - - @pytest.mark.asyncio - async def test_all_strategies_fail(self, mock_registry, sample_models): - """Test behavior when all strategies fail.""" - failing_strategy1 = MagicMock() - failing_strategy1.select_model = AsyncMock(side_effect=Exception("Strategy 1 failed")) - failing_strategy1.name = "failing1" - - failing_strategy2 = MagicMock() - failing_strategy2.select_model = AsyncMock(side_effect=Exception("Strategy 2 failed")) - failing_strategy2.name = "failing2" - - fallback = FallbackStrategy(strategies=[failing_strategy1, failing_strategy2]) - requirements = TaskRequirements(task_type="text_generation") - - with pytest.raises(ValueError, match="All fallback strategies failed"): - await fallback.select_model(mock_registry, requirements, sample_models) - - -class TestTaskRequirements: - """Test TaskRequirements data class.""" - - def test_to_dict(self): - """Test conversion to dictionary.""" - requirements = TaskRequirements( - task_type="text_generation", - context_window=4096, - max_cost_per_1k_tokens=0.01, - required_capabilities=["function_calling"], - exclude_providers={"provider1", "provider2"}, - ) - - result = requirements.to_dict() - - assert result["task_type"] == "text_generation" - assert result["context_window"] == 4096 - assert result["max_cost_per_1k_tokens"] == 0.01 - assert result["required_capabilities"] == ["function_calling"] - assert set(result["exclude_providers"]) == {"provider1", "provider2"} - - -class TestSelectionResult: - """Test SelectionResult data class.""" - - def test_to_dict(self, sample_models): - """Test conversion to dictionary.""" - model = sample_models[0] - result = SelectionResult( - model=model, - provider="test_provider", - confidence_score=0.85, - selection_reason="Test selection", - alternatives=[(sample_models[1], "alt_provider", 0.75)], - estimated_cost=0.001, - ) - - result_dict = result.to_dict() - - assert result_dict["model"]["name"] == model.name - assert result_dict["provider"] == "test_provider" - assert result_dict["confidence_score"] == 0.85 - assert result_dict["selection_reason"] == "Test selection" - assert len(result_dict["alternatives"]) == 1 - assert result_dict["estimated_cost"] == 0.001 \ No newline at end of file diff --git a/tests/test_anthropic_model_names.py b/tests/test_anthropic_model_names.py deleted file mode 100644 index d1b29e46..00000000 --- a/tests/test_anthropic_model_names.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Model-name resolution for the Anthropic adapter. - -`_normalize_model_name` used to substring-match the family name and rewrite -*every* id containing "haiku"/"opus"/"sonnet" to a hard-coded 2024 model, so - - AnthropicModel(name="claude-haiku-4-5-20251001") - -actually requested `claude-3-haiku-20240307` and the API returned -404 not_found_error. No current Claude model was reachable, and the caller's -explicit choice was discarded silently. - -These tests are hermetic: they exercise resolution only, never the network. -Whether the alias *targets* exist is a question only the live API can answer, -and that is asserted in tests/test_live_anthropic.py. -""" - -import pytest - -from orchestrator.models.anthropic_model import AnthropicModel - -pytestmark = pytest.mark.unit - - -def _resolve(name: str) -> str: - """Call the resolver without constructing a client (no API key needed).""" - return AnthropicModel._normalize_model_name(AnthropicModel, name) - - -@pytest.mark.parametrize( - "name", - [ - # Dated ids across generations -- all must survive untouched. - "claude-haiku-4-5-20251001", - "claude-opus-4-1-20250805", - "claude-sonnet-4-5-20250929", - "claude-3-5-sonnet-20241022", - "claude-3-haiku-20240307", - # Rolling aliases are already fully qualified. - "claude-3-5-sonnet-latest", - "claude-opus-4-latest", - ], -) -def test_qualified_ids_pass_through_unchanged(name): - """The regression that broke every current model. - - A caller who names an exact model must get that exact model. Silently - substituting a different one is wrong even when the substitute works. - """ - assert _resolve(name) == name - - -@pytest.mark.parametrize("family", ["haiku", "opus", "sonnet"]) -def test_bare_family_names_are_deferred_not_guessed(family): - """A bare family name survives __init__ untouched. - - It is resolved against the Models API on first use. Resolving here would - require a network call during construction, and hard-coding a target is - what rotted twice already: the 2024 dated ids 404 today, and invented - "-latest" aliases 404 as well -- both confirmed against the live API. - """ - assert _resolve(family) == family - assert family in AnthropicModel._FAMILIES - - -def test_no_hardcoded_model_id_table_exists(): - """Guards against reintroducing a table that cannot be kept correct. - - Every hard-coded mapping in this module has become wrong within a year. - Model ids now come from the API. - """ - assert not hasattr(AnthropicModel, "_FAMILY_ALIASES"), ( - "hard-coded family->id mapping reintroduced; resolve from the Models " - "API instead (see resolve_family_alias)" - ) - - -@pytest.mark.parametrize( - ("name", "expected"), - [("claude-2.1", "claude-2.1"), ("claude-2", "claude-2.0"), - ("claude-instant", "claude-instant-1.2")], -) -def test_legacy_generations_keep_their_exact_ids(name, expected): - assert _resolve(name) == expected - - -def test_unknown_names_are_passed_through_not_guessed(): - """An unknown id must reach the API and produce a clear 404. - - Substituting a "closest match" turns a one-line error into a silent - behaviour change that is very hard to notice. - """ - assert _resolve("totally-unknown-model") == "totally-unknown-model" - - -def test_family_match_does_not_hijack_a_qualified_id(): - """The exact shape of the original bug, pinned so it cannot return.""" - name = "claude-haiku-4-5-20251001" - assert "haiku" in name, "test premise: the id contains a family name" - assert _resolve(name) == name, ( - "a qualified id containing a family name was rewritten -- this is the " - "regression that made every current Claude model unreachable" - ) diff --git a/tests/test_auto_tags_documentation.py b/tests/test_auto_tags_documentation.py deleted file mode 100644 index 9fb58a4a..00000000 --- a/tests/test_auto_tags_documentation.py +++ /dev/null @@ -1,567 +0,0 @@ -#!/usr/bin/env python3 -"""Test all AUTO tag documentation examples with real execution.""" - -import asyncio -import os -import tempfile - -from orchestrator import Orchestrator -from orchestrator.models.openai_model import OpenAIModel -from orchestrator.integrations.ollama_model import OllamaModel - -from tests.test_infrastructure import create_test_orchestrator, TestModel, TestProvider - - -async def setup_orchestrator(): - """Set up orchestrator with real models.""" - # Initialize orchestrator with models - from orchestrator.models.registry_singleton import get_model_registry - - # Get singleton registry - registry = get_model_registry() - - # Check if models are already registered - existing_models = registry.list_models() - models_registered = [] - - # Only register if not already present - if "ollama:llama3.2:1b" not in existing_models: - try: - llama = OllamaModel("llama3.2:1b") - registry.register_model(llama) - models_registered.append("llama3.2:1b") - print("✓ Registered llama3.2:1b") - except Exception as e: - print(f"✗ Failed to register llama3.2:1b: {e}") - else: - models_registered.append("llama3.2:1b") - - # Try OpenAI - if os.getenv("OPENAI_API_KEY") and "openai:gpt-3.5-turbo" not in existing_models: - try: - gpt35 = OpenAIModel("gpt-3.5-turbo") - registry.register_model(gpt35) - models_registered.append("gpt-3.5-turbo") - print("✓ Registered gpt-3.5-turbo") - except Exception as e: - print(f"✗ Failed to register gpt-3.5-turbo: {e}") - elif "openai:gpt-3.5-turbo" in existing_models: - models_registered.append("gpt-3.5-turbo") - - if not models_registered: - raise RuntimeError("No models available for testing") - - # Create orchestrator with initialized registry - orchestrator = create_test_orchestrator() - orchestrator.model_registry = registry - - return orchestrator, models_registered - - -async def test_dynamic_data_analyzer(): - """Test Example 1: Dynamic Data Analysis.""" - print("\n=== Testing Dynamic Data Analyzer ===") - - # Create test data file - with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as f: - f.write("name,age,score\n") - f.write("Alice,25,85\n") - f.write("Bob,30,92\n") - f.write("Charlie,28,78\n") - test_file = f.name - - try: - pipeline_yaml = """ -name: dynamic-data-analyzer -description: Analyze data with AI-determined methods -version: "1.0.0" - -inputs: - data_file: - type: string - description: Path to data file - required: true - -steps: - - id: read_data - action: filesystem - tool_config: - action: "read" - parameters: - path: "{{ data_file }}" - - - id: analyze_data - action: llm-generate - parameters: - prompt: | - Analyze this CSV data and provide insights: - {{ read_data.result }} - analysis_type: Based on the data which appears to be CSV with names and scores, should we use 'statistical', 'qualitative', or 'mixed' analysis? Just answer with one word. - - - id: save_analysis - action: filesystem - tool_config: - action: "write" - parameters: - path: "/tmp/analysis_report.md" - content: | - # Analysis Report - Type: {{ analyze_data.analysis_type }} - - {{ analyze_data.result }} -""" - - # Save pipeline - pipeline_file = "/tmp/test_dynamic_analyzer.yaml" - with open(pipeline_file, "w") as f: - f.write(pipeline_yaml) - - # Execute pipeline - orchestrator, _ = await setup_orchestrator() - result = await orchestrator.execute_yaml( - pipeline_file, inputs={"data_file": test_file} - ) - - print("✓ Pipeline executed successfully") - print( - f" Analysis type chosen: {result.get('analyze_data', {}).get('analysis_type', 'N/A')}" - ) - print(" Analysis saved to: /tmp/analysis_report.md") - - # Verify output file exists - if os.path.exists("/tmp/analysis_report.md"): - print("✓ Output file created successfully") - - return True - - except Exception as e: - print(f"✗ Test failed: {e}") - return False - finally: - # Cleanup - if os.path.exists(test_file): - os.unlink(test_file) - - -async def test_intelligent_error_handler(): - """Test Example 2: Intelligent Error Handling.""" - print("\n=== Testing Intelligent Error Handler ===") - - pipeline_yaml = """ -name: smart-error-handler -description: Handle errors intelligently based on context -version: "1.0.0" - -steps: - - id: risky_operation - action: web-search - parameters: - query: "latest AI news" - num_results: 3 - error_handling: - retry: - max_attempts: For searching AI news which is moderately important, how many retry attempts should we make? Answer with just a number between 1-5. - - - id: process_results - action: llm-generate - condition: "{{ risky_operation.status != 'failed' }}" - parameters: - prompt: | - Summarize these AI news results: - {{ risky_operation.result }} - max_length: 200 -""" - - try: - # Save pipeline - pipeline_file = "/tmp/test_error_handler.yaml" - with open(pipeline_file, "w") as f: - f.write(pipeline_yaml) - - # Execute pipeline - orchestrator, _ = await setup_orchestrator() - result = await orchestrator.execute_yaml(pipeline_file) - - print("✓ Pipeline executed successfully") - print( - f" Web search status: {result.get('risky_operation', {}).get('status', 'N/A')}" - ) - - return True - - except Exception as e: - print(f"✗ Test failed: {e}") - return False - - -async def test_dynamic_tool_selection(): - """Test Example 3: Dynamic Tool Selection.""" - print("\n=== Testing Dynamic Tool Selection ===") - - pipeline_yaml = """ -name: smart-researcher -description: Research with dynamically selected tools -version: "1.0.0" - -inputs: - topic: - type: string - description: Research topic - required: true - default: "quantum computing" - -steps: - - id: determine_approach - action: llm-generate - parameters: - prompt: | - Research topic: {{ topic }} - Determine the best research approach. - questions: - approach: For researching '{{ topic }}', is it better to use 'web_search', 'academic_sources', or 'both'? Answer with just one of these options. - include_visuals: For the topic '{{ topic }}', would visual data be helpful? Answer just 'yes' or 'no'. - - - id: web_research - action: web-search - condition: "'web_search' in determine_approach.approach or 'both' in determine_approach.approach" - parameters: - query: "{{ topic }}" - num_results: For the topic '{{ topic }}', how many search results would be appropriate? Answer with just a number between 3-10. - - - id: summarize - action: llm-generate - parameters: - prompt: | - Topic: {{ topic }} - Research approach: {{ determine_approach.approach }} - Include visuals: {{ determine_approach.include_visuals }} - {% if web_research.result %} - Web results: {{ web_research.result }} - {% endif %} - - Provide a brief summary of the research approach and findings. -""" - - try: - # Save pipeline - pipeline_file = "/tmp/test_tool_selection.yaml" - with open(pipeline_file, "w") as f: - f.write(pipeline_yaml) - - # Execute pipeline - orchestrator, _ = await setup_orchestrator() - result = await orchestrator.execute_yaml( - pipeline_file, inputs={"topic": "artificial general intelligence"} - ) - - print("✓ Pipeline executed successfully") - print( - f" Approach chosen: {result.get('determine_approach', {}).get('approach', 'N/A')}" - ) - print( - f" Include visuals: {result.get('determine_approach', {}).get('include_visuals', 'N/A')}" - ) - if "web_research" in result: - print(" Web search performed: Yes") - - return True - - except Exception as e: - print(f"✗ Test failed: {e}") - return False - - -async def test_auto_tags_in_control_flow(): - """Test AUTO tags in control flow structures.""" - print("\n=== Testing AUTO Tags in Control Flow ===") - - # Create test data - test_data = {"quality": "high", "size": 1000, "format": "csv"} - with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: - import json - - json.dump(test_data, f) - test_file = f.name - - pipeline_yaml = """ -name: auto-control-flow -description: Test AUTO tags in control flow -version: "1.0.0" - -inputs: - data_file: - type: string - description: Path to data file - required: true - -steps: - - id: read_data - action: filesystem - tool_config: - action: "read" - parameters: - path: "{{ data_file }}" - - - id: check_data_quality - action: validation - parameters: - data: "{{ read_data.result }}" - schema: - type: object - properties: - quality: - type: string - size: - type: number - - - id: decide_processing - action: llm-generate - parameters: - prompt: "Data validation passed. Determine processing approach." - should_process: The data has quality='high' and size=1000. Should we proceed with processing? Answer only 'true' or 'false'. - - - id: process_data - action: llm-generate - condition: "{{ decide_processing.should_process == 'true' }}" - parameters: - prompt: | - Process this data: - {{ read_data.result }} - method: For high quality data of size 1000, what processing method is best: 'quick', 'standard', or 'comprehensive'? Answer with just one word. -""" - - try: - # Save pipeline - pipeline_file = "/tmp/test_control_flow.yaml" - with open(pipeline_file, "w") as f: - f.write(pipeline_yaml) - - # Execute pipeline - orchestrator, _ = await setup_orchestrator() - result = await orchestrator.execute_yaml( - pipeline_file, inputs={"data_file": test_file} - ) - - print("✓ Pipeline executed successfully") - print( - f" Should process decision: {result.get('decide_processing', {}).get('should_process', 'N/A')}" - ) - if "process_data" in result: - print( - f" Processing method: {result.get('process_data', {}).get('method', 'N/A')}" - ) - - return True - - except Exception as e: - print(f"✗ Test failed: {e}") - return False - finally: - if os.path.exists(test_file): - os.unlink(test_file) - - -async def test_auto_tag_best_practices(): - """Test AUTO tag best practices examples.""" - print("\n=== Testing AUTO Tag Best Practices ===") - - pipeline_yaml = """ -name: auto-best-practices -description: Demonstrate AUTO tag best practices -version: "1.0.0" - -steps: - - id: good_specific_choice - action: llm-generate - parameters: - prompt: "Generate a report" - format: Choose output format: 'json', 'yaml', or 'xml' - - - id: good_context_aware - action: llm-generate - parameters: - prompt: "Analyze data" - data_size: "50MB" - num_columns: "20" - analysis_depth: Given that this is a 50MB dataset with 20 columns, choose analysis depth: 'quick' (5 min), 'standard' (15 min), or 'comprehensive' (1 hour). Answer with just one word: quick, standard, or comprehensive. - - - id: good_type_hints - action: llm-generate - parameters: - prompt: "Configure system" - num_retries: For a moderately important operation, how many retries are appropriate? Answer with just a number between 1 and 5. - include_logs: Should we include detailed logs? Answer only 'true' or 'false'. - - - id: good_fallback - action: llm-generate - parameters: - prompt: "Process request" - strategy: Choose processing strategy: 'fast', 'balanced', or 'thorough'. Answer with just one word. - error_handling: - on_error: - - id: use_default - action: llm-generate - parameters: - prompt: "Using default strategy" - strategy: "balanced" -""" - - try: - # Save pipeline - pipeline_file = "/tmp/test_best_practices.yaml" - with open(pipeline_file, "w") as f: - f.write(pipeline_yaml) - - # Execute pipeline - orchestrator, _ = await setup_orchestrator() - result = await orchestrator.execute_yaml(pipeline_file) - - print("✓ Pipeline executed successfully") - print( - f" Format chosen: {result.get('good_specific_choice', {}).get('format', 'N/A')}" - ) - print( - f" Analysis depth: {result.get('good_context_aware', {}).get('analysis_depth', 'N/A')}" - ) - print( - f" Num retries: {result.get('good_type_hints', {}).get('num_retries', 'N/A')}" - ) - print( - f" Include logs: {result.get('good_type_hints', {}).get('include_logs', 'N/A')}" - ) - print(f" Strategy: {result.get('good_fallback', {}).get('strategy', 'N/A')}") - - # Verify type constraints - num_retries = result.get("good_type_hints", {}).get("num_retries") - if num_retries and isinstance(num_retries, (int, str)): - try: - retry_int = int(num_retries) - if 1 <= retry_int <= 5: - print("✓ Retry count within valid range") - except Exception: - pass - - return True - - except Exception as e: - print(f"✗ Test failed: {e}") - return False - - -async def test_performance_optimizations(): - """Test AUTO tag performance optimization patterns.""" - print("\n=== Testing Performance Optimizations ===") - - pipeline_yaml = """ -name: auto-performance -description: Test performance optimization patterns -version: "1.0.0" - -steps: - - id: batch_decisions - action: llm-generate - parameters: - prompt: "Make multiple decisions for report generation" - decisions: - output_format: What's the best format for a technical report: 'pdf', 'html', or 'markdown'? Answer with just one word. - include_summary: Should a technical report include an executive summary? Answer only 'true' or 'false'. - detail_level: For a technical audience, what detail level is appropriate: 'low', 'medium', or 'high'? Answer with just one word. - - - id: use_decisions - action: report-generator - parameters: - title: "Technical Report" - format: "{{ batch_decisions.output_format }}" - include_summary: "{{ batch_decisions.include_summary }}" - detail_level: "{{ batch_decisions.detail_level }}" - content: | - # Report Content - This report uses batched AUTO tag decisions: - - Format: {{ batch_decisions.output_format }} - - Summary: {{ batch_decisions.include_summary }} - - Detail: {{ batch_decisions.detail_level }} -""" - - try: - # Save pipeline - pipeline_file = "/tmp/test_performance.yaml" - with open(pipeline_file, "w") as f: - f.write(pipeline_yaml) - - # Execute pipeline - orchestrator, _ = await setup_orchestrator() - result = await orchestrator.execute_yaml(pipeline_file) - - print("✓ Pipeline executed successfully") - print(" Batched decisions made:") - print( - f" - Format: {result.get('batch_decisions', {}).get('output_format', 'N/A')}" - ) - print( - f" - Include summary: {result.get('batch_decisions', {}).get('include_summary', 'N/A')}" - ) - print( - f" - Detail level: {result.get('batch_decisions', {}).get('detail_level', 'N/A')}" - ) - - return True - - except Exception as e: - print(f"✗ Test failed: {e}") - return False - - -async def main(): - """Run all AUTO tag documentation tests.""" - print("🚀 TESTING AUTO TAG DOCUMENTATION EXAMPLES") - print("=" * 50) - - # Check if we have models available - try: - orchestrator, models = await setup_orchestrator() - print(f"\nAvailable models: {', '.join(models)}") - except Exception as e: - print(f"\n❌ Cannot run tests: {e}") - print("\nPlease ensure:") - print(" - Ollama is running (for local models)") - print(" - API keys are set (OPENAI_API_KEY, ANTHROPIC_API_KEY)") - return - - # Run all tests - tests = [ - ("Dynamic Data Analyzer", test_dynamic_data_analyzer), - ("Intelligent Error Handler", test_intelligent_error_handler), - ("Dynamic Tool Selection", test_dynamic_tool_selection), - ("AUTO Tags in Control Flow", test_auto_tags_in_control_flow), - ("AUTO Tag Best Practices", test_auto_tag_best_practices), - ("Performance Optimizations", test_performance_optimizations), - ] - - results = [] - for test_name, test_func in tests: - print(f"\n{'='*50}") - success = await test_func() - results.append((test_name, success)) - - # Summary - print("\n" + "=" * 50) - print("TEST SUMMARY") - print("=" * 50) - - passed = sum(1 for _, success in results if success) - total = len(results) - - for test_name, success in results: - status = "✓ PASSED" if success else "✗ FAILED" - print(f"{status} - {test_name}") - - print(f"\nTotal: {passed}/{total} tests passed") - - if passed == total: - print("\n✅ All AUTO tag documentation examples are working correctly!") - else: - print("\n❌ Some tests failed. Please check the errors above.") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/tests/test_cli_hermetic_and_quiet.py b/tests/test_cli_hermetic_and_quiet.py index 0e4c8981..0b74dc92 100644 --- a/tests/test_cli_hermetic_and_quiet.py +++ b/tests/test_cli_hermetic_and_quiet.py @@ -310,44 +310,3 @@ def test_tool_only_run_reports_no_provider_registration_warnings(tmp_path): result = _run_cli(["run", str(BASIC), "-i", "greeting=hi"], cwd=work, home=home) combined = result.stdout + result.stderr assert "registering" not in combined.lower(), combined - - -def test_missing_provider_library_is_reported_once(tmp_path): - """The cause is named once, not wrapped in a bogus installation failure. - - ``AnthropicModel(...)`` used to raise "Failed to install Anthropic library: - Anthropic library is not installed ..." -- one cause reported twice, with - the outer half naming an installation that was never attempted. - """ - probe = tmp_path / "probe.py" - probe.write_text( - "import orchestrator.integrations.anthropic_model as m\n" - "if m.ANTHROPIC_AVAILABLE:\n" - " print('SKIP')\n" - "else:\n" - " try:\n" - " m.AnthropicModel(model_name='claude-sonnet-4-20250514', api_key='x')\n" - " except ImportError as exc:\n" - " print('ERR', exc)\n" - ) - - env = _hermetic_env(_decoy_home(tmp_path)) - result = subprocess.run( - [sys.executable, str(probe)], - cwd=str(tmp_path), - env=env, - capture_output=True, - text=True, - timeout=120, - ) - assert result.returncode == 0, f"{result.stdout}\n{result.stderr}" - - if result.stdout.strip() == "SKIP": - pytest.skip("anthropic library is installed; nothing to report") - - message = result.stdout.strip() - assert message.startswith("ERR ") - assert "is not installed" in message - assert "Failed to install" not in message, message - # The cause is stated once. - assert message.count("is not installed") == 1, message diff --git a/tests/test_creative_image_pipeline.py b/tests/test_creative_image_pipeline.py deleted file mode 100644 index 6bf6f57d..00000000 --- a/tests/test_creative_image_pipeline.py +++ /dev/null @@ -1,619 +0,0 @@ -""" -Comprehensive test suite for creative_image_pipeline. -Tests all functionality with REAL API calls (NO MOCKS). -""" - -import pytest - -pytest.importorskip("PIL", reason="requires the [multimedia] extra") - -import os -import json -import asyncio -from pathlib import Path -import pytest -import yaml -import tempfile -from typing import Dict, Any -import requests -from PIL import Image -import io - -from orchestrator.orchestrator import Orchestrator -from orchestrator.models.model_registry import ModelRegistry -from orchestrator.models.openai_model import OpenAIModel - -from tests.test_infrastructure import create_test_orchestrator, TestModel, TestProvider - - -@pytest.fixture -def orchestrator(): - """Create orchestrator instance with image generation models.""" - # Get the global model registry - from orchestrator.models.registry_singleton import get_model_registry - registry = get_model_registry() - - # Register DALL-E 3 model - api_key = os.getenv("OPENAI_API_KEY") - if not api_key: - pytest.skip("OpenAI API key not set") - - try: - dalle3 = OpenAIModel( - name="dall-e-3", - api_key=api_key - ) - registry.register_model(dalle3) - except Exception as e: - pytest.skip(f"DALL-E 3 not available: {e}") - - # Register GPT-4 Vision model - try: - gpt4v = OpenAIModel( - name="gpt-4-vision-preview", - api_key=api_key - ) - registry.register_model(gpt4v) - except Exception: - pass # Optional - - # Register a basic text model for other operations - try: - gpt35 = OpenAIModel( - name="gpt-3.5-turbo", - api_key=api_key - ) - registry.register_model(gpt35) - except Exception: - pass - - # Now create orchestrator with models already registered - orch = Orchestrator(model_registry=registry) - - return orch - - -@pytest.fixture -def pipeline_yaml(): - """Load the creative_image_pipeline.""" - pipeline_path = Path("examples/creative_image_pipeline.yaml") - with open(pipeline_path, 'r') as f: - return yaml.safe_load(f) - - -@pytest.fixture -def output_dir(): - """Create temporary output directory.""" - with tempfile.TemporaryDirectory() as tmpdir: - yield tmpdir - - -def download_image(url: str) -> Image.Image: - """Download image from URL and return PIL Image.""" - response = requests.get(url, timeout=30) - response.raise_for_status() - return Image.open(io.BytesIO(response.content)) - - -def verify_image_file(filepath: str) -> bool: - """Verify image file exists and is valid.""" - if not os.path.exists(filepath): - return False - try: - img = Image.open(filepath) - img.verify() - return True - except Exception: - return False - - -class TestCoreImageGeneration: - """Test core image generation functionality with real APIs.""" - - @pytest.mark.asyncio - @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not set") - async def test_dalle3_generation(self, orchestrator, output_dir): - """Test DALL-E 3 image generation with real API.""" - pipeline_dict = { - "id": "test-dalle3", - "name": "Test DALL-E 3 Generation", - "steps": [{ - "id": "generate", - "tool": "image-generation", - "action": "execute", - "parameters": { - "prompt": "A simple red circle on white background", - "size": "1024x1024", - "output_format": "file", - "output_path": output_dir - } - }] - } - - # Execute pipeline using the correct method - result = await orchestrator.execute_pipeline_from_dict(pipeline_dict) - - # Verify generation succeeded - assert "outputs" in result - assert "generate" in result["outputs"] - assert result["outputs"]["generate"]["success"] is True - assert "images" in result["outputs"]["generate"]["result"] - assert len(result["outputs"]["generate"]["result"]["images"]) > 0 - - # Verify image file was created - image_path = result["outputs"]["generate"]["result"]["images"][0]["path"] - assert verify_image_file(image_path) - - # Verify image dimensions - img = Image.open(image_path) - assert img.size == (1024, 1024) - - @pytest.mark.asyncio - @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not set") - async def test_image_analysis_real(self, orchestrator, output_dir): - """Test image analysis with GPT-4 Vision.""" - # First generate an image - pipeline = { - "id": "test-analysis", - "name": "Test Analysis", - "steps": [ - { - "id": "generate", - "tool": "image-generation", - "action": "execute", - "parameters": { - "prompt": "A blue square with the number 7 in white", - "size": "512x512", - "output_format": "file", - "output_path": output_dir - } - }, - { - "id": "analyze", - "tool": "image-analysis", - "action": "execute", - "parameters": { - "image": "{{ generate.images[0].path }}", - "analysis_type": "describe", - "detail_level": "high" - }, - "dependencies": ["generate"] - } - ] - } - - result = await orchestrator.execute_pipeline_from_dict(pipeline) - - # Verify analysis succeeded - assert result["outputs"]["analyze"]["success"] is True - assert "analysis" in result["outputs"]["analyze"] - assert "result" in result["outputs"]["analyze"]["analysis"] - - @pytest.mark.asyncio - @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not set") - async def test_style_variations(self, orchestrator, output_dir): - """Test generating multiple style variations.""" - pipeline = { - "id": "test-styles", - "name": "Test Styles", - "parameters": { - "base_prompt": "A peaceful garden", - "art_styles": ["photorealistic", "watercolor", "abstract"] - }, - "steps": [ - { - "id": "style1", - "tool": "image-generation", - "action": "execute", - "parameters": { - "prompt": "{{ base_prompt }}, {{ art_styles[0] }} style", - "size": "512x512", - "output_format": "file", - "output_path": f"{output_dir}/style1" - } - }, - { - "id": "style2", - "tool": "image-generation", - "action": "execute", - "parameters": { - "prompt": "{{ base_prompt }}, {{ art_styles[1] }} style", - "size": "512x512", - "output_format": "file", - "output_path": f"{output_dir}/style2" - } - } - ] - } - - result = await orchestrator.execute_pipeline_from_dict(pipeline) - - # Verify both styles generated - assert result["outputs"]["style1"]["success"] is True - assert result["outputs"]["style2"]["success"] is True - - # Verify different files created - path1 = result["outputs"]["style1"]["result"]["images"][0]["path"] - path2 = result["outputs"]["style2"]["result"]["images"][0]["path"] - assert path1 != path2 - assert verify_image_file(path1) - assert verify_image_file(path2) - - @pytest.mark.asyncio - async def test_prompt_optimization(self, orchestrator): - """Test prompt optimization for image generation.""" - pipeline = { - "id": "test-optimize", - "name": "Test Optimize", - "steps": [{ - "id": "optimize", - "tool": "prompt-optimization", - "action": "execute", - "parameters": { - "prompt": "sunset", - "task": "image-generation", - "optimization_goal": "artistic_quality" - } - }] - } - - result = await orchestrator.execute_pipeline_from_dict(pipeline) - - # Verify optimization succeeded - assert "optimized_prompt" in result["outputs"]["optimize"] - optimized = result["outputs"]["optimize"]["optimized_prompt"] - - # Optimized prompt should be longer/more detailed - assert len(optimized) > len("sunset") - - @pytest.mark.asyncio - @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not set") - async def test_batch_generation(self, orchestrator, output_dir): - """Test generating multiple images in batch.""" - pipeline = { - "id": "test-batch", - "name": "Test Batch", - "steps": [{ - "id": "batch", - "tool": "image-generation", - "action": "execute", - "parameters": { - "prompt": "A geometric pattern", - "size": "512x512", - "num_images": 3, - "output_format": "file", - "output_path": output_dir - } - }] - } - - result = await orchestrator.execute_pipeline_from_dict(pipeline) - - # Note: DALL-E 3 only supports n=1, so this will make multiple calls - assert result["outputs"]["batch"]["success"] is True - images = result["outputs"]["batch"]["result"]["images"] - - # Verify we got multiple images - assert len(images) >= 1 # At least one image - - # Verify all image files exist - for img_data in images: - assert verify_image_file(img_data["path"]) - - @pytest.mark.asyncio - @pytest.mark.timeout(120) - @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not set") - async def test_full_pipeline(self, orchestrator, pipeline_yaml): - """Test complete creative_image_pipeline with real APIs.""" - # Use simpler prompts to reduce costs - inputs = { - "base_prompt": "A simple geometric shape", - "num_variations": 2, - "art_styles": ["minimal", "bold"] - } - - result = await orchestrator.execute_pipeline_from_dict(pipeline_yaml, inputs) - - # Verify key steps completed - assert "generate_base_image" in result - assert result["outputs"]["generate_base_image"]["success"] is True - - # Verify report was created - assert "save_gallery_report" in result - report_path = result["save_gallery_report"]["path"] - assert os.path.exists(report_path) - - -class TestAPIIntegration: - """Test API integration and error handling.""" - - @pytest.mark.asyncio - async def test_openai_authentication(self, orchestrator): - """Verify OpenAI API authentication.""" - if not os.getenv("OPENAI_API_KEY"): - pytest.skip("OpenAI API key not set") - - pipeline = { - "id": "test-auth", - "name": "Test Auth", - "steps": [{ - "id": "test", - "tool": "image-generation", - "action": "execute", - "parameters": { - "prompt": "test", - "size": "256x256", - "output_format": "url" - } - }] - } - - # Should not raise authentication error - result = await orchestrator.execute_pipeline_from_dict(pipeline) - assert "error" not in result["test"] or "authentication" not in str(result["test"].get("error", "")).lower() - - @pytest.mark.asyncio - async def test_invalid_size_handling(self, orchestrator): - """Test handling of invalid image sizes.""" - pipeline = { - "id": "test-size", - "name": "Test Size", - "steps": [{ - "id": "invalid", - "tool": "image-generation", - "action": "execute", - "parameters": { - "prompt": "test", - "size": "999x999", # Invalid size - "output_format": "url" - } - }] - } - - result = await orchestrator.execute_pipeline_from_dict(pipeline) - - # Should handle invalid size gracefully - assert result["outputs"]["invalid"]["success"] is False - assert "size" in str(result["outputs"]["invalid"]["error"]).lower() - - @pytest.mark.asyncio - async def test_api_error_handling(self, orchestrator): - """Test handling of API errors.""" - pipeline = { - "id": "test-error", - "name": "Test Error", - "steps": [{ - "id": "error", - "tool": "image-generation", - "action": "execute", - "parameters": { - "prompt": "x" * 5000, # Exceeds prompt limit - "size": "1024x1024", - "output_format": "url" - } - }] - } - - result = await orchestrator.execute_pipeline_from_dict(pipeline) - - # Should handle error gracefully - if not result["outputs"]["error"]["success"]: - assert "error" in result["outputs"]["error"] - - -class TestImageQuality: - """Test image quality and properties.""" - - @pytest.mark.asyncio - @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not set") - async def test_image_resolution(self, orchestrator, output_dir): - """Verify actual image dimensions.""" - sizes = ["256x256", "512x512", "1024x1024"] - - for size in sizes: - pipeline = { - "id": f"test-{size}", - "name": f"Test {size}", - "steps": [{ - "id": "generate", - "tool": "image-generation", - "action": "execute", - "parameters": { - "prompt": "A test pattern", - "size": size, - "output_format": "file", - "output_path": output_dir - } - }] - } - - result = await orchestrator.execute_pipeline_from_dict(pipeline) - - if result["outputs"]["generate"]["success"]: - image_path = result["outputs"]["generate"]["result"]["images"][0]["path"] - img = Image.open(image_path) - - expected_size = tuple(map(int, size.split('x'))) - assert img.size == expected_size - - @pytest.mark.asyncio - @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not set") - async def test_image_format(self, orchestrator, output_dir): - """Verify image format and encoding.""" - pipeline = { - "id": "test-format", - "name": "Test Format", - "steps": [{ - "id": "generate", - "tool": "image-generation", - "action": "execute", - "parameters": { - "prompt": "A simple icon", - "size": "256x256", - "output_format": "file", - "output_path": output_dir - } - }] - } - - result = await orchestrator.execute_pipeline_from_dict(pipeline) - - if result["outputs"]["generate"]["success"]: - image_path = result["outputs"]["generate"]["result"]["images"][0]["path"] - img = Image.open(image_path) - - # Verify format - assert img.format in ["PNG", "JPEG", "WEBP"] - - # Verify mode - assert img.mode in ["RGB", "RGBA"] - - @pytest.mark.asyncio - @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not set") - async def test_image_download(self, orchestrator): - """Test downloading images from URLs.""" - pipeline = { - "id": "test-download", - "name": "Test Download", - "steps": [{ - "id": "generate", - "tool": "image-generation", - "action": "execute", - "parameters": { - "prompt": "A downloadable image", - "size": "256x256", - "output_format": "url" - } - }] - } - - result = await orchestrator.execute_pipeline_from_dict(pipeline) - - if result["outputs"]["generate"]["success"]: - url = result["outputs"]["generate"]["result"]["images"][0]["url"] - - # Download and verify - img = download_image(url) - assert img.size[0] > 0 - assert img.size[1] > 0 - - -class TestRealWorldScenarios: - """Test real-world use cases.""" - - @pytest.mark.asyncio - @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not set") - async def test_complex_prompt(self, orchestrator, output_dir): - """Test with detailed, complex prompts.""" - pipeline = { - "id": "test-complex", - "name": "Test Complex", - "steps": [{ - "id": "generate", - "tool": "image-generation", - "action": "execute", - "parameters": { - "prompt": "A futuristic city at sunset with flying cars, " - "holographic billboards, neon lights reflecting on wet streets, " - "cyberpunk aesthetic, highly detailed, cinematic lighting", - "size": "1024x1024", - "output_format": "file", - "output_path": output_dir - } - }] - } - - result = await orchestrator.execute_pipeline_from_dict(pipeline) - - assert result["outputs"]["generate"]["success"] is True - assert verify_image_file(result["outputs"]["generate"]["result"]["images"][0]["path"]) - - @pytest.mark.asyncio - @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not set") - async def test_artistic_styles(self, orchestrator, output_dir): - """Test different artistic styles.""" - styles = [ - "photorealistic", - "oil painting", - "watercolor", - "pencil sketch", - "digital art" - ] - - base_prompt = "A mountain landscape" - - for style in styles[:2]: # Limit to 2 to reduce costs - pipeline = { - "id": f"test-{style}", - "name": f"Test {style}", - "steps": [{ - "id": "generate", - "tool": "image-generation", - "action": "execute", - "parameters": { - "prompt": f"{base_prompt}, {style} style", - "size": "512x512", - "style": style, - "output_format": "file", - "output_path": f"{output_dir}/{style}" - } - }] - } - - result = await orchestrator.execute_pipeline_from_dict(pipeline) - assert result["outputs"]["generate"]["success"] is True - - @pytest.mark.asyncio - @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OpenAI API key not set") - async def test_sequential_refinement(self, orchestrator, output_dir): - """Test iterative image refinement.""" - pipeline = { - "id": "test-refine", - "name": "Test Refine", - "steps": [ - { - "id": "initial", - "tool": "image-generation", - "action": "execute", - "parameters": { - "prompt": "A basic house", - "size": "512x512", - "output_format": "file", - "output_path": f"{output_dir}/v1" - } - }, - { - "id": "analyze", - "tool": "image-analysis", - "action": "execute", - "parameters": { - "image": "{{ initial.images[0].path }}", - "analysis_type": "describe" - }, - "dependencies": ["initial"] - }, - { - "id": "refined", - "tool": "image-generation", - "action": "execute", - "parameters": { - "prompt": "A detailed modern house with glass windows, " - "landscaped garden, and architectural details", - "size": "512x512", - "output_format": "file", - "output_path": f"{output_dir}/v2" - }, - "dependencies": ["analyze"] - } - ] - } - - result = await orchestrator.execute_pipeline_from_dict(pipeline) - - # Verify refinement process - assert result["outputs"]["initial"]["success"] is True - assert result["outputs"]["refined"]["success"] is True - - # Both images should exist - assert verify_image_file(result["outputs"]["initial"]["result"]["images"][0]["path"]) - assert verify_image_file(result["outputs"]["refined"]["result"]["images"][0]["path"]) \ No newline at end of file diff --git a/tests/test_design_simulations.py b/tests/test_design_simulations.py new file mode 100644 index 00000000..00a2e279 --- /dev/null +++ b/tests/test_design_simulations.py @@ -0,0 +1,282 @@ +"""The #485 design simulations still demonstrate what the review said they did. + +`scripts/simulations/` backs the design review on #485 with four models. Their +value is entirely in a handful of *structural* claims -- claims that hold for +any parameter values, and that the review leans on. If a refactor quietly +breaks one, the review's argument stops being reproducible and the scripts +become decoration. + +So this file asserts the structure, not the tables. A golden-output test would +fail for every cosmetic change and pass for a silently wrong model, which is +the wrong way round. The specific numbers in those tables are only as good as +each script's `ASSUMPTIONS` dict, and are expected to move when someone +replaces a guess with a measurement -- see `scripts/simulations/README.md`. + +Determinism is asserted separately: every stochastic model is seeded, so the +review's numbers can be reproduced exactly. +""" + +import importlib.util +import random +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.unit] + +REPO = Path(__file__).resolve().parent.parent +SIMULATIONS = REPO / "scripts" / "simulations" + + +def _module(name: str): + """Load a simulation by path; `scripts/` is not an importable package.""" + spec = importlib.util.spec_from_file_location(name, SIMULATIONS / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +branching = _module("decomposition_branching") +scratchpad = _module("scratchpad_contention") +context = _module("context_recursion") +ASSUMPTIONS_DEPTH_CAP = context.ASSUMPTIONS["depth_cap"] +red_team = _module("red_team_gate") + + +# --- decomposition_branching ------------------------------------------------ + + +@pytest.mark.parametrize( + "b, f, expected", + [ + (5, 0.05, "subcritical"), + (10, 0.05, "subcritical"), + (5, 0.20, "critical"), + (10, 0.10, "critical"), + (10, 0.30, "SUPERCRITICAL"), + ], +) +def test_regime_is_decided_by_mean_offspring(b, f, expected): + """m = b*f is the whole classification; nothing else enters into it.""" + assert branching.regime(b, f) == expected + + +def test_subcritical_trees_stay_small_and_supercritical_ones_explode(): + """The claim the review rests on: crossing m = 1 changes the outcome by orders.""" + sub = branching.leaf_distribution(10, 0.05) + sup = branching.leaf_distribution(10, 0.30) + assert sub["m"] < 1 < sup["m"] + assert sub["median"] < 100 + assert sup["median"] > 10_000 + assert sup["median"] > 100 * sub["median"] + + +def test_leaf_count_rises_monotonically_with_ambiguity_rate(): + """More ambiguity is never cheaper. Guards against a sign or bounds slip.""" + medians = [branching.leaf_distribution(10, f)["median"] for f in (0.05, 0.1, 0.15, 0.2, 0.3)] + assert medians == sorted(medians) + + +def test_branching_is_seeded_and_reproducible(): + a = branching.branch(10, 0.3, random.Random(7)) + b = branching.branch(10, 0.3, random.Random(7)) + assert a == b + + +def test_depth_cap_bounds_the_tree(): + """Without the cap, a supercritical config would not return at all.""" + _, _, max_depth = branching.branch(10, 0.9, random.Random(1), depth_cap=4) + assert max_depth < 4 + + +def test_most_of_a_node_budget_is_spent_before_any_work_happens(): + """The review's "34% shared-state re-reading, 8% work" point.""" + assert branching.shared_state_share() > 0.3 + work_share = 12_000 / branching.tokens_per_node() + assert work_share < 0.1 + + +# --- scratchpad_contention -------------------------------------------------- + + +def test_locked_throughput_saturates_at_the_critical_section_ceiling(): + """Adding agents past saturation buys latency, not throughput.""" + ceiling = scratchpad.throughput_ceiling(8.0) + saturated = [scratchpad.simulate_lock(n, 8.0, 60.0)["steps_per_hour"] for n in (16, 32, 64, 128)] + assert ceiling == pytest.approx(450.0) + for observed in saturated: + assert observed <= ceiling * 1.05 + assert observed > ceiling * 0.9 + # 8x the fleet moves throughput by less than 5%: the definition of saturated. + assert abs(saturated[-1] - saturated[0]) / saturated[0] < 0.05 + + +def test_lock_wait_grows_with_fleet_size_once_saturated(): + waits = [scratchpad.simulate_lock(n, 8.0, 60.0)["mean_wait"] for n in (16, 32, 64, 128)] + assert waits == sorted(waits) + assert waits[-1] > 10 * waits[0] / 10 # strictly increasing, and by a lot + assert waits[-1] > 600 + + +def test_moving_the_model_call_out_of_the_lock_lifts_the_ceiling(): + """The review's headline fix, and the ~61x figure at 512 agents.""" + locked = scratchpad.throughput_ceiling(8.0) + unlocked = scratchpad.simulate_lock(512, 0.03, 68.0)["steps_per_hour"] + assert unlocked / locked > 50 + + +def test_sealed_segments_turn_quadratic_churn_into_linear(): + """Naive re-summarisation grows quadratically; sealed segments do not.""" + small, large = scratchpad.churn(1_000), scratchpad.churn(100_000) + naive_growth = large["naive"] / small["naive"] + sealed_growth = large["sealed"] / small["sealed"] + assert naive_growth > 1_000 # ~100x the notes -> ~10,000x the work + assert sealed_growth < 200 # ~100x the notes -> ~100x the work + assert large["naive"] / large["sealed"] > 10_000 + + +def test_lock_simulation_is_seeded_and_reproducible(): + a = scratchpad.simulate_lock(32, 8.0, 60.0, seed=0) + b = scratchpad.simulate_lock(32, 8.0, 60.0, seed=0) + assert a == b + + +def test_lock_utilisation_never_exceeds_one(): + """An earlier draft counted service past the horizon and reported 127%.""" + for n in (2, 16, 128): + assert scratchpad.simulate_lock(n, 8.0, 60.0)["utilisation"] <= 1.0 + + +# --- context_recursion ------------------------------------------------------ + + +def test_summary_tree_converges_for_any_real_compression_ratio(): + for r in (0.1, 0.5, 0.9): + tree = context.summary_tree(5_000_000, 1_000_000, r=r) + assert tree["converges"] + assert tree["depth_required"] < float("inf") + + +def test_summary_tree_does_not_converge_when_summaries_do_not_shrink(): + """r >= 1 is the failure mode; it must be reported, not silently capped.""" + for r in (1.0, 1.1): + tree = context.summary_tree(5_000_000, 1_000_000, r=r) + assert not tree["converges"] + assert tree["depth_required"] == float("inf") + + +def test_converging_and_fitting_within_the_depth_cap_are_different_claims(): + """r=0.9 terminates mathematically but needs 29 levels. Do not conflate them.""" + gentle = context.summary_tree(5_000_000, 1_000_000, r=0.9) + assert gentle["converges"] + assert gentle["hit_cap"] + assert gentle["depth_required"] > ASSUMPTIONS_DEPTH_CAP + + aggressive = context.summary_tree(5_000_000, 1_000_000, r=0.1) + assert aggressive["converges"] + assert not aggressive["hit_cap"] + assert aggressive["top"] <= aggressive["payload"] + + +def test_top_level_fidelity_is_set_by_target_size_not_by_compression_ratio(): + """Fidelity ~ payload/N whatever r is; r only buys you fewer lossy hops.""" + n_tokens, ctx = 5_000_000, 1_000_000 + fidelities = [] + for r in (0.1, 0.5, 0.9): + tree = context.summary_tree(n_tokens, ctx, r=r) + fidelities.append(r ** tree["depth_required"]) + forced = context.summary_tree(n_tokens, ctx)["payload"] / n_tokens + for observed in fidelities: + assert observed <= forced * 1.05 + assert observed > forced / 10 + # But the hop count -- and so the accumulated distortion -- differs hugely. + hops = [context.summary_tree(n_tokens, ctx, r=r)["depth_required"] for r in (0.1, 0.9)] + assert hops[1] > 10 * hops[0] + + +def test_depth_is_logarithmic_not_linear_in_document_size(): + """400x the corpus must not cost 400x the depth.""" + small = context.summary_tree(250_000, 1_000_000)["depth"] + large = context.summary_tree(100_000_000, 1_000_000)["depth"] + assert large <= small + 5 + assert large <= 5 + + +def test_total_cost_is_a_small_multiple_of_the_corpus(): + tree = context.summary_tree(100_000_000, 1_000_000) + assert tree["tokens"] < 2.5 * 100_000_000 + + +def test_fidelity_decays_geometrically_with_depth(): + """Why the summary tree cannot be the retrieval path.""" + tree = context.summary_tree(100_000_000, 128_000, r=0.2) + assert tree["fidelity"] == pytest.approx(0.2 ** tree["depth"]) + assert tree["fidelity"] < 0.001 + + +def test_overheads_shrink_the_usable_chunk_below_the_nominal_payload(): + """Ignoring instructions + output reserve overstates capacity.""" + tree = context.summary_tree(1, 1_000_000, r=0.4) + assert tree["chunk"] < tree["payload"] + assert tree["chunk"] > 0.7 * tree["payload"] + + +def test_impossible_overheads_raise_rather_than_loop(): + with pytest.raises(ValueError): + context.summary_tree(1_000_000, 1_000, instr=100_000) + + +def test_specified_budget_consumes_half_the_context_before_any_work(): + assert context.budget_subtotal() == pytest.approx(0.50) + + +# --- red_team_gate ---------------------------------------------------------- + + +def test_the_ledger_makes_review_rounds_independent_of_scope_drift(): + """The concern ledger's entire purpose, and it delivers exactly that.""" + with_ledger = [red_team.loop_stats(d, ledger=True)["mean_rounds"] for d in (0.0, 0.2, 0.4, 0.6)] + assert with_ledger == pytest.approx([with_ledger[0]] * 4) + + +def test_without_a_ledger_drift_inflates_rounds_and_leaves_a_hung_tail(): + baseline = red_team.loop_stats(0.0, ledger=False) + drifting = red_team.loop_stats(0.6, ledger=False) + assert drifting["mean_rounds"] > 2 * baseline["mean_rounds"] + assert drifting["never_clean_pct"] > 1.0 + assert red_team.loop_stats(0.6, ledger=True)["never_clean_pct"] == 0.0 + + +def test_a_clean_verdict_is_weak_evidence_at_a_realistic_detection_rate(): + """The uncomfortable result: at p_detect=0.5, "clean" is wrong more often than right.""" + low = red_team.loop_stats(0.0, True, p_detect=0.5, trials=4_000, seed=3) + assert low["p_truly_clean"] < 0.5 + assert low["residual_given_clean"] > 0.5 + + +def test_certification_strength_rises_with_detection_rate(): + rates = [ + red_team.loop_stats(0.0, True, p_detect=p, trials=4_000, seed=3)["p_truly_clean"] + for p in (0.3, 0.5, 0.7, 0.9) + ] + assert rates == sorted(rates) + assert rates[0] < 0.2 and rates[-1] > 0.9 + + +def test_same_family_reviewers_cannot_beat_the_blind_spot_ceiling(): + """No amount of "fresh session, same model" review passes 1 - rho.""" + for rho in (0.1, 0.3, 0.5): + assert red_team.catch_rate(rho, 1_000) <= 1 - rho + 1e-9 + assert red_team.catch_rate(rho, 5) < 1 - rho + + +def test_one_cross_family_reviewer_beats_five_same_family_ones_once_rho_is_real(): + assert red_team.cross_family_catch_rate(0.0) < red_team.catch_rate(0.0, 5) + for rho in (0.2, 0.3, 0.5): + assert red_team.cross_family_catch_rate(rho) > red_team.catch_rate(rho, 5) + + +def test_review_loop_is_seeded_and_reproducible(): + a = red_team.review_loop(6, 0.6, 0.15, 0.4, False, random.Random(11)) + b = red_team.review_loop(6, 0.6, 0.15, 0.4, False, random.Random(11)) + assert a == b diff --git a/tests/test_domain_routing.py b/tests/test_domain_routing.py deleted file mode 100644 index c792d843..00000000 --- a/tests/test_domain_routing.py +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/env python3 -"""Test domain-specific model routing.""" - -import os -import pytest - -from orchestrator.models.model_registry import ModelRegistry -from orchestrator.models.domain_router import DomainRouter, DomainConfig -from orchestrator.models.openai_model import OpenAIModel -from orchestrator.integrations.ollama_model import OllamaModel - -from tests.test_infrastructure import create_test_orchestrator, TestModel, TestProvider - - -async def setup_registry(): - """Set up model registry with domain capabilities.""" - registry = ModelRegistry() - - # Register models with domain expertise - try: - # General purpose model - llama = OllamaModel("llama3.1:8b") - llama.capabilities.domains = ["general", "educational"] - llama.capabilities.accuracy_score = 0.85 - llama._expertise = ["general", "educational"] # For compatibility - registry.register_model(llama) - print("✓ Registered llama3.1:8b (general, educational)") - except Exception as e: - print(f"✗ Failed to register llama3.1:8b: {e}") - - if os.getenv("OPENAI_API_KEY"): - try: - # GPT-3.5 for general/educational - gpt35 = OpenAIModel("gpt-3.5-turbo") - gpt35.capabilities.domains = ["general", "educational", "creative"] - gpt35.capabilities.accuracy_score = 0.85 - gpt35._expertise = [ - "general", - "educational", - "creative", - ] # For compatibility - registry.register_model(gpt35) - print("✓ Registered gpt-3.5-turbo (general, educational, creative)") - - # GPT-4 for advanced domains - gpt4 = OpenAIModel("gpt-4") - gpt4.capabilities.domains = [ - "general", - "technical", - "medical", - "legal", - "scientific", - "financial", - ] - gpt4.capabilities.accuracy_score = 0.95 - gpt4._expertise = [ - "general", - "technical", - "medical", - "legal", - "scientific", - "financial", - ] # For compatibility - registry.register_model(gpt4) - print( - "✓ Registered gpt-4 (technical, medical, legal, scientific, financial)" - ) - except Exception as e: - print(f"✗ Failed to register OpenAI models: {e}") - - return registry - - -@pytest.fixture -async def registry(): - """Create model registry fixture.""" - return await setup_registry() - - -@pytest.fixture -async def router(registry): - """Create domain router fixture.""" - # Create router (it initializes with default domains) - return DomainRouter(registry) - - -@pytest.mark.asyncio -async def test_domain_detection(router: DomainRouter): - """Test domain detection in various texts.""" - print("\n=== Testing Domain Detection ===") - - test_texts = [ - # Medical - ( - "The patient presented with symptoms of acute respiratory distress. " - "Initial diagnosis suggests pneumonia, requiring immediate antibiotic treatment.", - "medical"), - # Legal - ( - "According to Section 5.2 of the contract, the liability for damages " - "shall not exceed the total contract value. This clause is subject to " - "jurisdiction of the state court.", - "legal"), - # Creative - ( - "Write a short story about a robot who discovers it can feel emotions. " - "The narrative should explore themes of consciousness and identity.", - "creative"), - # Technical - ( - "We need to implement a microservices architecture with proper API " - "gateway integration. The system should handle 10k requests per second.", - "technical"), - # Scientific - ( - "Our hypothesis suggests that increased CO2 levels correlate with " - "temperature rise. The experimental methodology includes controlled " - "variables and peer-reviewed analysis.", - "scientific"), - # Financial - ( - "The portfolio shows a 15% return on investment this quarter. " - "Risk assessment indicates moderate exposure to market volatility.", - "financial"), - # Educational - ( - "Can you explain how photosynthesis works? I'm a student trying " - "to understand the concept for my biology class.", - "educational"), - # Multi-domain - ( - "As a medical researcher, I need to analyze patient data to test " - "my hypothesis about treatment efficacy. The results will be " - "published in a peer-reviewed journal.", - "medical/scientific"), - ] - - for text, expected in test_texts: - print(f"\nText: {text[:60]}...") - print(f"Expected: {expected}") - - detected = router.detect_domains(text) - if detected: - print( - f"Detected: {', '.join([f'{d[0]} ({d[1]:.2f})' for d in detected[:3]])}" - ) - else: - print("Detected: None") - - # Test analysis function - print("\n--- Full Analysis Example ---") - analysis = router.analyze_text(test_texts[0][0]) - print(f"Text length: {analysis['text_length']}") - print(f"Primary domain: {analysis['primary_domain']}") - print(f"All domains: {analysis['detected_domains']}") - - -@pytest.mark.asyncio -async def test_domain_routing(router: DomainRouter): - """Test model selection based on domain.""" - print("\n=== Testing Domain-Based Model Selection ===") - - test_cases = [ - { - "text": "Diagnose the patient's condition based on these symptoms", - "expected_domain": "medical", - "expected_accuracy": 0.8, - }, - { - "text": "Review this contract for potential legal issues", - "expected_domain": "legal", - "expected_accuracy": 0.8, - }, - { - "text": "Write a creative story about time travel", - "expected_domain": "creative", - "expected_accuracy": 0.8, - }, - { - "text": "Explain the water cycle to a 5th grade student", - "expected_domain": "educational", - "expected_accuracy": 0.8, - }, - ] - - for case in test_cases: - print(f"\nText: {case['text']}") - print(f"Expected domain: {case['expected_domain']}") - - try: - # Route by domain - model = await router.route_by_domain(case["text"]) - - print(f"Selected model: {model.provider}:{model.name}") - print(f"Model domains: {model.capabilities.domains}") - print(f"Model accuracy: {model.capabilities.accuracy_score}") - - # Check if model meets domain requirements - if case["expected_domain"] in model.capabilities.domains: - print("✓ Model has required domain expertise") - else: - print("✗ Model lacks required domain expertise") - - if model.capabilities.accuracy_score >= case["expected_accuracy"]: - print("✓ Model meets accuracy requirement") - else: - print("✗ Model below accuracy requirement") - - except Exception as e: - print(f"✗ Routing failed: {e}") - - -@pytest.mark.asyncio -async def test_custom_domain(router: DomainRouter): - """Test registering and using custom domains.""" - print("\n=== Testing Custom Domain Registration ===") - - # Create custom domain for gaming - gaming_domain = DomainConfig( - name="gaming", - keywords=["game", "player", "level", "quest", "boss", "gameplay", "mechanics"], - patterns=[ - r"\b(game|player|gameplay|mechanic)\b", - r"\b(level|quest|boss|npc|character)\b", - ], - preferred_models=["gpt-4", "claude-3-opus"], - required_capabilities=["creative", "gaming"], - min_accuracy_score=0.8) - - # Register the domain - router.register_domain(gaming_domain) - print("✓ Registered custom 'gaming' domain") - - # Test detection - gaming_text = "Design a boss battle for level 5 with unique gameplay mechanics" - detected = router.detect_domains(gaming_text) - - print(f"\nText: {gaming_text}") - print(f"Detected domains: {detected}") - - if detected and detected[0][0] == "gaming": - print("✓ Custom domain detected correctly") - else: - print("✗ Custom domain not detected") - - # List all domains - print(f"\nAll registered domains: {router.list_domains()}") - - -@pytest.mark.asyncio -async def test_domain_override(router: DomainRouter): - """Test forcing specific domain selection.""" - print("\n=== Testing Domain Override ===") - - text = "This is a general text without specific domain indicators" - - # Test without override - print(f"\nText: {text}") - detected = router.detect_domains(text) - print(f"Auto-detected domains: {detected[:3] if detected else 'None'}") - - # Test with override - for domain in ["technical", "creative", "medical"]: - try: - model = await router.route_by_domain(text, domain_override=domain) - print(f"\nForced domain: {domain}") - print(f"Selected model: {model.provider}:{model.name}") - print(f"Model domains: {model.capabilities.domains}") - except Exception as e: - print(f"\nForced domain: {domain}") - print(f"✗ Selection failed: {e}") - - -@pytest.mark.asyncio -async def test_multi_domain_handling(router: DomainRouter): - """Test handling of multi-domain content.""" - print("\n=== Testing Multi-Domain Content ===") - - # Text that spans multiple domains - multi_domain_text = """ - As a medical AI researcher, I'm analyzing patient data to validate - our hypothesis about a new treatment protocol. The results will be - submitted for peer review and publication in a scientific journal. - """ - - print(f"Multi-domain text: {multi_domain_text.strip()[:100]}...") - - # Detect all domains - detected = router.detect_domains(multi_domain_text, threshold=0.2) - print("\nDetected domains:") - for domain, confidence in detected: - print(f" - {domain}: {confidence:.2f}") - - # Route based on primary domain - try: - model = await router.route_by_domain(multi_domain_text) - print(f"\nSelected model: {model.provider}:{model.name}") - print(f"Model domains: {model.capabilities.domains}") - - # Check coverage - detected_names = [d[0] for d in detected] - covered = [d for d in detected_names if d in model.capabilities.domains] - print(f"Domain coverage: {len(covered)}/{len(detected_names)} domains covered") - - except Exception as e: - print(f"\n✗ Selection failed: {e}") - - -@pytest.mark.asyncio -async def test_real_generation_with_domain( - registry: ModelRegistry, router: DomainRouter -): - """Test real generation with domain-appropriate model.""" - print("\n=== Testing Real Generation with Domain Routing ===") - - # Different domain prompts - prompts = [ - { - "text": "Explain how machine learning works to a beginner", - "domain": "educational", - }, - {"text": "Write a haiku about artificial intelligence", "domain": "creative"}, - ] - - for prompt_info in prompts: - prompt = prompt_info["text"] - expected_domain = prompt_info["domain"] - - print(f"\nPrompt: {prompt}") - print(f"Expected domain: {expected_domain}") - - try: - # Select model based on domain - model = await router.route_by_domain(prompt) - print(f"Selected model: {model.provider}:{model.name}") - - # Generate response - response = await model.generate(prompt, temperature=0.7, max_tokens=100) - print(f"Response: {response.strip()[:150]}...") - - # Update metrics - registry.update_model_performance( - model, - success=True, - latency=0.5, - cost=0.0 if model.cost.is_free else 0.002) - - print("✓ Generation successful") - - except Exception as e: - print(f"✗ Generation failed: {e}") - - -# This file now uses pytest - no main function needed diff --git a/tests/test_huggingface_credentials.py b/tests/test_huggingface_credentials.py new file mode 100644 index 00000000..a03736c2 --- /dev/null +++ b/tests/test_huggingface_credentials.py @@ -0,0 +1,145 @@ +"""Credential resolution and secret-hygiene for the HuggingFace adapter. + +The token is a real secret. The tests that matter most here are the ones +asserting it never reaches a log line, a repr, or an exception message -- a +credential that leaks into a traceback ends up in CI logs and issue reports. +""" + +import logging + +import pytest + +from orchestrator.models.huggingface_credentials import ( + HF_TOKEN_ENV_VAR, + HuggingFaceCredentialError, + resolve_huggingface_api_key, +) + +pytestmark = pytest.mark.unit + +FAKE_KEY = "hf_0123456789abcdef0123456789abcdef" + + +@pytest.fixture(autouse=True) +def _isolate(monkeypatch, tmp_path): + """Never read the developer's real credential stores during tests.""" + monkeypatch.delenv(HF_TOKEN_ENV_VAR, raising=False) + monkeypatch.setattr( + "orchestrator.models.huggingface_credentials._ORCHESTRATOR_ENV_FILE", + tmp_path / "orchestrator.env", + ) + monkeypatch.setattr( + "orchestrator.models.huggingface_credentials._HF_CLI_TOKEN_FILE", + tmp_path / "hf-cli-token", + ) + + +# --------------------------------------------------------------------------- +# Resolution order +# --------------------------------------------------------------------------- + +def test_environment_variable_wins(monkeypatch, tmp_path): + (tmp_path / "orchestrator.env").write_text(f"{HF_TOKEN_ENV_VAR}=from-env-file\n") + (tmp_path / "hf-cli-token").write_text("from-cli-store\n") + monkeypatch.setenv(HF_TOKEN_ENV_VAR, FAKE_KEY) + + resolved = resolve_huggingface_api_key() + + assert resolved.key == FAKE_KEY + assert resolved.source == f"${HF_TOKEN_ENV_VAR}" + + +def test_orchestrator_env_file_is_second(tmp_path): + (tmp_path / "orchestrator.env").write_text( + f"# a comment\n\n{HF_TOKEN_ENV_VAR}={FAKE_KEY}\nOTHER=x\n" + ) + (tmp_path / "hf-cli-token").write_text("from-cli-store\n") + + resolved = resolve_huggingface_api_key() + + assert resolved.key == FAKE_KEY + assert "orchestrator.env" in resolved.source + + +def test_hf_cli_token_store_is_the_fallback(tmp_path): + """`hf auth login` is how HF users actually get a token; share that copy.""" + (tmp_path / "hf-cli-token").write_text(f"{FAKE_KEY}\n") + + resolved = resolve_huggingface_api_key() + + assert resolved.key == FAKE_KEY + assert "hf-cli-token" in resolved.source + + +def test_quoted_env_file_values_are_unwrapped(tmp_path): + (tmp_path / "orchestrator.env").write_text(f'{HF_TOKEN_ENV_VAR}="{FAKE_KEY}"\n') + assert resolve_huggingface_api_key().key == FAKE_KEY + + +def test_cli_store_tolerates_surrounding_whitespace(tmp_path): + (tmp_path / "hf-cli-token").write_text(f" {FAKE_KEY} \n") + assert resolve_huggingface_api_key().key == FAKE_KEY + + +def test_blank_environment_value_is_not_a_credential(monkeypatch, tmp_path): + """An exported-but-empty variable must not shadow a real stored key.""" + monkeypatch.setenv(HF_TOKEN_ENV_VAR, " ") + (tmp_path / "hf-cli-token").write_text(f"{FAKE_KEY}\n") + assert resolve_huggingface_api_key().key == FAKE_KEY + + +def test_empty_cli_store_is_not_a_credential(tmp_path): + (tmp_path / "hf-cli-token").write_text("\n") + assert resolve_huggingface_api_key(required=False) is None + + +def test_missing_credential_raises_with_actionable_guidance(): + with pytest.raises(HuggingFaceCredentialError) as excinfo: + resolve_huggingface_api_key() + message = str(excinfo.value) + assert HF_TOKEN_ENV_VAR in message + assert "huggingface.co/settings/tokens" in message, ( + "the error must say where to get a token" + ) + + +def test_optional_resolution_returns_none_instead_of_raising(): + assert resolve_huggingface_api_key(required=False) is None + + +def test_unreadable_env_file_does_not_crash_resolution(monkeypatch, tmp_path): + """A store that cannot be read falls through to the next source.""" + env_file = tmp_path / "orchestrator.env" + env_file.write_text(f"{HF_TOKEN_ENV_VAR}={FAKE_KEY}\n") + monkeypatch.setattr( + "orchestrator.models.huggingface_credentials._ORCHESTRATOR_ENV_FILE", + tmp_path / "missing-dir" / "orchestrator.env", + ) + (tmp_path / "hf-cli-token").write_text(f"{FAKE_KEY}\n") + assert resolve_huggingface_api_key().key == FAKE_KEY + + +# --------------------------------------------------------------------------- +# Secret hygiene -- the token must never be disclosed +# --------------------------------------------------------------------------- + +def test_repr_does_not_leak_the_key(tmp_path): + """A repr lands in tracebacks and pytest diffs, so it must be masked.""" + (tmp_path / "hf-cli-token").write_text(f"{FAKE_KEY}\n") + credential = resolve_huggingface_api_key() + assert FAKE_KEY not in repr(credential) + + +def test_resolution_does_not_log_the_key(tmp_path, caplog): + (tmp_path / "hf-cli-token").write_text(f"{FAKE_KEY}\n") + with caplog.at_level(logging.DEBUG): + resolve_huggingface_api_key() + assert FAKE_KEY not in caplog.text + + +def test_error_message_does_not_echo_a_partial_key(monkeypatch): + """Even a rejected value must not be quoted back into the message.""" + monkeypatch.setenv(HF_TOKEN_ENV_VAR, "") + with pytest.raises(HuggingFaceCredentialError) as excinfo: + resolve_huggingface_api_key() + assert FAKE_KEY not in str(excinfo.value) diff --git a/tests/test_huggingface_model.py b/tests/test_huggingface_model.py new file mode 100644 index 00000000..349fac49 --- /dev/null +++ b/tests/test_huggingface_model.py @@ -0,0 +1,1141 @@ +"""Unit and contract tests for the HuggingFace Inference API adapter. + +Hermetic throughout: the router is faked with deterministic stand-ins (a +recording ``_post``, a fake aiohttp session, a static catalog), per ADR 0001's +test-layer policy. Live acceptance lives in ``test_live_huggingface.py``. + +The wire contract under test (verified against the live router, 2026-08-21): + +- ``POST https://router.huggingface.co/v1/chat/completions`` with a + ``Bearer $HF_TOKEN`` header; OpenAI-compatible request/response. +- ``GET /v1/models`` lists chat models; each entry carries ``providers`` with + ``status`` (``live``/``error``), ``pricing`` in USD **per million** tokens, + an ``is_free`` promo flag, ``context_length`` and ``throughput``. +- A reasoning model that exhausts its budget returns ``content`` absent and + ``reasoning_content`` present with ``finish_reason: "length"`` -- observed + live with ``prism-ml/Ternary-Bonsai-27B-AWQ-4bit`` at ``max_tokens=32`` + (31 of 32 completion tokens were reasoning). That is truncation, not an + empty answer. +""" + +import asyncio +import json + +import pytest + +from orchestrator.core.model import ModelCapabilities, ModelCost +from orchestrator.models.huggingface_model import ( + DEFAULT_MAX_TOKENS, + DEFAULT_REQUEST_TIMEOUT_SECONDS, + HuggingFaceInferenceModel, + HuggingFaceModelError, + InsecureEndpoint, + ModelLoading, + ModelUnavailable, + PaidModelRefused, + PaymentRequired, + RateLimited, + ReasoningTruncated, + ReservedRequestField, + validate_base_url, +) + +pytestmark = pytest.mark.unit + +FAKE_KEY = "hf_0123456789abcdef0123456789abcdef" + +FREE = ModelCost(is_free=True) +PAID = ModelCost(input_cost_per_1k_tokens=0.001, output_cost_per_1k_tokens=0.002) + + +@pytest.fixture(autouse=True) +def _no_paid_optin(monkeypatch): + """The paid opt-in must never leak in from the developer's shell.""" + monkeypatch.delenv("ORCHESTRATOR_ALLOW_PAID_MODELS", raising=False) + + +def _free_model(**kwargs): + return HuggingFaceInferenceModel( + name="prism-ml/Ternary-Bonsai-27B-gguf", + api_key=FAKE_KEY, + cost=FREE, + **kwargs, + ) + + +# --------------------------------------------------------------------------- +# Cost policy -- unknown pricing is treated as paid, never as free +# --------------------------------------------------------------------------- + +def test_free_model_constructs_without_optin(): + assert _free_model().cost.is_free + + +def test_paid_model_is_refused_by_default(): + with pytest.raises(PaidModelRefused) as excinfo: + HuggingFaceInferenceModel(name="x/y", api_key=FAKE_KEY, cost=PAID) + assert "ORCHESTRATOR_ALLOW_PAID_MODELS" in str(excinfo.value) + + +def test_paid_model_allowed_with_explicit_optin(monkeypatch): + monkeypatch.setenv("ORCHESTRATOR_ALLOW_PAID_MODELS", "1") + model = HuggingFaceInferenceModel(name="x/y", api_key=FAKE_KEY, cost=PAID) + assert not model.cost.is_free + + +@pytest.mark.parametrize("value", ["0", "true", "yes", "2", ""]) +def test_only_the_exact_value_1_enables_paid_usage(monkeypatch, value): + monkeypatch.setenv("ORCHESTRATOR_ALLOW_PAID_MODELS", value) + with pytest.raises(PaidModelRefused): + HuggingFaceInferenceModel(name="x/y", api_key=FAKE_KEY, cost=PAID) + + +def test_unpriced_model_is_refused_rather_than_assumed_free(): + """A model built without catalog pricing has an *unknown* cost.""" + with pytest.raises(PaidModelRefused): + HuggingFaceInferenceModel(name="x/y", api_key=FAKE_KEY) + + +def test_unpriced_model_refuses_to_estimate_cost(monkeypatch): + """A zero-filled default would report $0.00 for a model that may bill.""" + monkeypatch.setenv("ORCHESTRATOR_ALLOW_PAID_MODELS", "1") + model = HuggingFaceInferenceModel(name="x/y", api_key=FAKE_KEY) + with pytest.raises(HuggingFaceModelError, match="without pricing"): + asyncio.run(model.estimate_cost("hello")) + + +def test_estimate_cost_is_exactly_zero_for_free_models(): + model = _free_model() + assert asyncio.run(model.estimate_cost("hello")) == 0.0 + + +def test_estimate_cost_uses_real_pricing_for_paid_models(monkeypatch): + monkeypatch.setenv("ORCHESTRATOR_ALLOW_PAID_MODELS", "1") + model = HuggingFaceInferenceModel(name="x/y", api_key=FAKE_KEY, cost=PAID) + estimate = asyncio.run(model.estimate_cost("hello", max_tokens=1000)) + assert estimate > 0.0 + + +# --------------------------------------------------------------------------- +# Catalog pricing -- the router reports USD per *million* tokens, per provider +# --------------------------------------------------------------------------- + +def _entry(model_id, providers, modalities=("text",)): + return { + "id": model_id, + "object": "model", + "created": 1, + "owned_by": model_id.split("/")[0], + "architecture": { + "input_modalities": list(modalities), + "output_modalities": ["text"], + }, + "providers": providers, + } + + +def _provider(name, *, status="live", pricing=..., is_free=..., throughput=None): + entry = {"provider": name, "status": status} + if pricing is not ...: + entry["pricing"] = pricing + if is_free is not ...: + entry["is_free"] = is_free + if throughput is not None: + entry["throughput"] = throughput + return entry + + +def test_zero_priced_live_provider_makes_the_model_free(): + from orchestrator.models.providers.huggingface_provider import ( + model_cost_from_catalog, + ) + + entry = _entry("a/b", [_provider("together", pricing={"input": 0, "output": 0})]) + assert model_cost_from_catalog(entry).is_free + + +def test_is_free_promo_flag_makes_the_model_free_without_pricing(): + from orchestrator.models.providers.huggingface_provider import ( + model_cost_from_catalog, + ) + + entry = _entry("a/b", [_provider("groq", is_free=True)]) + assert model_cost_from_catalog(entry).is_free + + +def test_priced_model_is_not_free_and_converts_per_million_to_per_1k(): + from orchestrator.models.providers.huggingface_provider import ( + model_cost_from_catalog, + ) + + entry = _entry( + "a/b", [_provider("novita", pricing={"input": 1.69, "output": 3.38})] + ) + cost = model_cost_from_catalog(entry) + assert not cost.is_free + assert cost.input_cost_per_1k_tokens == pytest.approx(0.00169) + assert cost.output_cost_per_1k_tokens == pytest.approx(0.00338) + + +def test_paid_cost_uses_the_most_expensive_live_route(): + """Routing is server-side; budgeting on the cheapest route understates.""" + from orchestrator.models.providers.huggingface_provider import ( + model_cost_from_catalog, + ) + + entry = _entry( + "a/b", + [ + _provider("cheap", pricing={"input": 1.0, "output": 2.0}), + _provider("dear", pricing={"input": 5.0, "output": 4.0}), + ], + ) + cost = model_cost_from_catalog(entry) + assert cost.input_cost_per_1k_tokens == pytest.approx(0.005) + assert cost.output_cost_per_1k_tokens == pytest.approx(0.004) + + +def test_unpriced_entry_is_treated_as_paid(): + """Absence of a price is not evidence of zero price.""" + from orchestrator.models.providers.huggingface_provider import ( + model_cost_from_catalog, + ) + + assert not model_cost_from_catalog(_entry("a/b", [_provider("novita")])).is_free + assert not model_cost_from_catalog(_entry("a/b", [])).is_free + + +def test_a_single_nonzero_price_makes_a_model_paid(): + from orchestrator.models.providers.huggingface_provider import ( + model_cost_from_catalog, + ) + + entry = _entry("a/b", [_provider("x", pricing={"input": 0, "output": 0.5})]) + assert not model_cost_from_catalog(entry).is_free + + +def test_providers_in_error_state_do_not_count(): + """A free route that is down must not mark the model free.""" + from orchestrator.models.providers.huggingface_provider import ( + model_cost_from_catalog, + ) + + entry = _entry( + "a/b", + [ + _provider("together", status="error", pricing={"input": 0, "output": 0}), + _provider("novita", pricing={"input": 1.0, "output": 1.0}), + ], + ) + assert not model_cost_from_catalog(entry).is_free + + +def test_free_route_is_pinned_to_the_free_provider(): + """An unpinned request routes :fastest -- which may be a paid provider.""" + from orchestrator.models.providers.huggingface_provider import ( + free_route_from_catalog, + ) + + entry = _entry( + "a/b", + [ + _provider("novita", pricing={"input": 1.0, "output": 1.0}), + _provider("together", pricing={"input": 0, "output": 0}), + ], + ) + assert free_route_from_catalog(entry) == "together" + + +def test_paid_model_has_no_free_route(): + from orchestrator.models.providers.huggingface_provider import ( + free_route_from_catalog, + ) + + entry = _entry("a/b", [_provider("novita", pricing={"input": 1, "output": 1})]) + assert free_route_from_catalog(entry) is None + + +def test_only_free_models_are_selected_for_registration(): + """A paid model registered as if free is the accident that costs money.""" + from orchestrator.models.providers.huggingface_provider import ( + free_models_from_catalog, + ) + + catalog = { + "free/zero": _entry( + "free/zero", [_provider("together", pricing={"input": 0, "output": 0})] + ), + "free/promo": _entry("free/promo", [_provider("groq", is_free=True)]), + "paid/model": _entry( + "paid/model", [_provider("novita", pricing={"input": 1, "output": 2})] + ), + "unknown/model": _entry("unknown/model", [_provider("novita")]), + } + + free = free_models_from_catalog(catalog) + + assert set(free) == {"free/zero", "free/promo"} + assert all(cost.is_free for cost in free.values()) + + +# --------------------------------------------------------------------------- +# Response extraction -- a reasoning model's scratchpad is not an answer +# --------------------------------------------------------------------------- + +def test_plain_content_is_returned(): + response = {"choices": [{"message": {"content": "pong"}}]} + assert ( + HuggingFaceInferenceModel._extract_text(response, "a/b") == "pong" + ) + + +def test_reasoning_model_content_is_preferred_over_scratchpad(): + response = { + "choices": [ + { + "message": {"content": "pong", "reasoning_content": "thinking..."}, + "finish_reason": "stop", + } + ] + } + assert HuggingFaceInferenceModel._extract_text(response, "a/b") == "pong" + + +def test_reasoning_truncation_raises_instead_of_returning_empty(): + """Observed live: content absent, reasoning_content present, length.""" + response = { + "choices": [ + { + "message": {"role": "assistant", "reasoning_content": "thinking..."}, + "finish_reason": "length", + } + ], + "usage": {"completion_tokens_details": {"reasoning_tokens": 31}}, + } + with pytest.raises(ReasoningTruncated) as excinfo: + HuggingFaceInferenceModel._extract_text(response, "a/b") + assert "max_tokens" in str(excinfo.value), "the error must name the fix" + + +def test_empty_response_without_reasoning_raises(): + response = {"choices": [{"message": {"content": ""}, "finish_reason": "stop"}]} + with pytest.raises(HuggingFaceModelError): + HuggingFaceInferenceModel._extract_text(response, "a/b") + + +def test_missing_choices_raises(): + with pytest.raises(HuggingFaceModelError): + HuggingFaceInferenceModel._extract_text({}, "a/b") + + +def test_default_max_tokens_is_large_enough_for_a_reasoning_model(): + """The budget must let a reasoning model finish thinking and still answer.""" + assert DEFAULT_MAX_TOKENS >= 1024 + + +def test_unavailable_loading_and_truncated_are_all_huggingface_errors(): + """Callers that only care 'did it fail' must still catch them.""" + assert issubclass(ModelUnavailable, HuggingFaceModelError) + assert issubclass(ModelLoading, ModelUnavailable) + assert issubclass(ReasoningTruncated, HuggingFaceModelError) + assert issubclass(RateLimited, HuggingFaceModelError) + assert issubclass(PaymentRequired, HuggingFaceModelError) + + +# --------------------------------------------------------------------------- +# Error classification -- which failures are transient and model-specific +# --------------------------------------------------------------------------- + +class _FakeResponse: + """A stand-in for an aiohttp response context manager.""" + + def __init__(self, status, body, headers=None): + self.status = status + self._body = body + self.headers = headers or {} + + async def text(self): + return self._body + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc_info): + return False + + +class _FakeSession: + """Serves canned responses (or raises) from ``model._post``'s level down.""" + + def __init__(self, *script): + # Each entry is a _FakeResponse or an Exception to raise. + self._script = list(script) + self.closed = False + + def post(self, url, json=None): + item = self._script.pop(0) + if isinstance(item, Exception): + raise item + return item + + async def close(self): + self.closed = True + + +def _model_with_session(session): + model = _free_model() + + async def get_session(): + return session + + model._get_session = get_session + return model + + +def test_503_model_loading_is_transient_and_carries_the_estimate(): + body = json.dumps( + {"error": "Model a/b is currently loading", "estimated_time": 42.5} + ) + model = _model_with_session(_FakeSession(_FakeResponse(503, body))) + + with pytest.raises(ModelLoading) as excinfo: + asyncio.run(model.generate("hi")) + assert excinfo.value.estimated_seconds == pytest.approx(42.5) + + +def test_503_without_a_loading_marker_is_a_plain_outage(): + model = _model_with_session(_FakeSession(_FakeResponse(503, "upstream error"))) + + with pytest.raises(ModelUnavailable) as excinfo: + asyncio.run(model.generate("hi")) + assert not isinstance(excinfo.value, ModelLoading) + + +def test_502_provider_error_is_an_outage(): + body = json.dumps({"error": {"message": "provider error", "type": "x"}}) + model = _model_with_session(_FakeSession(_FakeResponse(502, body))) + + with pytest.raises(ModelUnavailable): + asyncio.run(model.generate("hi")) + + +def test_429_is_rate_limited_and_reads_retry_after(): + model = _model_with_session( + _FakeSession(_FakeResponse(429, "too many requests", {"Retry-After": "17"})) + ) + + with pytest.raises(RateLimited) as excinfo: + asyncio.run(model.generate("hi")) + assert excinfo.value.retry_after == pytest.approx(17.0) + + +def test_429_without_a_header_has_no_estimate(): + model = _model_with_session(_FakeSession(_FakeResponse(429, "slow down"))) + + with pytest.raises(RateLimited) as excinfo: + asyncio.run(model.generate("hi")) + assert excinfo.value.retry_after is None + + +def test_402_is_payment_required_not_an_outage(): + """Observed live: depleted monthly credits return HTTP 402. Account-level, + so it must not be mistaken for a flapping endpoint or a bad request.""" + body = json.dumps( + {"error": "You have depleted your monthly included credits. Purchase " + "pre-paid credits to continue using Inference Providers."} + ) + model = _model_with_session(_FakeSession(_FakeResponse(402, body))) + + with pytest.raises(PaymentRequired) as excinfo: + asyncio.run(model.generate("hi")) + assert not isinstance(excinfo.value, (ModelUnavailable, RateLimited)) + + +def test_401_is_a_credential_error_not_a_transient_outage(): + """An invalid token must not be mistaken for a flapping endpoint.""" + body = json.dumps({"error": "Invalid username or password."}) + model = _model_with_session(_FakeSession(_FakeResponse(401, body))) + + with pytest.raises(HuggingFaceModelError) as excinfo: + asyncio.run(model.generate("hi")) + assert not isinstance(excinfo.value, (ModelUnavailable, RateLimited)) + + +def test_400_is_a_request_error_not_an_outage(): + body = json.dumps({"error": {"message": "bad request"}}) + model = _model_with_session(_FakeSession(_FakeResponse(400, body))) + + with pytest.raises(HuggingFaceModelError) as excinfo: + asyncio.run(model.generate("hi")) + assert not isinstance(excinfo.value, ModelUnavailable) + + +def test_transport_errors_are_retried_then_raised(): + import aiohttp + + model = _model_with_session( + _FakeSession(aiohttp.ClientError("reset"), aiohttp.ClientError("reset")) + ) + model._max_retries = 1 + model._retry_delay = 0 + + with pytest.raises(HuggingFaceModelError, match="after 2 attempts"): + asyncio.run(model.generate("hi")) + + +def test_a_transport_error_then_success_is_a_retry_not_a_failure(): + import aiohttp + + ok = _FakeResponse( + 200, json.dumps({"choices": [{"message": {"content": "pong"}}]}) + ) + model = _model_with_session(_FakeSession(aiohttp.ClientError("blip"), ok)) + model._retry_delay = 0 + + assert asyncio.run(model.generate("hi")) == "pong" + + +def test_model_outages_are_not_retried_in_place(): + """A downed backend stays down for minutes; generate_free moves on instead.""" + first = _FakeResponse(502, "provider error") + second = _FakeResponse( + 200, json.dumps({"choices": [{"message": {"content": "pong"}}]}) + ) + session = _FakeSession(first, second) + model = _model_with_session(session) + model._retry_delay = 0 + + with pytest.raises(ModelUnavailable): + asyncio.run(model.generate("hi")) + assert session._script == [second], "the second attempt must never happen" + + +def test_non_json_success_body_is_reported_not_misparsed(): + model = _model_with_session( + _FakeSession(_FakeResponse(200, "maintenance")) + ) + + with pytest.raises(HuggingFaceModelError, match="non-JSON"): + asyncio.run(model.generate("hi")) + + +def test_error_bodies_cannot_inject_newlines_into_logs(): + body = 'line one\r\nline two "quoted"\nline three' + model = _model_with_session(_FakeSession(_FakeResponse(400, body))) + + with pytest.raises(HuggingFaceModelError) as excinfo: + asyncio.run(model.generate("hi")) + message = str(excinfo.value) + assert "\n" not in message and "\r" not in message + + +def test_error_bodies_are_truncated(): + model = _model_with_session(_FakeSession(_FakeResponse(400, "x" * 5000))) + + with pytest.raises(HuggingFaceModelError) as excinfo: + asyncio.run(model.generate("hi")) + assert len(str(excinfo.value)) < 1000 + + +# --------------------------------------------------------------------------- +# Request-body integrity -- the cost gate is only worth as much as the field +# it checked, so `model` must survive to the wire unchanged +# --------------------------------------------------------------------------- + +def _free_model_recording_its_payload(**kwargs): + model = _free_model(**kwargs) + sent = {} + + async def record(path, payload): + sent.update(payload) + return {"choices": [{"message": {"content": "ok"}}]} + + model._post = record + return model, sent + + +@pytest.mark.parametrize("field", ["model", "messages", "stream"]) +def test_reserved_request_fields_cannot_be_overridden(field): + """Overriding `model` would swap an approved free model for a paid one.""" + model, _ = _free_model_recording_its_payload() + with pytest.raises(ReservedRequestField, match=field): + asyncio.run(model.generate("hi", **{field: "paid/model"})) + + +def test_the_checked_model_is_the_model_actually_sent(): + """The positive half: what the policy approved is what goes on the wire.""" + model, sent = _free_model_recording_its_payload() + asyncio.run(model.generate("hi")) + assert sent["model"] == model.name + + +def test_a_free_model_is_pinned_to_its_free_provider_on_the_wire(): + """Without the pin the router picks :fastest, which may bill.""" + model, sent = _free_model_recording_its_payload(route="together") + asyncio.run(model.generate("hi")) + assert sent["model"] == "prism-ml/Ternary-Bonsai-27B-gguf:together" + + +def test_a_model_without_a_route_sends_its_bare_id(): + model, sent = _free_model_recording_its_payload() + asyncio.run(model.generate("hi")) + assert ":" not in sent["model"] + + +def test_ordinary_sampling_kwargs_are_still_forwarded(): + model, sent = _free_model_recording_its_payload() + asyncio.run(model.generate("hi", top_p=0.9, frequency_penalty=0.5)) + assert sent["top_p"] == 0.9 + assert sent["frequency_penalty"] == 0.5 + assert sent["model"] == model.name, "controlled fields still win" + + +def test_system_prompt_becomes_a_system_message(): + model, sent = _free_model_recording_its_payload() + asyncio.run(model.generate("hi", system_prompt="be terse")) + assert sent["messages"][0] == {"role": "system", "content": "be terse"} + assert sent["messages"][1] == {"role": "user", "content": "hi"} + + +def test_structured_generation_also_refuses_a_model_override(): + model, _ = _free_model_recording_its_payload() + with pytest.raises(ReservedRequestField): + asyncio.run( + model.generate_structured( + "hi", schema={"type": "object"}, model="paid/model" + ) + ) + + +# --------------------------------------------------------------------------- +# Structured output must match the schema it asked for +# --------------------------------------------------------------------------- + +def _model_replying(text): + model = _free_model() + + async def reply(path, payload): + return {"choices": [{"message": {"content": text}}]} + + model._post = reply + return model + + +def test_structured_output_matching_the_schema_is_returned(): + model = _model_replying('{"city": "Paris", "country": "France"}') + schema = { + "type": "object", + "properties": {"city": {"type": "string"}, "country": {"type": "string"}}, + "required": ["city", "country"], + } + result = asyncio.run(model.generate_structured("capital of France", schema)) + assert result == {"city": "Paris", "country": "France"} + + +def test_structured_output_missing_a_required_key_is_rejected(): + model = _model_replying('{"city": "Paris"}') + schema = { + "type": "object", + "properties": {"city": {"type": "string"}, "country": {"type": "string"}}, + "required": ["city", "country"], + } + with pytest.raises(HuggingFaceModelError, match="schema"): + asyncio.run(model.generate_structured("capital of France", schema)) + + +def test_structured_output_that_is_not_json_is_rejected(): + model = _model_replying("I cannot help with that.") + with pytest.raises(HuggingFaceModelError, match="valid JSON"): + asyncio.run(model.generate_structured("hi", {"type": "object"})) + + +def test_a_fenced_reply_is_still_unwrapped_and_validated(): + model = _model_replying('```json\n{"a": 1}\n```') + schema = { + "type": "object", + "properties": {"a": {"type": "number"}}, + "required": ["a"], + } + assert asyncio.run(model.generate_structured("hi", schema)) == {"a": 1} + + +# --------------------------------------------------------------------------- +# Sessions -- a fallback chain must not pay a TLS handshake per attempt +# --------------------------------------------------------------------------- + +def test_aclose_is_idempotent_and_safe_before_any_request(): + model = _free_model() + + async def close_twice(): + await model.aclose() + await model.aclose() + + asyncio.run(close_twice()) + assert model._session is None + + +def test_session_is_reused_rather_than_rebuilt_per_request(): + """Constructing a session opens no connection, so this stays hermetic.""" + model = _free_model() + + async def run(): + first = await model._get_session() + second = await model._get_session() + assert first is second, "each request must not build a new session" + assert not first.closed + await model.aclose() + return first + + session = asyncio.run(run()) + assert session.closed, "aclose() must actually close the session" + + +def test_async_context_manager_closes_a_real_session(): + model = _free_model() + + async def use(): + async with model as m: + return await m._get_session() + + session = asyncio.run(use()) + assert session.closed, "leaving the context must release the connection" + assert model._session is None + + +def test_session_carries_the_bearer_token_and_json_content_type(): + model = _free_model() + + async def run(): + session = await model._get_session() + headers = session.headers + await model.aclose() + return headers + + headers = asyncio.run(run()) + assert headers["Authorization"] == f"Bearer {FAKE_KEY}" + assert headers["Content-Type"] == "application/json" + + +# --------------------------------------------------------------------------- +# Endpoint safety -- every request carries the bearer token +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize( + "url", + [ + "http://evil.example/v1", # plaintext to a remote host + "http://router.huggingface.co/v1", # a plausible typo of the real URL + "ftp://router.huggingface.co/v1", + "router.huggingface.co/v1", # no scheme at all + "https://", # no host + ], +) +def test_unsafe_base_urls_are_refused(url): + with pytest.raises(InsecureEndpoint): + validate_base_url(url) + + +@pytest.mark.parametrize( + "url", + [ + "https://router.huggingface.co/v1", + "http://localhost:8000/v1", + "http://127.0.0.1:8000", + "http://[::1]:8000", + ], +) +def test_https_and_loopback_are_accepted(url): + """Loopback stays usable so a local mock router remains testable.""" + assert validate_base_url(url) == url.rstrip("/") + + +def test_trailing_slash_is_normalised(): + assert validate_base_url("https://router.huggingface.co/v1/") == ( + "https://router.huggingface.co/v1" + ) + + +def test_model_refuses_to_construct_against_a_plaintext_endpoint(): + """The check must run at construction, before any token is sent.""" + with pytest.raises(InsecureEndpoint): + _free_model(base_url="http://evil.example/v1") + + +def test_provider_also_validates_its_endpoint(): + """The catalog fetch carries the token too, so it needs the same check.""" + from orchestrator.models.providers.base import ProviderConfig + from orchestrator.models.providers.huggingface_provider import ( + HuggingFaceProvider, + ) + + with pytest.raises(InsecureEndpoint): + HuggingFaceProvider( + ProviderConfig(name="huggingface", base_url="http://evil.example/v1") + ) + + +# --------------------------------------------------------------------------- +# Provider behaviour +# --------------------------------------------------------------------------- + +def _stuffed_provider(catalog): + """A provider with a hand-loaded catalog and no network.""" + from orchestrator.models.providers.base import ProviderConfig + from orchestrator.models.providers.huggingface_provider import ( + HuggingFaceProvider, + model_cost_from_catalog, + ) + + provider = HuggingFaceProvider.__new__(HuggingFaceProvider) + provider.config = ProviderConfig(name="huggingface", api_key=FAKE_KEY) + provider._base_url = "http://localhost:9/v1" # loopback; never contacted + provider._catalog = catalog + provider._costs = {m: model_cost_from_catalog(e) for m, e in catalog.items()} + return provider + + +def test_provider_transport_config_reaches_the_model(): + """Regression guard: the config must not be silently ignored.""" + from orchestrator.models.providers.base import ProviderConfig + from orchestrator.models.providers.huggingface_provider import ( + HuggingFaceProvider, + ) + + provider = HuggingFaceProvider( + ProviderConfig( + name="huggingface", api_key=FAKE_KEY, timeout=12.5, + max_retries=7, retry_delay=0.25, + ) + ) + entry = _entry("a/b", [_provider("x", pricing={"input": 0, "output": 0})]) + provider._catalog = {"a/b": entry} + provider._costs = {"a/b": ModelCost(is_free=True)} + + model = asyncio.run(provider.create_model("a/b")) + assert model._timeout == 12.5 + assert model._max_retries == 7 + assert model._retry_delay == 0.25 + + +def test_default_provider_timeout_suits_generation_not_metadata(): + """ProviderConfig's 30s default would cut off a slow generation.""" + from orchestrator.models.providers.huggingface_provider import ( + HuggingFaceProvider, + ) + + provider = HuggingFaceProvider() + assert provider.config.timeout == DEFAULT_REQUEST_TIMEOUT_SECONDS + assert provider.config.timeout > 30.0 + + +def test_create_model_unknown_id_names_what_is_available(): + provider = _stuffed_provider( + {"a/b": _entry("a/b", [_provider("x", pricing={"input": 0, "output": 0})])} + ) + with pytest.raises(HuggingFaceModelError, match="a/b"): + asyncio.run(provider.create_model("not/there")) + + +def test_create_model_pins_the_free_route(): + provider = _stuffed_provider( + { + "a/b": _entry( + "a/b", + [ + _provider("novita", pricing={"input": 1, "output": 1}), + _provider("together", pricing={"input": 0, "output": 0}), + ], + ) + } + ) + model = asyncio.run(provider.create_model("a/b")) + assert model.route == "together" + + +def test_create_model_for_an_unpriced_entry_marks_pricing_unknown(monkeypatch): + """Unknown price must not become a zero-filled, confidently wrong cost.""" + monkeypatch.setenv("ORCHESTRATOR_ALLOW_PAID_MODELS", "1") + provider = _stuffed_provider( + {"a/b": _entry("a/b", [_provider("novita")])} # live but unpriced + ) + + model = asyncio.run(provider.create_model("a/b")) + + assert not model._pricing_is_known + with pytest.raises(HuggingFaceModelError, match="without pricing"): + asyncio.run(model.estimate_cost("hello")) + + +def test_create_model_for_an_unpriced_entry_is_refused_without_optin(): + provider = _stuffed_provider({"a/b": _entry("a/b", [_provider("novita")])}) + with pytest.raises(PaidModelRefused): + asyncio.run(provider.create_model("a/b")) + + +def test_free_preference_orders_by_probed_throughput_and_drops_nothing(): + """No hard-coded model list: the catalog's probe data ranks the free set.""" + provider = _stuffed_provider( + { + "slow/model": _entry( + "slow/model", + [_provider("x", pricing={"input": 0, "output": 0}, throughput=10.0)], + ), + "fast/model": _entry( + "fast/model", + [_provider("y", pricing={"input": 0, "output": 0}, throughput=99.0)], + ), + "unprobed/model": _entry( + "unprobed/model", [_provider("z", pricing={"input": 0, "output": 0})] + ), + } + ) + + ordered = provider.free_models_by_preference() + + assert ordered[0] == "fast/model" + assert ordered[-1] == "unprobed/model", "unprobed models sort last" + assert set(ordered) == set(provider._catalog), "no model may be dropped" + + +def test_capabilities_come_from_the_catalog_entry(): + provider = _stuffed_provider( + { + "v/m": _entry( + "v/m", + [ + _provider("x", pricing={"input": 0, "output": 0}), + ], + modalities=("text", "image"), + ) + } + ) + provider._catalog["v/m"]["providers"][0]["context_length"] = 131072 + provider._catalog["v/m"]["providers"][0]["supports_structured_output"] = True + + caps = provider.get_model_capabilities("v/m") + assert isinstance(caps, ModelCapabilities) + assert caps.vision_capable + assert caps.context_window == 131072 + assert caps.supports_structured_output + + +def test_generate_free_falls_back_past_loading_models(): + """A cold-starting model must not strand the caller.""" + from orchestrator.models.providers.huggingface_provider import ( + HuggingFaceProvider, + ) + + provider = HuggingFaceProvider.__new__(HuggingFaceProvider) + provider._catalog = {"a/loading": {}, "b/ready": {}} + provider._costs = {m: ModelCost(is_free=True) for m in provider._catalog} + built = [] + + async def fake_create(model_id, **kwargs): + model = HuggingFaceInferenceModel( + name=model_id, api_key=FAKE_KEY, cost=ModelCost(is_free=True) + ) + if model_id == "a/loading": + async def loading(path, payload): + raise ModelLoading("loading", estimated_seconds=30.0) + + model._post = loading + else: + async def ok(path, payload): + return {"choices": [{"message": {"content": "pong"}}]} + + model._post = ok + built.append(model) + return model + + provider.create_model = fake_create + + text, model_id = asyncio.run(provider.generate_free("hi")) + assert text == "pong" + assert model_id == "b/ready" + assert all(m._session is None for m in built), "every attempt must be closed" + + +def test_generate_free_moves_past_a_truncated_reasoning_model(): + """A reasoning model that spent its budget thinking is not an answer.""" + from orchestrator.models.providers.huggingface_provider import ( + HuggingFaceProvider, + ) + + provider = HuggingFaceProvider.__new__(HuggingFaceProvider) + provider._catalog = {"a/thinker": {}, "b/plain": {}} + provider._costs = {m: ModelCost(is_free=True) for m in provider._catalog} + + async def fake_create(model_id, **kwargs): + model = HuggingFaceInferenceModel( + name=model_id, api_key=FAKE_KEY, cost=ModelCost(is_free=True) + ) + + async def truncated(path, payload): + raise ReasoningTruncated("spent the budget thinking") + + async def ok(path, payload): + return {"choices": [{"message": {"content": "pong"}}]} + + model._post = truncated if model_id == "a/thinker" else ok + return model + + provider.create_model = fake_create + + text, model_id = asyncio.run(provider.generate_free("hi")) + assert (text, model_id) == ("pong", "b/plain") + + +@pytest.mark.parametrize( + "account_level_error", + [ + RateLimited("slow down", retry_after=60.0), + PaymentRequired("monthly credits depleted"), + ], +) +def test_generate_free_does_not_walk_past_account_level_errors( + account_level_error, +): + """429/402 are account-level: trying every other model just hammers the + same quota or bills the same empty balance.""" + from orchestrator.models.providers.huggingface_provider import ( + HuggingFaceProvider, + ) + + provider = HuggingFaceProvider.__new__(HuggingFaceProvider) + provider._catalog = {"a/limited": {}, "b/never-tried": {}} + provider._costs = {m: ModelCost(is_free=True) for m in provider._catalog} + tried = [] + + async def fake_create(model_id, **kwargs): + tried.append(model_id) + model = HuggingFaceInferenceModel( + name=model_id, api_key=FAKE_KEY, cost=ModelCost(is_free=True) + ) + + async def limited(path, payload): + raise account_level_error + + model._post = limited + return model + + provider.create_model = fake_create + + with pytest.raises(type(account_level_error)): + asyncio.run(provider.generate_free("hi")) + assert tried == ["a/limited"], "no second model may be attempted" + + +def test_generate_free_raises_when_every_candidate_is_down(): + from orchestrator.models.providers.huggingface_provider import ( + HuggingFaceProvider, + ) + + provider = HuggingFaceProvider.__new__(HuggingFaceProvider) + provider._catalog = {"a/down": {}, "b/down": {}} + provider._costs = {m: ModelCost(is_free=True) for m in provider._catalog} + + async def fake_create(model_id, **kwargs): + model = HuggingFaceInferenceModel( + name=model_id, api_key=FAKE_KEY, cost=ModelCost(is_free=True) + ) + + async def down(path, payload): + raise ModelUnavailable(f"{model_id} backend is down") + + model._post = down + return model + + provider.create_model = fake_create + + with pytest.raises(HuggingFaceModelError, match="no free HuggingFace model"): + asyncio.run(provider.generate_free("hi")) + + +def test_generate_free_with_an_empty_free_set_raises(): + from orchestrator.models.providers.huggingface_provider import ( + HuggingFaceProvider, + ) + + provider = HuggingFaceProvider.__new__(HuggingFaceProvider) + provider._catalog = {} + provider._costs = {} + + with pytest.raises(HuggingFaceModelError): + asyncio.run(provider.generate_free("hi")) + + +# --------------------------------------------------------------------------- +# Registry integration -- only free models may be registered +# --------------------------------------------------------------------------- + +def _seal_hf_credentials(monkeypatch, tmp_path): + monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.setattr( + "orchestrator.models.huggingface_credentials._ORCHESTRATOR_ENV_FILE", + tmp_path / "nope.env", + ) + monkeypatch.setattr( + "orchestrator.models.huggingface_credentials._HF_CLI_TOKEN_FILE", + tmp_path / "nope-token", + ) + + +def test_hf_registry_population_is_skipped_without_a_credential( + monkeypatch, tmp_path +): + """No credential means no HuggingFace models -- and no network call.""" + from orchestrator._api import _register_free_huggingface_models + from orchestrator.models.model_registry import ModelRegistry + + _seal_hf_credentials(monkeypatch, tmp_path) + + registry = ModelRegistry() + assert _register_free_huggingface_models(registry) == 0 + assert registry.list_models() == [] + + +def test_hf_registry_population_survives_an_unreachable_router(monkeypatch): + """An outage must degrade to 'no HuggingFace models', not break startup.""" + from orchestrator import _api + from orchestrator.models.model_registry import ModelRegistry + + monkeypatch.setenv("HF_TOKEN", FAKE_KEY) + + def unreachable(*args, **kwargs): + raise OSError("Name or service not known") + + monkeypatch.setattr( + "orchestrator.models.providers.huggingface_provider.fetch_catalog_sync", + unreachable, + ) + + registry = ModelRegistry() + assert _api._register_free_huggingface_models(registry) == 0 + + +def test_hf_registry_population_registers_only_free_models(monkeypatch): + from orchestrator import _api + from orchestrator.models.model_registry import ModelRegistry + + monkeypatch.setenv("HF_TOKEN", FAKE_KEY) + catalog = { + "free/zero": _entry( + "free/zero", [_provider("together", pricing={"input": 0, "output": 0})] + ), + "paid/model": _entry( + "paid/model", [_provider("novita", pricing={"input": 1, "output": 2})] + ), + } + monkeypatch.setattr( + "orchestrator.models.providers.huggingface_provider.fetch_catalog_sync", + lambda *args, **kwargs: catalog, + ) + + registry = ModelRegistry() + registered = _api._register_free_huggingface_models(registry) + + assert registered == 1 + assert registry.list_models() == ["huggingface:free/zero"] + model = registry.models["huggingface:free/zero"] + assert model.cost.is_free + assert model.route == "together", "the registered model keeps its free pin" diff --git a/tests/test_intelligent_routing.py b/tests/test_intelligent_routing.py deleted file mode 100644 index 18a1816c..00000000 --- a/tests/test_intelligent_routing.py +++ /dev/null @@ -1,261 +0,0 @@ -#!/usr/bin/env python3 -"""Test intelligent model routing with real models.""" - -import os -import pytest - -from orchestrator.models.model_registry import ModelRegistry -from orchestrator.models.model_selector import ModelSelector, ModelSelectionCriteria -from orchestrator.models.openai_model import OpenAIModel -from orchestrator.models.anthropic_model import AnthropicModel -from orchestrator.integrations.ollama_model import OllamaModel -from orchestrator.core.model import ModelCost - -from tests.test_infrastructure import create_test_orchestrator, TestModel, TestProvider - - -async def setup_test_registry() -> ModelRegistry: - """Set up a model registry with test models.""" - registry = ModelRegistry() - - # Register local models (Ollama) - free models - try: - # Small fast model - llama_small = OllamaModel( - model_name="llama3.2:1b", base_url="http://localhost:11434" - ) - llama_small.capabilities.domains = ["general"] - llama_small.capabilities.speed_rating = "fast" - llama_small.capabilities.accuracy_score = 0.75 - llama_small.cost = ModelCost(is_free=True) - llama_small._size_billions = 1.0 - registry.register_model(llama_small) - print("✓ Registered llama3.2:1b") - except Exception as e: - print(f"✗ Failed to register llama3.2:1b: {e}") - - try: - # Medium general model - llama_medium = OllamaModel( - model_name="llama3.1:8b", base_url="http://localhost:11434" - ) - llama_medium.capabilities.domains = ["general", "technical"] - llama_medium.capabilities.speed_rating = "medium" - llama_medium.capabilities.accuracy_score = 0.85 - llama_medium.cost = ModelCost(is_free=True) - llama_medium._size_billions = 8.0 - registry.register_model(llama_medium) - print("✓ Registered llama3.1:8b") - except Exception as e: - print(f"✗ Failed to register llama3.1:8b: {e}") - - # Register API models if keys are available - if os.getenv("OPENAI_API_KEY"): - try: - # GPT-3.5 - fast and cheap - gpt35 = OpenAIModel("gpt-3.5-turbo") - registry.register_model(gpt35) - print("✓ Registered gpt-3.5-turbo") - - # GPT-4 - powerful but expensive - gpt4 = OpenAIModel("gpt-4") - registry.register_model(gpt4) - print("✓ Registered gpt-4") - except Exception as e: - print(f"✗ Failed to register OpenAI models: {e}") - - if os.getenv("ANTHROPIC_API_KEY"): - try: - # Claude models - claude = AnthropicModel("claude-3-sonnet-20240229") - claude.capabilities.domains = ["general", "technical", "creative"] - claude.capabilities.accuracy_score = 0.9 - claude.capabilities.speed_rating = "medium" - registry.register_model(claude) - print("✓ Registered claude-3-sonnet") - except Exception as e: - print(f"✗ Failed to register Anthropic models: {e}") - - return registry - - -@pytest.fixture -async def registry(): - """Create test registry fixture.""" - return await setup_test_registry() - - -@pytest.mark.asyncio -async def test_basic_selection(registry: ModelRegistry): - """Test basic model selection.""" - print("\n=== Testing Basic Model Selection ===") - - selector = ModelSelector(registry) - - # Test 1: Select a fast model - print("\n1. Selecting a fast model:") - criteria = ModelSelectionCriteria( - speed_preference="fast", selection_strategy="performance_optimized" - ) - - try: - model = await selector.select_model(criteria) - print(f" Selected: {model.provider}:{model.name}") - print(f" Speed: {model.capabilities.speed_rating}") - print(f" Cost: Free={model.cost.is_free}") - except Exception as e: - print(f" Error: {e}") - - # Test 2: Select a free model - print("\n2. Selecting a free model:") - criteria = ModelSelectionCriteria( - prefer_free_models=True, selection_strategy="cost_optimized" - ) - - try: - model = await selector.select_model(criteria) - print(f" Selected: {model.provider}:{model.name}") - print(f" Free: {model.cost.is_free}") - except Exception as e: - print(f" Error: {e}") - - # Test 3: Select an accurate model - print("\n3. Selecting an accurate model:") - criteria = ModelSelectionCriteria( - min_accuracy_score=0.9, selection_strategy="accuracy_optimized" - ) - - try: - model = await selector.select_model(criteria) - print(f" Selected: {model.provider}:{model.name}") - print(f" Accuracy: {model.capabilities.accuracy_score}") - except Exception as e: - print(f" Error: {e}") - - -@pytest.mark.asyncio -async def test_auto_tag_parsing(registry: ModelRegistry): - """Test AUTO tag parsing.""" - print("\n=== Testing AUTO Tag Parsing ===") - - selector = ModelSelector(registry) - - # Test various AUTO tags - auto_tags = [ - "Select a fast model for quick responses", - "Choose the best model for code generation", - "Pick a cost-effective model for general chat", - "Select an accurate model for technical analysis", - "Choose a model that can handle 32k context", - ] - - for i, auto_tag in enumerate(auto_tags, 1): - print(f"\n{i}. AUTO: {auto_tag}") - - try: - model = await selector.select_model(ModelSelectionCriteria(), auto_tag) - print(f" Selected: {model.provider}:{model.name}") - print( - f" Capabilities: speed={model.capabilities.speed_rating}, accuracy={model.capabilities.accuracy_score}" - ) - except Exception as e: - print(f" Error: {e}") - - -@pytest.mark.asyncio -async def test_capability_matching(registry: ModelRegistry): - """Test capability-based selection.""" - print("\n=== Testing Capability Matching ===") - - selector = ModelSelector(registry) - - # Test 1: Code-specialized model - print("\n1. Selecting code-specialized model:") - criteria = ModelSelectionCriteria( - required_capabilities=["code"], required_tasks=["code", "generate"] - ) - - try: - model = await selector.select_model(criteria) - print(f" Selected: {model.provider}:{model.name}") - print(f" Code specialized: {model.capabilities.code_specialized}") - except Exception as e: - print(f" Error: {e}") - - # Test 2: Function calling model - print("\n2. Selecting model with function calling:") - criteria = ModelSelectionCriteria(required_capabilities=["tools"]) - - try: - model = await selector.select_model(criteria) - print(f" Selected: {model.provider}:{model.name}") - print(f" Supports functions: {model.capabilities.supports_function_calling}") - except Exception as e: - print(f" Error: {e}") - - -@pytest.mark.asyncio -async def test_real_generation(registry: ModelRegistry): - """Test actual generation with selected models.""" - print("\n=== Testing Real Generation ===") - - selector = ModelSelector(registry) - - # Select a fast free model for testing - criteria = ModelSelectionCriteria( - prefer_free_models=True, - speed_preference="fast", - selection_strategy="cost_optimized") - - try: - model = await selector.select_model(criteria) - print(f"\nSelected model: {model.provider}:{model.name}") - - # Test generation - prompt = "What is 2+2? Give a one word answer." - print(f"Prompt: {prompt}") - - response = await model.generate(prompt, temperature=0) - print(f"Response: {response.strip()}") - - # Update metrics based on success - registry.update_model_performance( - model, success=True, latency=0.5, cost=0.0 if model.cost.is_free else 0.001 - ) - - print("✓ Generation successful, metrics updated") - - except Exception as e: - print(f"✗ Generation failed: {e}") - - -@pytest.mark.asyncio -async def test_cost_calculation(registry: ModelRegistry): - """Test cost calculation for different models.""" - print("\n=== Testing Cost Calculation ===") - - available_models = await registry.get_available_models() - - for model_key in available_models[:3]: # Test first 3 models - # Parse provider and model name correctly - parts = model_key.split(":", 1) - if len(parts) == 2: - provider, model_name = parts - else: - provider = "" - model_name = model_key - - model = registry.get_model(model_name, provider) - - print(f"\nModel: {model_key}") - print(f" Free: {model.cost.is_free}") - - if not model.cost.is_free: - # Calculate cost for 1000 input + 500 output tokens - cost = model.cost.calculate_cost(1000, 500) - print(f" Cost for 1K input + 500 output tokens: ${cost:.4f}") - print(f" Input rate: ${model.cost.input_cost_per_1k_tokens}/1K tokens") - print(f" Output rate: ${model.cost.output_cost_per_1k_tokens}/1K tokens") - - -# This file now uses pytest - no main function needed diff --git a/tests/test_intelligent_routing_comprehensive.py b/tests/test_intelligent_routing_comprehensive.py deleted file mode 100644 index f9269616..00000000 --- a/tests/test_intelligent_routing_comprehensive.py +++ /dev/null @@ -1,454 +0,0 @@ -#!/usr/bin/env python3 -"""Comprehensive test of intelligent model routing with real API calls.""" - -import asyncio -import os -import time -import json -import pytest - -from orchestrator.models.model_registry import ModelRegistry -from orchestrator.models.model_selector import ModelSelector, ModelSelectionCriteria -from orchestrator.models.load_balancer import LoadBalancer, ModelPoolConfig -from orchestrator.models.domain_router import DomainRouter -from orchestrator.models.openai_model import OpenAIModel -from orchestrator.integrations.ollama_model import OllamaModel -from orchestrator.models.anthropic_model import AnthropicModel - -from tests.test_infrastructure import create_test_orchestrator, TestModel, TestProvider - - -async def setup_comprehensive_registry(): - """Set up registry with all available models.""" - registry = ModelRegistry() - registry.enable_auto_registration() - - models_added = [] - - # Local models (Ollama) - ollama_models = [ - ("llama3.2:1b", ["general", "fast"], 0.7, "fast", 1.0), - ("llama3.1:8b", ["general", "reasoning"], 0.85, "medium", 8.0), - ("mistral:7b", ["general", "code"], 0.8, "medium", 7.0), - ] - - for model_name, domains, accuracy, speed, size in ollama_models: - try: - model = OllamaModel(model_name) - model.capabilities.domains = domains - model.capabilities.accuracy_score = accuracy - model.capabilities.speed_rating = speed - model._expertise = domains - model._size_billions = size - registry.register_model(model) - models_added.append(f"ollama:{model_name}") - print(f"✓ Registered {model_name}") - except Exception as e: - print(f"✗ Failed to register {model_name}: {e}") - - # OpenAI models - if os.getenv("OPENAI_API_KEY"): - openai_models = [ - ("gpt-3.5-turbo", ["general", "code", "creative"], 0.85, "fast", True), - ( - "gpt-4", - ["general", "reasoning", "code", "technical", "medical", "legal"], - 0.95, - "medium", - True), - ] - - for model_name, domains, accuracy, speed, is_code_specialized in openai_models: - try: - model = OpenAIModel(model_name) - model.capabilities.domains = domains - model.capabilities.accuracy_score = accuracy - model.capabilities.speed_rating = speed - model.capabilities.code_specialized = is_code_specialized - model._expertise = domains - registry.register_model(model) - models_added.append(f"openai:{model_name}") - print(f"✓ Registered {model_name}") - except Exception as e: - print(f"✗ Failed to register {model_name}: {e}") - - # Anthropic models - if os.getenv("ANTHROPIC_API_KEY"): - try: - claude = AnthropicModel("claude-3-sonnet-20240229") - claude.capabilities.domains = ["general", "reasoning", "code", "creative"] - claude.capabilities.accuracy_score = 0.9 - claude.capabilities.speed_rating = "medium" - claude.capabilities.code_specialized = True - claude._expertise = claude.capabilities.domains - registry.register_model(claude) - models_added.append("anthropic:claude-3-sonnet-20240229") - print("✓ Registered claude-3-sonnet") - except Exception as e: - print(f"✗ Failed to register Claude: {e}") - - return registry, models_added - - -@pytest.fixture -async def registry(): - """Create test registry fixture.""" - registry, models = await setup_comprehensive_registry() - return registry - - -@pytest.mark.asyncio -async def test_auto_tag_routing(registry: ModelRegistry): - """Test AUTO tag model selection with real generation.""" - print("\n=== Testing AUTO Tag Model Selection ===") - - selector = ModelSelector(registry) - - test_cases = [ - { - "auto_tag": "Select a fast, cost-effective model for simple text generation", - "prompt": "What is the capital of France?", - "expected_type": "fast/cheap", - }, - { - "auto_tag": "Choose the best model for complex code generation with high accuracy", - "prompt": "Write a Python function to calculate fibonacci numbers recursively", - "expected_type": "code/accurate", - }, - { - "auto_tag": "Pick a creative model for storytelling", - "prompt": "Write the opening line of a mystery novel", - "expected_type": "creative", - }, - { - "auto_tag": "Select an accurate model for technical analysis requiring 8k context", - "prompt": "Explain the concept of quantum entanglement", - "expected_type": "technical/accurate", - }, - ] - - for i, test in enumerate(test_cases, 1): - print(f"\n{i}. AUTO: {test['auto_tag']}") - print(f" Prompt: {test['prompt'][:50]}...") - - try: - # Select model using AUTO tag - criteria = ModelSelectionCriteria() - model = await selector.select_model(criteria, test["auto_tag"]) - - print(f" Selected: {model.provider}:{model.name}") - print( - f" Properties: speed={model.capabilities.speed_rating}, " - + f"accuracy={model.capabilities.accuracy_score}, " - + f"cost={'free' if model.cost.is_free else 'paid'}" - ) - - # Generate response - start_time = time.time() - response = await model.generate( - test["prompt"], temperature=0.7, max_tokens=50 - ) - latency = time.time() - start_time - - print(f" Response: {response.strip()[:100]}...") - print(f" Latency: {latency:.2f}s") - - # Update metrics - registry.update_model_performance(model, success=True, latency=latency) - - except Exception as e: - print(f" ✗ Failed: {e}") - - -@pytest.mark.asyncio -async def test_cost_optimized_routing(registry: ModelRegistry): - """Test cost-optimized model selection.""" - print("\n=== Testing Cost-Optimized Routing ===") - - selector = ModelSelector(registry) - - # Generate 10 simple prompts - prompts = [ - "What is 2+2?", - "Name a color", - "What day comes after Monday?", - "Is water wet?", - "What's the opposite of hot?", - "Count to 5", - "Name a fruit", - "What's 10 minus 3?", - "Is the sky blue?", - "What's the first letter of the alphabet?", - ] - - total_cost = 0.0 - models_used = {} - - for prompt in prompts: - try: - # Select cost-optimized model - criteria = ModelSelectionCriteria( - prefer_free_models=True, - selection_strategy="cost_optimized", - max_cost_per_1k_tokens=0.01) - - model = await selector.select_model(criteria) - model_key = f"{model.provider}:{model.name}" - models_used[model_key] = models_used.get(model_key, 0) + 1 - - # Generate response - response = await model.generate(prompt, temperature=0, max_tokens=10) - - # Calculate cost - if not model.cost.is_free: - # Estimate tokens - input_tokens = len(prompt.split()) * 1.5 # Rough estimate - output_tokens = len(response.split()) * 1.5 - cost = model.cost.calculate_cost(int(input_tokens), int(output_tokens)) - total_cost += cost - - except Exception as e: - print(f"Failed on '{prompt}': {e}") - - print(f"\nResults for {len(prompts)} prompts:") - print(f"Total estimated cost: ${total_cost:.4f}") - print("Models used:") - for model_key, count in models_used.items(): - print(f" {model_key}: {count} times") - - -@pytest.mark.asyncio -async def test_domain_specific_generation(registry: ModelRegistry): - """Test domain-specific routing with real generation.""" - print("\n=== Testing Domain-Specific Generation ===") - - router = DomainRouter(registry) - - domain_prompts = [ - { - "prompt": "Explain the symptoms and treatment for pneumonia", - "expected_domain": "medical", - }, - { - "prompt": "Draft a non-disclosure agreement template", - "expected_domain": "legal", - }, - { - "prompt": "Write a function to sort an array using quicksort", - "expected_domain": "technical/code", - }, - {"prompt": "Compose a haiku about the seasons", "expected_domain": "creative"}, - { - "prompt": "Explain photosynthesis to a 10-year-old", - "expected_domain": "educational", - }, - ] - - for test in domain_prompts: - print(f"\n--- {test['expected_domain'].upper()} Domain ---") - print(f"Prompt: {test['prompt']}") - - try: - # Detect domain - domains = router.detect_domains(test["prompt"]) - print( - f"Detected: {', '.join([f'{d[0]} ({d[1]:.2f})' for d in domains[:2]])}" - ) - - # Route and generate - model = await router.route_by_domain(test["prompt"]) - print(f"Model: {model.provider}:{model.name}") - - response = await model.generate( - test["prompt"], temperature=0.7, max_tokens=100 - ) - print(f"Response: {response.strip()[:150]}...") - - # Verify domain coverage - if domains and domains[0][0] in model.capabilities.domains: - print("✓ Model has appropriate domain expertise") - - except Exception as e: - print(f"✗ Failed: {e}") - - -@pytest.mark.asyncio -async def test_load_balanced_generation(registry: ModelRegistry): - """Test load balancing with real concurrent requests.""" - print("\n=== Testing Load-Balanced Generation ===") - - load_balancer = LoadBalancer(registry) - - # Configure pools - primary_pool = ModelPoolConfig( - models=[ - {"model": "ollama:llama3.2:1b", "weight": 0.6, "max_concurrent": 3}, - {"model": "ollama:llama3.1:8b", "weight": 0.4, "max_concurrent": 2}, - ] - ) - - # Add API models if available - api_models = [] - if "openai:gpt-3.5-turbo" in [ - f"{m.provider}:{m.name}" for m in registry.models.values() - ]: - api_models.append( - {"model": "openai:gpt-3.5-turbo", "weight": 0.7, "max_concurrent": 5} - ) - - if api_models: - api_pool = ModelPoolConfig(models=api_models, fallback_pool="local") - load_balancer.configure_pool("api", api_pool) - - load_balancer.configure_pool("local", primary_pool) - - # Launch concurrent requests - async def make_request(i: int, pool: str): - try: - model = await load_balancer.select_from_pool(pool) - prompt = f"Generate a random number between 1 and 100. (Request {i})" - - result = await load_balancer.execute_with_retry( - model, "generate", prompt, temperature=1.0, max_tokens=20 - ) - - return { - "request": i, - "model": f"{model.provider}:{model.name}", - "response": result.strip()[:50], - } - except Exception as e: - return {"request": i, "error": str(e)} - - # Test with local pool - print("\nTesting with local models (5 concurrent requests):") - tasks = [make_request(i, "local") for i in range(5)] - results = await asyncio.gather(*tasks) - - for result in results: - if "error" in result: - print(f" Request {result['request']}: Failed - {result['error']}") - else: - print( - f" Request {result['request']}: {result['model']} -> {result['response']}" - ) - - # Show pool statistics - stats = load_balancer.get_pool_status("local") - print("\nLocal Pool Statistics:") - for model_stat in stats["models"]: - print( - f" {model_stat['model']}: " - + f"{model_stat['successful_requests']}/{model_stat['total_requests']} successful, " - + f"avg latency: {model_stat['avg_latency']:.2f}s" - ) - - -@pytest.mark.asyncio -async def test_model_performance_tracking(registry: ModelRegistry): - """Test model performance tracking over multiple requests.""" - print("\n=== Testing Model Performance Tracking ===") - - selector = ModelSelector(registry) - - # Make multiple requests and track performance - prompts = [ - "What is machine learning?", - "Explain recursion in simple terms", - "What are the primary colors?", - "How does photosynthesis work?", - "What is the speed of light?", - ] - - print("\nMaking requests and tracking performance...") - - for i, prompt in enumerate(prompts): - try: - # Select model with balanced strategy - criteria = ModelSelectionCriteria(selection_strategy="balanced") - model = await selector.select_model(criteria) - - # Generate and time response - start = time.time() - response = await model.generate(prompt, temperature=0.5, max_tokens=50) - latency = time.time() - start - - # Update metrics - success = len(response.strip()) > 0 - cost = 0.001 if not model.cost.is_free else 0.0 - - registry.update_model_performance( - model, success=success, latency=latency, cost=cost - ) - - print( - f"{i+1}. {model.provider}:{model.name} - {latency:.2f}s - {'✓' if success else '✗'}" - ) - - except Exception as e: - print(f"{i+1}. Failed: {e}") - - # Show final statistics - print("\n=== Final Model Statistics ===") - stats = registry.get_model_statistics() - - print(f"Total models: {stats['total_models']}") - print(f"Healthy models: {stats['healthy_models']}") - - print("\nModel Performance:") - for model_key, perf in stats["selection_stats"]["model_performance"].items(): - if perf["attempts"] > 0: - print(f"\n{model_key}:") - print(f" Attempts: {perf['attempts']}") - print(f" Success rate: {perf['success_rate']:.1%}") - print(f" Avg reward: {perf['average_reward']:.3f}") - - -@pytest.mark.asyncio -async def test_failover_scenario(registry: ModelRegistry): - """Test failover when primary models fail.""" - print("\n=== Testing Failover Scenario ===") - - selector = ModelSelector(registry) - - # Try to select a model with impossible requirements first - print("1. Testing with impossible requirements (should failover):") - try: - criteria = ModelSelectionCriteria( - min_context_window=1000000, - min_accuracy_score=0.99, # 1M context (impossible) - ) - model = await selector.select_model(criteria) - print(f" Unexpected success: {model.provider}:{model.name}") - except Exception as e: - print(f" Expected failure: {e}") - - # Now with relaxed requirements - print("\n2. Testing with relaxed requirements:") - try: - criteria = ModelSelectionCriteria( - min_context_window=4096, min_accuracy_score=0.7 - ) - model = await selector.select_model(criteria) - print(f" Success: Selected {model.provider}:{model.name}") - - # Test generation - response = await model.generate("Hello, how are you?", max_tokens=20) - print(f" Response: {response.strip()}") - - except Exception as e: - print(f" Failed: {e}") - - -async def save_test_results(results: dict): - """Save test results to a file.""" - timestamp = time.strftime("%Y%m%d_%H%M%S") - filename = f"test_results_{timestamp}.json" - - with open(filename, "w") as f: - json.dump(results, f, indent=2) - - print(f"\nTest results saved to {filename}") - - -# This file now uses pytest - no main function needed diff --git a/tests/test_langchain_anthropic_integration.py b/tests/test_langchain_anthropic_integration.py deleted file mode 100644 index c0c00851..00000000 --- a/tests/test_langchain_anthropic_integration.py +++ /dev/null @@ -1,277 +0,0 @@ -"""Real integration tests for LangChain-enhanced Anthropic model.""" - -import pytest -import asyncio -import os - -from orchestrator.models.anthropic_model import AnthropicModel -from orchestrator.utils.api_keys_flexible import load_api_keys_optional - -from tests.test_infrastructure import create_test_orchestrator, TestModel, TestProvider - - -class TestLangChainAnthropicIntegration: - """Real integration tests for LangChain-enhanced Anthropic models.""" - - @pytest.mark.asyncio - async def test_anthropic_langchain_fallback_behavior(self): - """Test that Anthropic model gracefully handles LangChain availability.""" - - # Test with LangChain explicitly disabled - model = AnthropicModel("claude-3-haiku", use_langchain=False) - assert model._use_langchain is False - assert model.client is not None # Direct Anthropic client should be initialized - assert model.provider == "anthropic" - assert model.name == "claude-3-haiku" - - # Test that LangChain can be enabled when available - model_with_langchain = AnthropicModel("claude-3-haiku", use_langchain=True) - # Should either use LangChain (if available) or fall back to direct Anthropic - assert model_with_langchain.provider == "anthropic" - assert model_with_langchain.name == "claude-3-haiku" - - @pytest.mark.asyncio - async def test_anthropic_model_initialization_preserves_interface(self): - """Test that enhanced Anthropic model preserves existing interface.""" - - # Test with LangChain disabled - model = AnthropicModel("claude-3-sonnet", use_langchain=False) - - # Verify all existing attributes are preserved - assert hasattr(model, 'capabilities') - assert hasattr(model, 'requirements') - assert hasattr(model, 'cost') - assert hasattr(model, '_model_id') - assert hasattr(model, '_expertise') - assert hasattr(model, '_size_billions') - - # Verify model metadata - assert model.provider == "anthropic" - assert model.name == "claude-3-sonnet" - assert model.capabilities.supports_function_calling - assert model.capabilities.supports_structured_output - assert not model.cost.is_free - - def test_anthropic_model_capabilities_unchanged(self): - """Test that model capabilities detection is unchanged.""" - - # Test Claude Opus capabilities - model = AnthropicModel("claude-3-opus", use_langchain=False) - assert "reasoning" in model.capabilities.supported_tasks - assert "creative" in model.capabilities.supported_tasks - assert "vision" in model.capabilities.supported_tasks - assert model.capabilities.context_window == 200000 - assert model.capabilities.vision_capable - - # Test Claude Haiku capabilities - model = AnthropicModel("claude-3-haiku", use_langchain=False) - assert "code" in model.capabilities.supported_tasks - assert model.capabilities.context_window == 200000 - assert model.capabilities.vision_capable - assert model.capabilities.speed_rating == "fast" - - def test_anthropic_model_cost_estimation_unchanged(self): - """Test that cost estimation logic is preserved.""" - - # Test Claude Opus pricing - model = AnthropicModel("claude-3-opus", use_langchain=False) - assert model.cost.input_cost_per_1k_tokens == 0.015 # $15 per 1M = $0.015 per 1K - assert model.cost.output_cost_per_1k_tokens == 0.075 # $75 per 1M = $0.075 per 1K - - # Test Claude Haiku pricing - model = AnthropicModel("claude-3-haiku", use_langchain=False) - assert model.cost.input_cost_per_1k_tokens == 0.00025 # $0.25 per 1M - assert model.cost.output_cost_per_1k_tokens == 0.00125 # $1.25 per 1M - - @pytest.mark.asyncio - async def test_anthropic_model_methods_preserve_interface(self): - """Test that all model methods preserve their interface.""" - - model = AnthropicModel("claude-3-haiku", use_langchain=False) - - # Test method signatures haven't changed - assert hasattr(model, 'generate') - assert hasattr(model, 'generate_structured') - assert hasattr(model, 'health_check') - assert hasattr(model, 'estimate_cost') - - # Test that methods can be called without errors (with API key issues handled gracefully) - try: - await model.health_check() - # If we reach here, health check worked (API key available) - except (ValueError, RuntimeError) as e: - # Expected if no API key available - assert "API key" in str(e) or "Anthropic" in str(e) - - @pytest.mark.asyncio - async def test_existing_anthropic_compatibility(self): - """Test that existing Anthropic model code continues to work.""" - - # Test that existing initialization patterns work - model = AnthropicModel( - name="claude-3-haiku", - api_key=os.getenv("ANTHROPIC_API_KEY", "dummy-key"), - use_langchain=False - ) - - assert model.name == "claude-3-haiku" - assert model.provider == "anthropic" - - # Test that all existing attributes are accessible - assert hasattr(model, 'capabilities') - assert hasattr(model, 'requirements') - assert hasattr(model, 'cost') - assert hasattr(model, 'metrics') - - def test_anthropic_model_name_normalization(self): - """Test that model name normalization works correctly.""" - - # Test various Claude model name variations - test_cases = [ - ("claude-3-opus", "claude-3-opus-20240229"), - ("claude-3-sonnet", "claude-3-sonnet-20240229"), - ("claude-3-haiku", "claude-3-haiku-20240307"), - ("claude-3.5-sonnet", "claude-3-5-sonnet-20241022"), - ("claude-instant", "claude-instant-1.2"), - ] - - for input_name, expected_normalized in test_cases: - model = AnthropicModel(input_name, use_langchain=False) - assert model._model_id == expected_normalized - - def test_anthropic_model_expertise_detection(self): - """Test model expertise detection.""" - - # Test Opus expertise - model = AnthropicModel("claude-3-opus", use_langchain=False) - expertise = model._expertise - assert "reasoning" in expertise - assert "research" in expertise - assert "math" in expertise - - # Test Haiku expertise - model = AnthropicModel("claude-3-haiku", use_langchain=False) - expertise = model._expertise - assert "general" in expertise - assert "chat" in expertise - - @pytest.mark.asyncio - async def test_real_anthropic_generation_compatibility(self): - """Test real Anthropic API calls work the same with enhanced model.""" - - # Check if Anthropic API key is available using our key management system - available_keys = load_api_keys_optional() - if not available_keys.get("anthropic"): - pytest.skip("Anthropic API key not available") - """Test real Anthropic API calls work the same with enhanced model.""" - - # Test with LangChain disabled (original behavior) - model_direct = AnthropicModel("claude-3-haiku", use_langchain=False) - - try: - response_direct = await model_direct.generate( - "What is 2+2? Respond with just the number.", - temperature=0.0, - max_tokens=10 - ) - assert len(response_direct) > 0 - assert "4" in response_direct - - # Test health check - health = await model_direct.health_check() - assert health is True - - # Test cost estimation - cost = await model_direct.estimate_cost("Test prompt", 100) - assert cost > 0 - - except Exception as e: - pytest.skip(f"Anthropic API test failed (possibly rate limited): {e}") - - @pytest.mark.asyncio - async def test_structured_output_compatibility(self): - """Test structured output generation works correctly.""" - - # Check if Anthropic API key is available using our key management system - available_keys = load_api_keys_optional() - if not available_keys.get("anthropic"): - pytest.skip("Anthropic API key not available") - """Test structured output generation works correctly.""" - - model = AnthropicModel("claude-3-haiku", use_langchain=False) - - schema = { - "type": "object", - "properties": { - "result": {"type": "number"}, - "explanation": {"type": "string"} - }, - "required": ["result", "explanation"] - } - - try: - response = await model.generate_structured( - "What is 2+2?", - schema=schema, - temperature=0.0 - ) - - assert isinstance(response, dict) - assert "result" in response - assert "explanation" in response - assert response["result"] == 4 - - except Exception as e: - pytest.skip(f"Anthropic structured output test failed: {e}") - - def test_anthropic_model_size_estimation(self): - """Test model size estimation.""" - - size_tests = [ - ("claude-3-opus", 175.0), - ("claude-3-sonnet", 70.0), - ("claude-3-haiku", 20.0), - ("claude-instant", 10.0), - ] - - for model_name, expected_size in size_tests: - model = AnthropicModel(model_name, use_langchain=False) - assert model._size_billions == expected_size - - @pytest.mark.asyncio - async def test_anthropic_system_prompt_support(self): - """Test that system prompts are handled correctly.""" - - model = AnthropicModel("claude-3-haiku", use_langchain=False) - - # Test with API key available - try: - if os.getenv("ANTHROPIC_API_KEY"): - response = await model.generate( - "What is your name?", - system_prompt="You are a helpful assistant named Claude.", - temperature=0.0, - max_tokens=20 - ) - assert len(response) > 0 - # System prompt should influence the response - assert "claude" in response.lower() or "assistant" in response.lower() - else: - # Test that the method signature accepts system_prompt - # (will fail with API key error but signature is correct) - try: - await model.generate( - "Test", - system_prompt="Test system", - temperature=0.0, - max_tokens=5 - ) - except (ValueError, RuntimeError): - # Expected without API key - pass - except Exception as e: - pytest.skip(f"System prompt test failed: {e}") - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) \ No newline at end of file diff --git a/tests/test_langchain_migration_comprehensive.py b/tests/test_langchain_migration_comprehensive.py deleted file mode 100644 index 6f220439..00000000 --- a/tests/test_langchain_migration_comprehensive.py +++ /dev/null @@ -1,381 +0,0 @@ -"""Comprehensive integration tests for the LangChain model migration - Issue #202.""" - -import pytest -import asyncio -import os -from unittest.mock import patch - -from orchestrator.models.openai_model import OpenAIModel -from orchestrator.models.anthropic_model import AnthropicModel -from orchestrator.models.langchain_adapter import LangChainModelAdapter -from orchestrator.utils.auto_install import PACKAGE_MAPPINGS -from orchestrator.utils.api_keys_flexible import load_api_keys_optional - -from tests.test_infrastructure import create_test_orchestrator, TestModel, TestProvider - - -class TestLangChainMigrationComprehensive: - """Comprehensive tests for LangChain provider migration.""" - - def test_langchain_package_mappings_complete(self): - """Test that all required LangChain packages are mapped.""" - - required_mappings = { - "langchain_openai": "langchain-openai", - "langchain_anthropic": "langchain-anthropic", - "langchain_google_genai": "langchain-google-genai", - "langchain_community": "langchain-community", - "langchain_huggingface": "langchain-huggingface", - } - - for import_name, pip_name in required_mappings.items(): - assert import_name in PACKAGE_MAPPINGS - assert PACKAGE_MAPPINGS[import_name] == pip_name - - @pytest.mark.asyncio - async def test_enhanced_models_preserve_all_interfaces(self): - """Test that enhanced models preserve all existing interfaces.""" - - models_to_test = [ - ("openai", OpenAIModel, "gpt-3.5-turbo"), - ("anthropic", AnthropicModel, "claude-3-haiku"), - ] - - for provider, model_class, model_name in models_to_test: - # Test with LangChain disabled to verify fallback works - model = model_class(model_name, use_langchain=False) - - # Verify all required attributes exist - assert hasattr(model, 'name') - assert hasattr(model, 'provider') - assert hasattr(model, 'capabilities') - assert hasattr(model, 'requirements') - assert hasattr(model, 'cost') - assert hasattr(model, 'metrics') - - # Verify all required methods exist - assert hasattr(model, 'generate') - assert hasattr(model, 'generate_structured') - assert hasattr(model, 'health_check') - assert hasattr(model, 'estimate_cost') - - # Verify provider and name are correct - assert model.provider == provider - assert model.name == model_name - - @pytest.mark.asyncio - async def test_langchain_adapter_all_providers(self): - """Test LangChainModelAdapter with all supported providers.""" - - providers_to_test = [ - ("openai", "gpt-3.5-turbo"), - ("anthropic", "claude-3-haiku"), - ("google", "gemini-pro"), - ("ollama", "llama3.2:1b"), - ("huggingface", "microsoft/DialoGPT-small"), - ] - - for provider, model_name in providers_to_test: - try: - adapter = LangChainModelAdapter(provider, model_name) - - # Verify basic properties - assert adapter.provider == provider - assert adapter.name == model_name - assert hasattr(adapter, 'capabilities') - assert hasattr(adapter, 'cost') - - # Verify methods exist - assert hasattr(adapter, 'generate') - assert hasattr(adapter, 'health_check') - - except Exception as e: - # Expected for providers without API keys or Ollama not running - if "API key" not in str(e) and "Ollama" not in str(e) and "Failed to install" not in str(e): - raise e - - @pytest.mark.asyncio - async def test_backward_compatibility_no_breaking_changes(self): - """Test that existing code patterns continue to work.""" - - # Test existing OpenAI model usage patterns - try: - openai_model = OpenAIModel( - name="gpt-3.5-turbo", - api_key=os.getenv("OPENAI_API_KEY", "dummy-key") - ) - - # These should all work without modification - assert openai_model.name == "gpt-3.5-turbo" - assert openai_model.provider == "openai" - assert openai_model.capabilities.supports_function_calling - - except ValueError as e: - # Expected if no API key - assert "API key" in str(e) - - # Test existing Anthropic model usage patterns - try: - anthropic_model = AnthropicModel( - name="claude-3-haiku", - api_key=os.getenv("ANTHROPIC_API_KEY", "dummy-key") - ) - - # These should all work without modification - assert anthropic_model.name == "claude-3-haiku" - assert anthropic_model.provider == "anthropic" - assert anthropic_model.capabilities.supports_structured_output - - except ValueError as e: - # Expected if no API key - assert "API key" in str(e) - - def test_cost_analysis_preserved(self): - """Test that cost analysis functionality is fully preserved.""" - - # Test OpenAI cost analysis - openai_model = OpenAIModel("gpt-4-turbo", use_langchain=False) - assert openai_model.cost.input_cost_per_1k_tokens > 0 - assert openai_model.cost.output_cost_per_1k_tokens > 0 - assert not openai_model.cost.is_free - - # Test cost calculation - cost = openai_model.cost.calculate_cost(1000, 1000) - assert cost > 0 - - # Test Anthropic cost analysis - anthropic_model = AnthropicModel("claude-3-opus", use_langchain=False) - assert anthropic_model.cost.input_cost_per_1k_tokens > 0 - assert anthropic_model.cost.output_cost_per_1k_tokens > 0 - assert not anthropic_model.cost.is_free - - # Test cost efficiency calculation - efficiency = anthropic_model.cost.get_cost_efficiency_score(0.9) - assert efficiency > 0 - - def test_capability_detection_enhanced(self): - """Test that capability detection is preserved and enhanced.""" - - test_cases = [ - ("openai", OpenAIModel, "gpt-4-turbo", ["code", "reasoning", "creative"]), - ("openai", OpenAIModel, "gpt-3.5-turbo", ["code", "chat"]), - ("anthropic", AnthropicModel, "claude-3-opus", ["reasoning", "creative", "vision"]), - ("anthropic", AnthropicModel, "claude-3-haiku", ["code", "chat", "vision"]), - ] - - for provider, model_class, model_name, expected_tasks in test_cases: - model = model_class(model_name, use_langchain=False) - - # Check that expected capabilities are detected - for task in expected_tasks: - assert task in model.capabilities.supported_tasks - - # Check advanced capability flags - if "gpt-4" in model_name or "opus" in model_name or "sonnet" in model_name: - assert model.capabilities.supports_function_calling - assert model.capabilities.supports_structured_output - - # Check vision capabilities for Claude 3 models - if "claude-3" in model_name: - assert model.capabilities.vision_capable - - @pytest.mark.asyncio - async def test_fallback_behavior_robust(self): - """Test robust fallback behavior when LangChain is unavailable.""" - - # Test OpenAI fallback - openai_model = OpenAIModel("gpt-3.5-turbo", use_langchain=False) - assert not openai_model._use_langchain - assert openai_model.client is not None - - # Test Anthropic fallback - anthropic_model = AnthropicModel("claude-3-haiku", use_langchain=False) - assert not anthropic_model._use_langchain - assert anthropic_model.client is not None - - # Test that all methods are still callable - for model in [openai_model, anthropic_model]: - try: - await model.health_check() - except (ValueError, RuntimeError) as e: - # Expected without valid API keys - assert "API key" in str(e) or model.provider.title() in str(e) - - def test_model_metadata_preservation(self): - """Test that all model metadata is preserved.""" - - # Test OpenAI metadata - openai_model = OpenAIModel("gpt-4-turbo", use_langchain=False) - assert hasattr(openai_model, '_model_id') - assert hasattr(openai_model, '_expertise') - assert hasattr(openai_model, '_size_billions') - assert openai_model._expertise is not None - assert openai_model._size_billions > 0 - - # Test Anthropic metadata - anthropic_model = AnthropicModel("claude-3-opus", use_langchain=False) - assert hasattr(anthropic_model, '_model_id') - assert hasattr(anthropic_model, '_expertise') - assert hasattr(anthropic_model, '_size_billions') - assert anthropic_model._expertise is not None - assert anthropic_model._size_billions > 0 - - @pytest.mark.asyncio - async def test_api_key_handling_enhanced(self): - """Test enhanced API key handling using existing infrastructure.""" - - # Test that models use existing API key infrastructure - from orchestrator.utils.api_keys_flexible import load_api_keys_optional - - available_keys = load_api_keys_optional() - - # Test OpenAI key handling - if "openai" in available_keys: - model = OpenAIModel("gpt-3.5-turbo") - assert model.api_key == available_keys["openai"] - - # Test Anthropic key handling - if "anthropic" in available_keys: - model = AnthropicModel("claude-3-haiku") - assert model.api_key == available_keys["anthropic"] - - def test_model_serialization_compatibility(self): - """Test that model serialization/deserialization still works.""" - - # Test OpenAI model serialization - openai_model = OpenAIModel("gpt-3.5-turbo", use_langchain=False) - model_dict = openai_model.to_dict() - - assert "name" in model_dict - assert "provider" in model_dict - assert "capabilities" in model_dict - assert "requirements" in model_dict - assert "cost" in model_dict - assert "metrics" in model_dict - - # Test Anthropic model serialization - anthropic_model = AnthropicModel("claude-3-haiku", use_langchain=False) - model_dict = anthropic_model.to_dict() - - assert "name" in model_dict - assert "provider" in model_dict - assert model_dict["provider"] == "anthropic" - - @pytest.mark.asyncio - async def test_error_handling_robust(self): - """Test robust error handling in all scenarios.""" - - # Test invalid model names - try: - invalid_model = OpenAIModel("invalid-model-name", use_langchain=False) - # Should still create the model, just with different capabilities - assert invalid_model.name == "invalid-model-name" - except Exception: - # Any exception should be meaningful - pass - - # Test that error handling is preserved in enhanced models - # The enhanced models should handle errors the same way as original models - - # Test that models gracefully handle generation errors - model = OpenAIModel("gpt-3.5-turbo", use_langchain=False) - try: - # This might work if API key is available, or fail gracefully if not - await model.health_check() - except Exception as e: - # Should be a meaningful error message - assert len(str(e)) > 0 - - @pytest.mark.asyncio - async def test_cross_provider_consistency(self): - """Test consistency between different providers for same tasks.""" - - # Check if API keys are available using our key management system - available_keys = load_api_keys_optional() - if not (available_keys.get("openai") and available_keys.get("anthropic")): - pytest.skip("Both OpenAI and Anthropic API keys needed for cross-provider test") - """Test consistency between different providers for same tasks.""" - - openai_model = OpenAIModel("gpt-3.5-turbo") - anthropic_model = AnthropicModel("claude-3-haiku") - - test_prompt = "What is 2+2? Respond with just the number." - - try: - # Test both models with same prompt - openai_response = await openai_model.generate(test_prompt, temperature=0.0, max_tokens=5) - anthropic_response = await anthropic_model.generate(test_prompt, temperature=0.0, max_tokens=5) - - # Both should contain "4" - assert "4" in openai_response - assert "4" in anthropic_response - - # Both should be healthy - assert await openai_model.health_check() - assert await anthropic_model.health_check() - - except Exception as e: - pytest.skip(f"Cross-provider test failed (API issues): {e}") - - def test_phase1_completion_criteria(self): - """Verify that Phase 1 completion criteria are met.""" - - # ✅ Auto-install system extended - assert "langchain_openai" in PACKAGE_MAPPINGS - assert "langchain_anthropic" in PACKAGE_MAPPINGS - - # ✅ LangChainModelAdapter created and functional - adapter = LangChainModelAdapter("openai", "gpt-3.5-turbo") - assert adapter.provider == "openai" - - # ✅ OpenAI model enhanced with LangChain support - openai_model = OpenAIModel("gpt-3.5-turbo", use_langchain=True) - assert hasattr(openai_model, '_use_langchain') - assert hasattr(openai_model, 'langchain_model') - - # ✅ Anthropic model enhanced with LangChain support - anthropic_model = AnthropicModel("claude-3-haiku", use_langchain=True) - assert hasattr(anthropic_model, '_use_langchain') - assert hasattr(anthropic_model, 'langchain_model') - - # ✅ All existing interfaces preserved - for model in [openai_model, anthropic_model]: - assert hasattr(model, 'generate') - assert hasattr(model, 'generate_structured') - assert hasattr(model, 'health_check') - assert hasattr(model, 'estimate_cost') - - # ✅ No breaking changes - # All existing initialization patterns still work - # All existing method signatures unchanged - # All existing capabilities preserved - - def test_migration_success_metrics(self): - """Test that migration success metrics are met.""" - - # Performance: No degradation (models initialize successfully) - openai_model = OpenAIModel("gpt-3.5-turbo", use_langchain=False) - anthropic_model = AnthropicModel("claude-3-haiku", use_langchain=False) - - assert openai_model.name == "gpt-3.5-turbo" - assert anthropic_model.name == "claude-3-haiku" - - # Reliability: Robust fallback behavior - assert not openai_model._use_langchain # Falls back when langchain=False - assert not anthropic_model._use_langchain - - # Compatibility: All existing attributes accessible - for model in [openai_model, anthropic_model]: - assert hasattr(model, 'capabilities') - assert hasattr(model, 'cost') - assert hasattr(model, 'requirements') - - # Functionality: Cost tracking preserved - assert not openai_model.cost.is_free - assert not anthropic_model.cost.is_free - assert openai_model.cost.calculate_cost(1000, 1000) > 0 - assert anthropic_model.cost.calculate_cost(1000, 1000) > 0 - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) \ No newline at end of file diff --git a/tests/test_langchain_openai_integration.py b/tests/test_langchain_openai_integration.py deleted file mode 100644 index 35b7ab0f..00000000 --- a/tests/test_langchain_openai_integration.py +++ /dev/null @@ -1,209 +0,0 @@ -"""Real integration tests for LangChain-enhanced OpenAI model.""" - -import pytest -import asyncio -from unittest.mock import patch -import os - -from orchestrator.models.openai_model import OpenAIModel -from orchestrator.utils.api_keys_flexible import load_api_keys_optional - -from tests.test_infrastructure import create_test_orchestrator, TestModel, TestProvider - - -class TestLangChainOpenAIIntegration: - """Real integration tests for LangChain-enhanced OpenAI models.""" - - @pytest.mark.asyncio - async def test_openai_langchain_fallback_behavior(self): - """Test that OpenAI model gracefully handles LangChain availability.""" - - # Test with LangChain explicitly disabled - model = OpenAIModel("gpt-3.5-turbo", use_langchain=False) - assert model._use_langchain is False - assert model.client is not None # Direct OpenAI client should be initialized - assert model.provider == "openai" - assert model.name == "gpt-3.5-turbo" - - # Test that LangChain can be enabled when available - model_with_langchain = OpenAIModel("gpt-3.5-turbo", use_langchain=True) - # Should either use LangChain (if available) or fall back to direct OpenAI - assert model_with_langchain.provider == "openai" - assert model_with_langchain.name == "gpt-3.5-turbo" - - @pytest.mark.asyncio - async def test_openai_model_initialization_preserves_interface(self): - """Test that enhanced OpenAI model preserves existing interface.""" - - # Test with LangChain disabled - model = OpenAIModel("gpt-3.5-turbo", use_langchain=False) - - # Verify all existing attributes are preserved - assert hasattr(model, 'capabilities') - assert hasattr(model, 'requirements') - assert hasattr(model, 'cost') - assert hasattr(model, '_model_id') - assert hasattr(model, '_expertise') - assert hasattr(model, '_size_billions') - - # Verify model metadata - assert model.provider == "openai" - assert model.name == "gpt-3.5-turbo" - assert model.capabilities.supports_function_calling - assert model.capabilities.supports_structured_output - assert not model.cost.is_free - - def test_openai_model_capabilities_unchanged(self): - """Test that model capabilities detection is unchanged.""" - - # Test GPT-4 capabilities - model = OpenAIModel("gpt-4-turbo", use_langchain=False) - assert "reasoning" in model.capabilities.supported_tasks - assert "creative" in model.capabilities.supported_tasks - assert model.capabilities.context_window == 128000 - assert model.capabilities.vision_capable - - # Test GPT-3.5 capabilities - model = OpenAIModel("gpt-3.5-turbo", use_langchain=False) - assert "code" in model.capabilities.supported_tasks - assert model.capabilities.context_window == 4096 - assert not model.capabilities.vision_capable - - def test_openai_model_cost_estimation_unchanged(self): - """Test that cost estimation logic is preserved.""" - - # Test GPT-4 pricing - model = OpenAIModel("gpt-4-turbo", use_langchain=False) - assert model.cost.input_cost_per_1k_tokens == 0.01 - assert model.cost.output_cost_per_1k_tokens == 0.03 - - # Test GPT-3.5 pricing - model = OpenAIModel("gpt-3.5-turbo", use_langchain=False) - assert model.cost.input_cost_per_1k_tokens == 0.0005 - assert model.cost.output_cost_per_1k_tokens == 0.0015 - - @pytest.mark.asyncio - async def test_openai_model_methods_preserve_interface(self): - """Test that all model methods preserve their interface.""" - - model = OpenAIModel("gpt-3.5-turbo", use_langchain=False) - - # Test method signatures haven't changed - assert hasattr(model, 'generate') - assert hasattr(model, 'generate_structured') - assert hasattr(model, 'health_check') - assert hasattr(model, 'estimate_cost') - - # Test that methods can be called without errors (with API key issues handled gracefully) - try: - await model.health_check() - # If we reach here, health check worked (API key available) - except (ValueError, RuntimeError) as e: - # Expected if no API key available - assert "API key" in str(e) or "OpenAI" in str(e) - - @pytest.mark.asyncio - async def test_existing_openai_compatibility(self): - """Test that existing OpenAI model code continues to work.""" - - # Test that existing initialization patterns work - model = OpenAIModel( - name="gpt-3.5-turbo", - api_key=os.getenv("OPENAI_API_KEY", "dummy-key"), - use_langchain=False - ) - - assert model.name == "gpt-3.5-turbo" - assert model.provider == "openai" - - # Test that all existing attributes are accessible - assert hasattr(model, 'capabilities') - assert hasattr(model, 'requirements') - assert hasattr(model, 'cost') - assert hasattr(model, 'metrics') - - def test_langchain_package_mapping_exists(self): - """Test that LangChain packages are mapped in auto_install.""" - - from orchestrator.utils.auto_install import PACKAGE_MAPPINGS - - # Verify LangChain packages are mapped - assert "langchain_openai" in PACKAGE_MAPPINGS - assert PACKAGE_MAPPINGS["langchain_openai"] == "langchain-openai" - assert "langchain_anthropic" in PACKAGE_MAPPINGS - assert PACKAGE_MAPPINGS["langchain_anthropic"] == "langchain-anthropic" - assert "langchain_community" in PACKAGE_MAPPINGS - assert PACKAGE_MAPPINGS["langchain_community"] == "langchain-community" - - @pytest.mark.asyncio - async def test_real_openai_generation_compatibility(self): - """Test real OpenAI API calls work the same with enhanced model.""" - - # Check if OpenAI API key is available using our key management system - available_keys = load_api_keys_optional() - if not available_keys.get("openai"): - pytest.skip("OpenAI API key not available") - """Test real OpenAI API calls work the same with enhanced model.""" - - # Test with LangChain disabled (original behavior) - model_direct = OpenAIModel("gpt-3.5-turbo", use_langchain=False) - - try: - response_direct = await model_direct.generate( - "What is 2+2? Respond with just the number.", - temperature=0.0, - max_tokens=10 - ) - assert len(response_direct) > 0 - assert "4" in response_direct - - # Test health check - health = await model_direct.health_check() - assert health is True - - # Test cost estimation - cost = await model_direct.estimate_cost("Test prompt", 100) - assert cost > 0 - - except Exception as e: - pytest.skip(f"OpenAI API test failed (possibly rate limited): {e}") - - @pytest.mark.asyncio - async def test_structured_output_compatibility(self): - """Test structured output generation works correctly.""" - - # Check if OpenAI API key is available using our key management system - available_keys = load_api_keys_optional() - if not available_keys.get("openai"): - pytest.skip("OpenAI API key not available") - """Test structured output generation works correctly.""" - - model = OpenAIModel("gpt-3.5-turbo", use_langchain=False) - - schema = { - "type": "object", - "properties": { - "result": {"type": "number"}, - "explanation": {"type": "string"} - }, - "required": ["result", "explanation"] - } - - try: - response = await model.generate_structured( - "What is 2+2?", - schema=schema, - temperature=0.0 - ) - - assert isinstance(response, dict) - assert "result" in response - assert "explanation" in response - assert response["result"] == 4 - - except Exception as e: - pytest.skip(f"OpenAI structured output test failed: {e}") - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) \ No newline at end of file diff --git a/tests/test_live_anthropic.py b/tests/test_live_anthropic.py deleted file mode 100644 index 6a86cf17..00000000 --- a/tests/test_live_anthropic.py +++ /dev/null @@ -1,228 +0,0 @@ -"""Live acceptance test for the Anthropic provider contract. - -This is the only test that spends money. It exists because everything else in -the suite is hermetic: without it, "Anthropic is the supported provider" is an -untested claim. Before this file existed, `pytest -m live` collected zero -tests and exited successfully, so the live CI job proved nothing. - -Scope is deliberately minimal -- one cheap call on the cheapest model, with a -hard output cap -- because the point is to verify the *contract* (we call the -API correctly and get back the shape we expect), not model quality. - -Run with: - ANTHROPIC_API_KEY=... pytest -m live -v - -Without a key, `tests/conftest.py` skips these with a reason naming the -missing variable. They never silently pass. -""" - -import os - -import pytest - -pytestmark = [pytest.mark.live, pytest.mark.asyncio] - -# The cheapest current model. Kept as a module constant so a failure report -# names exactly what was exercised. -LIVE_MODEL = "claude-haiku-4-5-20251001" - -# A hard ceiling on spend per call. The prompts below want a handful of -# tokens; this is loose enough not to truncate a correct answer and tight -# enough that a runaway loop cannot become expensive. -MAX_TOKENS = 32 - - -def _require_anthropic_package(): - """Import the anthropic SDK, or skip -- unless live coverage is required. - - In the live CI job a missing extra must FAIL, not skip. Otherwise a broken - install makes the job pass green having executed nothing, which is exactly - the hole this file was written to close. - """ - try: - import anthropic # noqa: F401 - except ImportError as exc: - message = ( - f"the [anthropic] extra is not installed ({exc}); " - "install with: pip install 'py-orc[anthropic]'" - ) - if os.environ.get("ORCHESTRATOR_REQUIRE_LIVE") == "1": - pytest.fail( - "ORCHESTRATOR_REQUIRE_LIVE=1 demands real live coverage but " - + message - ) - pytest.skip(message) - - -#: Substrings identifying an account/billing precondition rather than a defect -#: in this codebase. These must not be reported as our failure: doing so sends -#: someone hunting a bug in the adapter when the real fix is to add credit. -_ACCOUNT_PRECONDITIONS = ( - "credit balance is too low", - "quota", - "rate_limit", -) - - -def _skip_if_account_blocked(exc: Exception) -> None: - """Skip (or fail, under REQUIRE_LIVE) on a billing/quota precondition.""" - message = str(exc) - if not any(marker in message.lower() for marker in _ACCOUNT_PRECONDITIONS): - return - reason = ( - "the Anthropic account cannot serve requests (billing/quota), so live " - f"provider behaviour was NOT verified: {message}" - ) - if os.environ.get("ORCHESTRATOR_REQUIRE_LIVE") == "1": - pytest.fail(reason) - pytest.skip(reason) - - -def _model(): - """Build a model against the real API, skipping with a precise reason.""" - _require_anthropic_package() - from orchestrator.models.anthropic_model import AnthropicModel - - api_key = os.environ.get("ANTHROPIC_API_KEY") - assert api_key, "conftest should have skipped: ANTHROPIC_API_KEY is unset" - - # use_langchain=False exercises the direct Anthropic path, so this test - # depends only on the [anthropic] extra rather than the langgraph stack. - return AnthropicModel(name=LIVE_MODEL, api_key=api_key, use_langchain=False) - - -async def test_generate_returns_text_from_the_real_api(): - """The provider contract: generate(prompt) -> non-empty str.""" - model = _model() - - try: - response = await model.generate( - prompt="Reply with exactly the word: pong", - temperature=0.0, - max_tokens=MAX_TOKENS, - ) - except Exception as exc: - _skip_if_account_blocked(exc) - raise - - # Report what was actually exercised, so a CI failure is diagnosable - # without re-running against a paid API. - print(f"\nlive model: {LIVE_MODEL}\nlive response: {response!r}") - - assert isinstance(response, str), ( - f"generate() must return str, got {type(response).__name__}" - ) - assert response.strip(), "generate() returned an empty response" - # Deliberately loose: this asserts the round trip worked, not that the - # model is obedient. A strict equality check here would make the test - # flaky for reasons that say nothing about our code. - assert "pong" in response.lower(), ( - f"expected the model to echo 'pong', got {response!r}" - ) - - -async def test_generate_respects_max_tokens(): - """max_tokens must actually reach the API, not be silently dropped. - - A provider adapter that ignores max_tokens is both a correctness bug and a - cost bug, and it is invisible to every hermetic test. - """ - model = _model() - - try: - response = await model.generate( - prompt="Count slowly from 1 to 500, one number per line.", - temperature=0.0, - max_tokens=MAX_TOKENS, - ) - except Exception as exc: - _skip_if_account_blocked(exc) - raise - - print(f"\nlive model: {LIVE_MODEL}\ntruncated length: {len(response)} chars") - - assert isinstance(response, str) - # 32 tokens cannot render 500 numbers. Generous char bound: even at ~6 - # chars/token this is far under what an uncapped answer would produce. - assert len(response) < 600, ( - f"max_tokens={MAX_TOKENS} appears not to have been applied; " - f"got {len(response)} chars" - ) - - -async def test_every_family_alias_resolves_to_a_real_model(): - """Each bare family name must resolve to a model the API actually serves. - - Resolution goes through the Models API, so only a live run can prove it - works -- a hermetic test could only assert that the code calls the code. - This is the test that caught two successive rounds of hard-coded ids being - wrong: pinned 2024 ids (retired) and invented "-latest" aliases (never - existed), both 404. - """ - _require_anthropic_package() - from orchestrator.models.anthropic_model import AnthropicModel - - failures = [] - for family in AnthropicModel._FAMILIES: - model = AnthropicModel( - name=family, api_key=os.environ["ANTHROPIC_API_KEY"], use_langchain=False - ) - try: - reply = await model.generate( - prompt="Reply with the single word: ok", - temperature=0.0, - max_tokens=MAX_TOKENS, - ) - resolved = AnthropicModel._family_cache.get(family, "") - print(f"\nfamily {family!r} -> {resolved!r}: OK ({reply.strip()[:40]!r})") - except Exception as exc: # noqa: BLE001 - reporting all failures at once - _skip_if_account_blocked(exc) - failures.append(f"{family!r}: {exc}") - - assert not failures, ( - "these model families could not be resolved to a servable model:\n " - + "\n ".join(failures) - ) - - -async def test_models_api_lists_servable_models(): - """Record what the account can actually serve. - - Printed so a CI log is a primary source for which ids exist, instead of - the guesswork that produced two rounds of 404s in this adapter. - """ - _require_anthropic_package() - from orchestrator.models.providers.anthropic_provider import AnthropicProvider - from orchestrator.models.providers.base import ProviderConfig - - provider = AnthropicProvider( - ProviderConfig(name="anthropic", api_key=os.environ["ANTHROPIC_API_KEY"]) - ) - await provider.initialize() - models = await provider.discover_models() - - print("\nmodels served to this account:") - for model_id in sorted(models): - print(f" {model_id}") - - assert models, "the Models API returned no models" - assert any("claude" in m.lower() for m in models) - - -async def test_health_check_reports_true_against_the_real_api(): - """The provider's own readiness probe must agree with reality.""" - _require_anthropic_package() - from orchestrator.models.providers.anthropic_provider import AnthropicProvider - from orchestrator.models.providers.base import ProviderConfig - - provider = AnthropicProvider( - ProviderConfig(name="anthropic", api_key=os.environ["ANTHROPIC_API_KEY"]) - ) - await provider.initialize() - - healthy = await provider.health_check() - print(f"\nlive provider health_check: {healthy}") - - assert healthy is True, ( - "health_check() returned falsey against a real API with a valid key" - ) diff --git a/tests/test_live_huggingface.py b/tests/test_live_huggingface.py new file mode 100644 index 00000000..5116e901 --- /dev/null +++ b/tests/test_live_huggingface.py @@ -0,0 +1,205 @@ +"""Live acceptance tests for the HuggingFace Inference API provider. + +These need a token and the network. Generation tests only ever use models the +live catalog reports with a free route (an ``is_free`` promo or explicit zero +pricing), pinned to that provider so the router cannot bill the account. + +Two lessons carried over from the Dartmouth live suite: + +1. **Flapping free endpoints skip, not fail.** A free route is a promo on one + provider and it flaps; an upstream outage is not a defect in this adapter. + The CI job compensates by requiring at least one real pass. +2. **A reasoning model that spends its whole budget thinking is not a pass.** + Empty ``content`` with only ``reasoning_content`` must surface as + ``ReasoningTruncated`` -- observed live on this router, where + prism-ml/Ternary-Bonsai-27B-AWQ-4bit at max_tokens=32 returned 31 + reasoning tokens and no answer. + +An invalid token is NOT a flap: it fails, so a misconfigured CI secret turns +the job red instead of green-with-skips. + +Run with: + pytest -m live -k huggingface -v +""" + +import pytest + +from orchestrator.models.huggingface_credentials import resolve_huggingface_api_key +from orchestrator.models.huggingface_model import ( + ALLOW_PAID_ENV_VAR, + HuggingFaceModelError, + ModelLoading, + ModelUnavailable, + PaidModelRefused, + PaymentRequired, + RateLimited, + ReasoningTruncated, +) +from orchestrator.models.providers.huggingface_provider import HuggingFaceProvider + +pytestmark = [pytest.mark.live, pytest.mark.asyncio] + + +def _require_credential(): + credential = resolve_huggingface_api_key(required=False) + if credential is None: + pytest.skip( + "no HuggingFace token; set HF_TOKEN, add one to " + "~/.orchestrator/.env, or run `hf auth login`" + ) + return credential + + +async def _provider() -> HuggingFaceProvider: + _require_credential() + provider = HuggingFaceProvider() + await provider.initialize() + return provider + + +def _skip_if_flapping(exc: Exception, model_id: str): + """Upstream conditions that must skip rather than fail. 401 is not one.""" + if isinstance(exc, RateLimited): + pytest.skip(f"account is rate-limited: {exc}") + if isinstance(exc, PaymentRequired): + pytest.skip(f"account credits are depleted (monthly reset): {exc}") + if isinstance(exc, (ModelLoading, ModelUnavailable)): + pytest.skip(f"{model_id} is not serving right now: {exc}") + + +async def test_catalog_reports_models_and_free_routes(): + """The premise: the router serves chat models, some with a free route.""" + provider = await _provider() + models = await provider.discover_models() + free = provider.list_free_models() + + print(f"\ncatalog: {len(models)} chat models, {len(free)} with a free route:") + for model_id in free: + print(f" {model_id}") + print(f"paid/unknown: {len(provider.list_paid_models())}") + + assert models, "the router catalog returned no chat models at all" + + +async def test_generate_against_a_free_routed_model(): + """Proves the token can actually run inference, at zero cost.""" + provider = await _provider() + free = provider.list_free_models() + if not free: + pytest.skip("no model currently has a free route (promo drought)") + + model_id = provider.free_models_by_preference()[0] + model = await provider.create_model(model_id) + try: + assert model.cost.is_free, "refusing to spend money in a test" + assert await model.estimate_cost("hello") == 0.0 + reply = await model.generate( + "Reply with exactly the word: pong", temperature=0.0 + ) + except (RateLimited, PaymentRequired, ModelUnavailable, ReasoningTruncated) as exc: + _skip_if_flapping(exc, model_id) + raise + finally: + await model.aclose() + print(f"\n{model_id} -> {reply.strip()[:80]!r}") + + assert isinstance(reply, str) and reply.strip(), ( + "an empty reply is not a pass -- see the reasoning-truncation lesson" + ) + assert "pong" in reply.lower() + + +async def test_truncated_reasoning_reports_the_real_cause(): + """A starved reasoning model must raise, not return an empty string.""" + provider = await _provider() + free = provider.list_free_models() + if not free: + pytest.skip("no model currently has a free route") + + # A non-reasoning model starved at max_tokens=8 still emits partial + # content, so only a reasoning model can demonstrate this. Try a few. + tried = [] + for model_id in provider.free_models_by_preference()[:3]: + model = await provider.create_model(model_id) + try: + # ModelUnavailable subclasses HuggingFaceModelError, so transient + # outages are caught and skipped BEFORE the truncation assertion. + await model.generate("Explain quantum computing.", max_tokens=8) + except (ModelUnavailable, RateLimited, PaymentRequired) as exc: + _skip_if_flapping(exc, model_id) + except ReasoningTruncated as exc: + assert "max_tokens" in str(exc), "the error must name the fix" + return + finally: + await model.aclose() + tried.append(model_id) + pytest.skip(f"no reasoning model among the current free routes: {tried}") + + +async def test_paid_models_are_refused_without_optin(monkeypatch): + """The guard must hold against the real catalog, not just fixtures.""" + monkeypatch.delenv(ALLOW_PAID_ENV_VAR, raising=False) + provider = await _provider() + + paid = provider.list_paid_models() + assert paid, "a 130-model catalog with nothing paid would be suspicious" + + with pytest.raises(PaidModelRefused): + await provider.create_model(paid[0]) + + +async def test_paid_pricing_flows_from_the_real_catalog(monkeypatch): + """Per-million USD in the catalog must arrive as per-1k on the model.""" + monkeypatch.setenv(ALLOW_PAID_ENV_VAR, "1") + provider = await _provider() + # Only a model with explicit catalog pricing can prove the flow; an + # unpriced entry is unknown-priced and its estimate rightly refuses. + priced = [ + m + for m in provider.list_paid_models() + if provider.get_model_cost(m).output_cost_per_1k_tokens > 0 + ] + if not priced: + pytest.skip("catalog currently reports no explicit prices") + + model = await provider.create_model(priced[0]) + estimate = await model.estimate_cost("hello", max_tokens=1000) + print(f"\n{priced[0]}: 1000-token estimate ${estimate:.6f}") + assert estimate > 0.0 + + +async def test_provider_health_check(): + provider = await _provider() + assert await provider.health_check() is True + + +async def test_generate_free_answers_via_the_fallback_chain(): + """The reason this helper exists: free routes flap, demos must not.""" + provider = await _provider() + if not provider.list_free_models(): + pytest.skip("no model currently has a free route (promo drought)") + + try: + reply, model_id = await provider.generate_free( + "Reply with exactly the word: pong", temperature=0.0 + ) + except RateLimited as exc: + pytest.skip(f"account is rate-limited: {exc}") + except PaymentRequired as exc: + pytest.skip(f"account credits are depleted (monthly reset): {exc}") + print(f"\ngenerate_free answered via {model_id} -> {reply.strip()[:60]!r}") + + assert reply.strip(), "an empty reply is not a pass" + assert "pong" in reply.lower() + assert model_id in provider.list_free_models(), "fallback must stay free" + + +async def test_free_preference_covers_the_whole_free_set(): + """Every free model appears exactly once, and nothing paid sneaks in.""" + provider = await _provider() + ordered = provider.free_models_by_preference() + + assert sorted(ordered) == sorted(provider.list_free_models()) + assert len(ordered) == len(set(ordered)), "a model appears twice" + paid = set(provider.list_paid_models()) + assert not (set(ordered) & paid), "a paid model leaked into the free chain" diff --git a/tests/test_load_balancer.py b/tests/test_load_balancer.py deleted file mode 100644 index d32aa64c..00000000 --- a/tests/test_load_balancer.py +++ /dev/null @@ -1,407 +0,0 @@ -#!/usr/bin/env python3 -"""Test load balancing and failover functionality.""" - -import asyncio -import os -import time -import random -import pytest - -from orchestrator.models.model_registry import ModelRegistry -from orchestrator.models.load_balancer import LoadBalancer, ModelPoolConfig -from orchestrator.models.openai_model import OpenAIModel -from orchestrator.integrations.ollama_model import OllamaModel - -from tests.test_infrastructure import create_test_orchestrator, TestModel, TestProvider - - -@pytest.fixture -async def registry(): - """Create a model registry for testing.""" - return ModelRegistry() - - -@pytest.fixture -async def load_balancer(registry): - """Create a load balancer with test models and pools.""" - load_balancer = LoadBalancer(registry) - - # Register some test models (this setup is simplified for testing) - models_registered = [] - - # Try to register local models if available - try: - llama_small = OllamaModel("llama3.2:1b") - registry.register_model(llama_small) - models_registered.append("ollama:llama3.2:1b") - except Exception: - pass - - # Configure test pools - if models_registered: - # Primary pool - primary_config = ModelPoolConfig( - models=[ - {"model": models_registered[0], "weight": 1.0, "max_concurrent": 10} - ], - always_available=False, - fallback_pool="backup") - load_balancer.configure_pool("primary", primary_config) - - # Backup pool - backup_config = ModelPoolConfig( - models=[ - {"model": models_registered[0], "weight": 1.0, "max_concurrent": 5} - ], - always_available=True) - load_balancer.configure_pool("backup", backup_config) - - return load_balancer - - -async def setup_models_and_pools(): - """Set up test models and pools.""" - registry = ModelRegistry() - load_balancer = LoadBalancer(registry) - - # Register models - models_registered = [] - - # Local models (always available for testing) - try: - llama_small = OllamaModel("llama3.2:1b") - registry.register_model(llama_small) - models_registered.append("ollama:llama3.2:1b") - print("✓ Registered llama3.2:1b") - except Exception as e: - print(f"✗ Failed to register llama3.2:1b: {e}") - - try: - llama_medium = OllamaModel("llama3.1:8b") - registry.register_model(llama_medium) - models_registered.append("ollama:llama3.1:8b") - print("✓ Registered llama3.1:8b") - except Exception as e: - print(f"✗ Failed to register llama3.1:8b: {e}") - - # API models - if os.getenv("OPENAI_API_KEY"): - try: - gpt35 = OpenAIModel("gpt-3.5-turbo") - registry.register_model(gpt35) - models_registered.append("openai:gpt-3.5-turbo") - print("✓ Registered gpt-3.5-turbo") - except Exception as e: - print(f"✗ Failed to register gpt-3.5-turbo: {e}") - - # Configure model pools - - # Primary pool - mix of models with different weights - primary_pool = ModelPoolConfig( - models=[ - { - "model": "ollama:llama3.2:1b", - "weight": 0.4, - "max_concurrent": 5, - }, # 40% of traffic - { - "model": "ollama:llama3.1:8b", - "weight": 0.6, - "max_concurrent": 3, - }, # 60% of traffic - ], - fallback_pool="emergency", - retry_config={"max_retries": 3, "backoff": "exponential", "initial_delay": 0.5}) - - # Emergency fallback pool - emergency_pool = ModelPoolConfig( - models=[{"model": "ollama:llama3.2:1b", "weight": 1.0, "max_concurrent": 10}], - always_available=True, # Always consider this pool available - retry_config={"max_retries": 5, "backoff": "linear", "initial_delay": 1.0}) - - # High-performance pool (if API models available) - if "openai:gpt-3.5-turbo" in models_registered: - perf_pool = ModelPoolConfig( - models=[ - {"model": "openai:gpt-3.5-turbo", "weight": 0.8, "max_concurrent": 20}, - {"model": "ollama:llama3.1:8b", "weight": 0.2, "max_concurrent": 5}, - ], - fallback_pool="primary") - load_balancer.configure_pool("performance", perf_pool) - - load_balancer.configure_pool("primary", primary_pool) - load_balancer.configure_pool("emergency", emergency_pool) - - return registry, load_balancer, models_registered - - -async def test_weighted_selection(load_balancer: LoadBalancer): - """Test weighted model selection.""" - print("\n=== Testing Weighted Selection ===") - - selection_counts = {} - num_selections = 100 - - for i in range(num_selections): - try: - model = await load_balancer.select_from_pool("primary") - model_id = f"{model.provider}:{model.name}" - selection_counts[model_id] = selection_counts.get(model_id, 0) + 1 - - # Release the model - load_balancer.model_states[model_id].current_requests -= 1 - - except Exception as e: - print(f"Selection {i} failed: {e}") - - print(f"\nSelection distribution over {num_selections} requests:") - for model_id, count in selection_counts.items(): - percentage = (count / num_selections) * 100 - print(f" {model_id}: {count} ({percentage:.1f}%)") - - # Check if distribution roughly matches weights - if "ollama:llama3.2:1b" in selection_counts: - small_pct = selection_counts["ollama:llama3.2:1b"] / num_selections - print(f"\nExpected ~40% for llama3.2:1b, got {small_pct*100:.1f}%") - - if "ollama:llama3.1:8b" in selection_counts: - medium_pct = selection_counts["ollama:llama3.1:8b"] / num_selections - print(f"Expected ~60% for llama3.1:8b, got {medium_pct*100:.1f}%") - - -async def test_concurrent_limits(load_balancer: LoadBalancer): - """Test concurrent request limiting.""" - print("\n=== Testing Concurrent Request Limits ===") - - # Try to exceed concurrent limit for a model - model_id = "ollama:llama3.2:1b" - max_concurrent = 5 # As configured in pool - - tasks = [] - - async def make_request(i): - try: - model = await load_balancer.select_from_pool("primary") - selected_id = f"{model.provider}:{model.name}" - - if selected_id == model_id: - # Simulate some work - await asyncio.sleep(0.1) - return selected_id - else: - # Release immediately if different model - load_balancer.model_states[selected_id].current_requests -= 1 - return selected_id - except Exception as e: - return f"Failed: {e}" - - # Launch more requests than max concurrent - for i in range(max_concurrent + 3): - tasks.append(make_request(i)) - - results = await asyncio.gather(*tasks) - - # Count how many requests went to each model - model_counts = {} - for result in results: - if result.startswith("Failed"): - print(f" {result}") - else: - model_counts[result] = model_counts.get(result, 0) + 1 - - print("\nConcurrent request distribution:") - for model, count in model_counts.items(): - print(f" {model}: {count}") - - print(f"\nWith max_concurrent={max_concurrent} for {model_id},") - print("excess requests should spill over to other models.") - - -async def test_failover(load_balancer: LoadBalancer): - """Test failover to backup pool.""" - print("\n=== Testing Failover ===") - - # Simulate all models in primary pool being unavailable - # by maxing out their concurrent requests - for model_info in load_balancer.pools["primary"].models: - model_id = model_info["model"] - state = load_balancer.model_states[model_id] - state.current_requests = state.max_concurrent - - print("Simulated primary pool exhaustion...") - - # Try to select - should failover to emergency pool - try: - model = await load_balancer.select_from_pool("primary") - print(f"✓ Failover successful! Selected: {model.provider}:{model.name}") - print(" (This should be from the emergency pool)") - except Exception as e: - print(f"✗ Failover failed: {e}") - - # Reset concurrent requests - for model_id in load_balancer.model_states: - load_balancer.model_states[model_id].current_requests = 0 - - -async def test_retry_with_backoff(load_balancer: LoadBalancer): - """Test retry logic with exponential backoff.""" - print("\n=== Testing Retry with Backoff ===") - - # Create a mock model that fails a few times - class FailingModel: - def __init__(self, fail_count=2): - self.provider = "test" - self.name = "failing-model" - self.attempts = 0 - self.fail_count = fail_count - - async def generate(self, prompt: str, **kwargs): - self.attempts += 1 - if self.attempts <= self.fail_count: - raise Exception(f"Simulated failure {self.attempts}") - return f"Success after {self.attempts} attempts" - - model = FailingModel(fail_count=2) - - start_time = time.time() - try: - result = await load_balancer.execute_with_retry( - model, "generate", "Test prompt" - ) - elapsed = time.time() - start_time - - print(f"✓ Retry successful after {model.attempts} attempts") - print(f" Result: {result}") - print(f" Total time: {elapsed:.2f}s (includes backoff delays)") - except Exception as e: - print(f"✗ All retries failed: {e}") - - -async def test_circuit_breaker(load_balancer: LoadBalancer): - """Test circuit breaker functionality.""" - print("\n=== Testing Circuit Breaker ===") - - model_id = "ollama:llama3.2:1b" - state = load_balancer.model_states[model_id] - - # Simulate multiple failures to trip circuit breaker - print("Simulating consecutive failures...") - for i in range(6): - await load_balancer._update_failure_metrics(model_id) - - print(f"Circuit breaker state: {'OPEN' if state.circuit_open else 'CLOSED'}") - print(f"Consecutive failures: {state.consecutive_failures}") - - # Try to select from pool - should skip the failed model - try: - model = await load_balancer.select_from_pool("primary") - selected_id = f"{model.provider}:{model.name}" - print(f"✓ Selected alternative model: {selected_id}") - print(f" (Should not be {model_id} due to open circuit)") - except Exception as e: - print(f"Selection failed: {e}") - - # Reset circuit breaker - state.circuit_open = False - state.consecutive_failures = 0 - - -async def test_pool_status(load_balancer: LoadBalancer): - """Test pool status reporting.""" - print("\n=== Testing Pool Status ===") - - # Make some requests to generate statistics - for _ in range(10): - try: - model = await load_balancer.select_from_pool("primary") - model_id = f"{model.provider}:{model.name}" - - # Simulate success - await load_balancer._update_success_metrics( - model_id, random.uniform(0.1, 0.5) - ) - - # Release - load_balancer.model_states[model_id].current_requests -= 1 - except Exception: - pass - - # Get pool status - status = load_balancer.get_pool_status("primary") - - print("\nPrimary Pool Status:") - print(f" Fallback pool: {status['fallback_pool']}") - print(f" Always available: {status['always_available']}") - - print("\nModel Statistics:") - for model_status in status["models"]: - print(f"\n Model: {model_status['model']}") - print(f" Weight: {model_status['weight']}") - print(f" Success rate: {model_status['success_rate']:.2%}") - print(f" Avg latency: {model_status['avg_latency']:.3f}s") - print( - f" Current requests: {model_status['current_requests']}/{model_status['max_concurrent']}" - ) - - -async def test_real_generation_with_lb( - registry: ModelRegistry, load_balancer: LoadBalancer -): - """Test real generation through load balancer.""" - print("\n=== Testing Real Generation with Load Balancing ===") - - prompt = "What is the capital of France? Give a one word answer." - - try: - # Select model from pool - model = await load_balancer.select_from_pool("primary") - print(f"Selected model: {model.provider}:{model.name}") - - # Execute with retry - result = await load_balancer.execute_with_retry( - model, "generate", prompt, temperature=0 - ) - - print(f"Prompt: {prompt}") - print(f"Response: {result.strip()}") - - # Update registry metrics - registry.update_model_performance( - model, success=True, latency=0.2, cost=0.0 if model.cost.is_free else 0.001 - ) - - print("✓ Generation successful") - - except Exception as e: - print(f"✗ Generation failed: {e}") - - -async def main(): - """Run all load balancer tests.""" - print("🚀 LOAD BALANCING AND FAILOVER TEST") - print("=" * 50) - - # Set up models and pools - registry, load_balancer, models = await setup_models_and_pools() - - if not models: - print("\n⚠️ No models registered! Ensure Ollama is running.") - return - - print(f"\nRegistered {len(models)} models") - - # Run tests - await test_weighted_selection(load_balancer) - await test_concurrent_limits(load_balancer) - await test_failover(load_balancer) - await test_retry_with_backoff(load_balancer) - await test_circuit_breaker(load_balancer) - await test_pool_status(load_balancer) - await test_real_generation_with_lb(registry, load_balancer) - - print("\n" + "=" * 50) - print("✓ All load balancing tests complete!") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/tests/test_minikernel.py b/tests/test_minikernel.py new file mode 100644 index 00000000..afda162a --- /dev/null +++ b/tests/test_minikernel.py @@ -0,0 +1,596 @@ +"""Tests for the #485 design kernel under scripts/prototypes/minikernel. + +Real SQLite databases on disk, real subprocesses, real signals, real capability +implementations. Nothing is mocked and nothing is simulated: where a test needs +a crash it kills a process, and where it needs a failing capability it calls +one that really fails. + +The network-backed planner is exercised by scripts/prototypes/measure_ambiguity.py +and probe_optimism.py, which cache their measurements; those are deliberately +not run here so the suite stays offline and free. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys + +import pytest + +PROTO = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "scripts", "prototypes") +if PROTO not in sys.path: + sys.path.insert(0, PROTO) + +from minikernel import ( # noqa: E402 + ArtifactVersion, Authority, BugReport, BugTracker, Budget, Capability, + CapabilityRegistry, Criterion, Finding, InsightPool, LLMPlanner, Message, + Plan, ReviewBoard, Runtime, SeparationOfDuty, SolvedProblemLibrary, Step, + Store, StubPlanner, validate) +from minikernel.library import signature_compatible, similarity # noqa: E402 +from minikernel.store import n_tokens # noqa: E402 + +ROOT_AUTH = Authority(net=frozenset({"example.com"}), spend_usd=5.0) + + +@pytest.fixture() +def store(tmp_path): + s = Store(str(tmp_path / "k.db"), segment_tokens=400) + yield s + s.close() + + +@pytest.fixture() +def registry(store): + reg = CapabilityRegistry(store) + reg.seed(Capability("echo", 1, lambda a: str(a.get("text", "")), "str", "str", + tests=[({"text": "x"}, "x")])) + reg.seed(Capability("upper", 1, lambda a: str(a.get("text", "")).upper(), + "str", "str", tests=[({"text": "x"}, "X")])) + return reg + + +class ScriptedPlanner: + """A planner whose output the test specifies. + + StubPlanner is seeded but its draw depends on the capability list, so tests + written against it are hostage to a lucky roll -- three of these tests + silently tested nothing until that was noticed. A test about recursion + should state the recursion it means. + """ + + def __init__(self, by_depth: dict[int, list[tuple[str, str]]]): + self.by_depth = by_depth + self.calls = 0 + + def decompose(self, problem, signature, capabilities, depth): + from minikernel.planner import PlanDraft + self.calls += 1 + spec = self.by_depth.get(depth, self.by_depth[max(self.by_depth)]) + steps = [] + for i, (kind, arg) in enumerate(spec): + if kind == "capability": + steps.append(Step(f"s{i}", "capability", ref=arg, + args={"text": f"{problem}#{i}"}, + output_schema="str")) + else: + steps.append(Step(f"s{i}", "decompose", + problem=f"{problem} :: {arg}", outputs=("y",), + output_schema="str")) + return PlanDraft(Plan(f"scripted-d{depth}", problem, tuple(steps), + signature=signature), "scripted") + + +def make_runtime(store, registry, **kw): + planner = kw.pop("planner", StubPlanner(seed=1, b=4, f=0.15, + capability_pool=("echo@1",))) + return Runtime(store, registry, SolvedProblemLibrary(store), planner, **kw) + + +# ------------------------------------------------------------------- store + +def test_token_estimate_is_conservative(): + text = "hello world " * 100 + assert n_tokens(text) >= len(text.split()) + + +def test_blobs_are_content_addressed_and_deduplicated(store): + a = store.put_blob("same body") + b = store.put_blob("same body") + assert a == b + assert store.get_blob(a) == "same body" + + +def test_events_are_append_only_with_per_run_sequence(store): + for i in range(5): + store.append_event("r", f"n{i}", "node_started", {"i": i}) + seqs = [e["seq"] for e in store.events("r")] + assert seqs == [1, 2, 3, 4, 5] + + +def test_descendant_runs_are_reachable_from_the_root(store): + store.append_event("root", "root", "run_created", {}) + store.append_event("root/a", "root/a", "run_created", {}, parent_run_id="root") + store.append_event("root/a/b", "root/a/b", "run_created", {}, + parent_run_id="root/a") + runs = {e["run_id"] for e in store.events("root")} + assert runs == {"root", "root/a", "root/a/b"} + + +def test_sealing_summarises_each_range_exactly_once(store): + calls = [] + + def summarize(text, level): + calls.append(level) + return f"L{level}: {text[:40]}" + + for i in range(120): + store.append_note("r", "n", "intent", f"note number {i} about calibration") + first = store.seal(summarize) + n_first = len(calls) + second = store.seal(summarize) + assert first, "expected segments to be sealed" + assert second == [], "re-sealing with no new notes must do nothing" + assert len(calls) == n_first, "no range may be summarised twice" + + +def test_a_summary_can_always_be_exchanged_for_its_source(store): + for i in range(120): + store.append_note("r", "n", "intent", f"note {i} about calibration drift") + segs = store.seal(lambda t, lvl: f"summary L{lvl}") + leaves = store.expand(segs[0]) + assert leaves and any("calibration drift" in x for x in leaves) + + +def test_context_window_respects_its_budget_and_reports_a_manifest(store): + for i in range(200): + store.append_note("r", "n", "intent", f"widget calibration note {i}") + store.seal(lambda t, lvl: f"summary L{lvl} of widget calibration") + win = store.compile_window(500, query="widget calibration") + assert win.tokens <= 500 + assert win.manifest() + assert {e["lane"] for e in win.manifest()} <= { + "tail", "summaries", "retrieved", "insights"} + + +def test_the_four_views_read_from_one_substrate(store, registry): + store.append_note("r", "n", "intent", "a plain thought") + store.append_note("r", "n", "insight", "a durable insight about drift") + registry.invoke("echo@1", {"text": "hi"}, "r", "n", "s", ROOT_AUTH) + assert len(store.view_scratchpad()) == 2 + assert len(store.view_insights()) == 1 + assert len(store.view_tool_history("echo@1")) == 1 + + +# ---------------------------------------------------------------------- IR + +@pytest.mark.parametrize("step,expected", [ + (Step("l", "loop", guard="true", then=(Step("i", "capability", ref="echo@1"),)), + "static iteration bound"), + (Step("u", "capability", ref="ghost@9"), "unknown capability"), + (Step("p", "capability", ref="echo@1", authority=Authority(subprocess=True)), + "beyond its parent"), + (Step("b", "branch"), "branch has no guard"), + (Step("d", "decompose", problem="x"), "declares no outputs"), +]) +def test_validator_rejects_each_unsafe_form(registry, step, expected): + errors = validate(Plan("p", "x", (step,)), registry.maturities(), ROOT_AUTH) + assert any(expected in e for e in errors), errors + + +def test_validator_accepts_a_well_formed_plan(registry): + plan = Plan("p", "x", (Step("a", "capability", ref="echo@1", + output_schema="str"),)) + assert validate(plan, registry.maturities(), ROOT_AUTH) == [] + + +def test_plan_hash_is_stable_and_input_sensitive(registry): + a = Plan("p", "x", (Step("a", "capability", ref="echo@1", output_schema="str"),)) + b = Plan("p", "x", (Step("a", "capability", ref="echo@1", output_schema="str"),)) + c = Plan("p", "y", (Step("a", "capability", ref="echo@1", output_schema="str"),)) + assert a.hash() == b.hash() and a.hash() != c.hash() + + +def test_authority_narrows_but_never_widens(): + parent = Authority(net=frozenset({"a", "b"}), spend_usd=1.0) + child = Authority(net=frozenset({"b", "c"}), spend_usd=0.5) + assert parent.narrow(child).net == frozenset({"b"}) + assert not parent.covers(Authority(subprocess=True)) + + +def test_budget_split_holds_a_reserve_back(): + b = Budget(1000, 1.0, 10, 60.0, reserve_frac=0.3) + share = b.child_share(2) + assert share.tokens == 350 + assert 2 * share.tokens < b.tokens + + +def test_budget_top_up_moves_credit_and_records_the_spend(): + parent, child = Budget(1000, 1.0, 10, 60.0), Budget(100, 0.1, 2, 6.0) + moved = child.top_up(parent, 400) + assert moved == 400 and child.tokens == 500 + assert parent.remaining().tokens == 600 + + +# --------------------------------------------------------------- capabilities + +def test_a_draft_capability_cannot_be_invoked(store, registry): + registry.register(Capability("x", 1, lambda a: 1, "str", "int", + tests=[({}, 1)], author_session="A")) + with pytest.raises(PermissionError): + registry.invoke("x@1", {}, "r", "n", "s", ROOT_AUTH) + + +def test_an_author_cannot_qualify_their_own_capability(store, registry): + registry.register(Capability("x", 1, lambda a: 1, "str", "int", + tests=[({}, 1)], author_session="A")) + with pytest.raises(SeparationOfDuty): + registry.qualify("x@1", "A") + + +def test_qualification_actually_runs_the_tests(store, registry): + registry.register(Capability("good", 1, lambda a: 2, "str", "int", + tests=[({}, 2)], author_session="A")) + registry.register(Capability("bad", 1, lambda a: 3, "str", "int", + tests=[({}, 2)], author_session="A")) + assert registry.qualify("good@1", "B")[0] is True + ok, failures = registry.qualify("bad@1", "B") + assert ok is False and "observed 3" in failures[0] + assert registry.get("bad@1").maturity == "quarantined" + + +def test_a_capability_without_tests_is_quarantined_not_trusted(store, registry): + registry.register(Capability("untested", 1, lambda a: 1, "str", "int", + author_session="A")) + ok, why = registry.qualify("untested@1", "B") + assert ok is False and registry.get("untested@1").maturity == "quarantined" + assert "no tests" in why[0] + + +def test_a_capability_cannot_exceed_the_callers_authority(store, registry): + registry.seed(Capability("net", 1, lambda a: "d", "str", "str", + authority=Authority(net=frozenset({"evil.test"})), + tests=[({}, "d")])) + with pytest.raises(PermissionError): + registry.invoke("net@1", {}, "r", "n", "s", ROOT_AUTH) + + +def test_capability_failures_are_recorded_before_being_raised(store, registry): + registry.seed(Capability("boom", 1, lambda a: 1 / 0, "str", "int", + tests=[({}, 0)])) + with pytest.raises(RuntimeError): + registry.invoke("boom@1", {}, "r", "n", "s", ROOT_AUTH) + history = store.view_tool_history("boom@1") + assert len(history) == 1 + assert "ZeroDivisionError" in json.loads(history[0]["payload"])["error"] + + +def test_bug_triage_is_independent_and_evidence_driven(store, registry): + registry.seed(Capability("flaky", 1, lambda a: 1 / 0, "str", "int", + tests=[({}, 0)])) + for _ in range(5): + with pytest.raises(RuntimeError): + registry.invoke("flaky@1", {}, "r", "n", "s", ROOT_AUTH) + tracker = BugTracker(store, registry) + rid = tracker.file(BugReport("B", "flaky@1", "use", "ok", "raises", "R")) + with pytest.raises(SeparationOfDuty): + tracker.triage(rid, "R") + assert tracker.triage(rid, "Q") == "revoked" + assert registry.get("flaky@1").maturity == "revoked" + + +# -------------------------------------------------------------------- review + +def test_an_author_cannot_review_their_own_artifact(store): + board = ReviewBoard(store) + art = ArtifactVersion("a", 1, "body", "A") + with pytest.raises(SeparationOfDuty): + board.run(art, [], "A", revise=lambda a, f: a) + + +def test_evidential_finding_blocks_and_a_revision_clears_it(store): + board = ReviewBoard(store) + art = ArtifactVersion("a", 1, "the answer is 41", "A") + crit = [Criterion("C1", "must say 42", lambda b: ("42" in b, f"body={b!r}"))] + final, outcome = board.run( + art, crit, "B", + revise=lambda a, f: ArtifactVersion(a.artifact_id, a.version + 1, + "the answer is 42", "A")) + assert outcome.passed and final.version == 2 and outcome.rounds == 2 + + +def test_a_pass_reports_residual_risk_rather_than_certifying_clean(store): + board = ReviewBoard(store, detection_rate_prior=0.6) + art = ArtifactVersion("a", 1, "42", "A") + _, outcome = board.run(art, [Criterion("C1", "ok", lambda b: (True, ""))], + "B", revise=lambda a, f: a) + assert outcome.passed and outcome.residual_risk == pytest.approx(0.4) + + +def test_a_finding_outside_the_frozen_criteria_cannot_block(store): + board = ReviewBoard(store) + art = ArtifactVersion("a", 1, "body", "A") + board.freeze("a", [Criterion("C1", "in scope")]) + drift = board.file(art, "B", Finding("F1", "C_OTHER", "blocker", "scope creep", + "evidence!")) + assert drift.status == "deferred" and not drift.blocking + + +def test_a_prose_worry_is_recorded_as_a_risk_not_a_blocker(store): + board = ReviewBoard(store) + art = ArtifactVersion("a", 1, "body", "A") + board.freeze("a", [Criterion("C1", "in scope")]) + worry = board.file(art, "B", Finding("F1", "C1", "blocker", "feels off", None)) + assert worry.status == "risk" and not worry.blocking + + +def test_review_escalates_instead_of_looping_forever(store): + board = ReviewBoard(store, max_rounds=3) + art = ArtifactVersion("a", 1, "never fixed", "A") + crit = [Criterion("C1", "impossible", lambda b: (False, "still wrong"))] + _, outcome = board.run(art, crit, "B", revise=lambda a, f: ArtifactVersion( + a.artifact_id, a.version + 1, a.body, a.author_session)) + assert not outcome.passed and outcome.escalated and outcome.rounds == 3 + + +def test_contradicting_insights_are_refused_at_insertion(store): + pool = InsightPool(store, ReviewBoard(store)) + ok, _ = pool.propose("r", "n", "the calibration drifts above 40 degrees", + "A", "B") + assert ok + ok2, why = pool.propose("r", "n", + "the calibration does not drift above 40 degrees", + "C", "B") + assert not ok2 and "contradicts" in why + + +def test_an_author_cannot_approve_their_own_insight(store): + pool = InsightPool(store, ReviewBoard(store)) + with pytest.raises(SeparationOfDuty): + pool.propose("r", "n", "a thought", "A", "A") + + +# ------------------------------------------------------------------- library + +@pytest.mark.parametrize("want,have,ok", [ + ("str->int", "str->int", True), + ("str->int", "str->str", False), + ("str->int", "any->int", True), + ("str,int->bool", "str->bool", False), +]) +def test_signature_compatibility(want, have, ok): + assert signature_compatible(want, have) is ok + + +def test_similarity_ignores_stopwords_and_order(): + assert similarity("summarise the sales corpus", + "corpus of sales to summarise") > 0.9 + + +def test_an_untyped_solution_is_never_published(store): + lib = SolvedProblemLibrary(store) + plan = Plan("p", "x", (Step("a", "capability", ref="echo@1", + output_schema="str"),)) + assert lib.publish("some problem", "any->any", plan, {}) is None + assert len(lib) == 0 + + +def test_a_typed_solution_is_retrieved_only_on_a_compatible_signature(store): + lib = SolvedProblemLibrary(store) + plan = Plan("p", "x", (Step("a", "capability", ref="echo@1", + output_schema="str"),)) + lib.publish("summarise the sales corpus for the north region", "any->str", + plan, {}) + assert lib.lookup("summarise the sales corpus for the north region", + "any->str") is not None + assert lib.lookup("summarise the sales corpus for the north region", + "any->int") is None + + +def test_budget_exhaustion_does_not_count_against_a_cached_plan(store): + lib = SolvedProblemLibrary(store) + plan = Plan("p", "x", (Step("a", "capability", ref="echo@1", + output_schema="str"),)) + key = lib.publish("a solvable problem statement", "any->str", plan, {}) + before = lib.entries[key].reliability + lib.record_use(key, "budget_exhausted") + assert lib.entries[key].reliability == before + lib.record_use(key, "failed") + assert lib.entries[key].reliability < before + + +def test_a_dead_end_is_remembered_only_at_the_allowance_that_failed(store): + lib = SolvedProblemLibrary(store) + lib.record_intractable("an unreachable problem statement here", "any->str", + "depth cap", depth_allowance=2) + assert lib.lookup_intractable("an unreachable problem statement here", + "any->str", 2) is not None + # more room than last time is a legitimate reason to try again + assert lib.lookup_intractable("an unreachable problem statement here", + "any->str", 5) is None + + +# ------------------------------------------------------------------- runtime + +def test_child_plans_get_their_own_run_with_a_recorded_parent(store, registry): + rt = make_runtime(store, registry, max_depth=3, review_plans=False, + planner=ScriptedPlanner({ + 0: [("capability", "echo@1"), ("decompose", "hard bit")], + 1: [("capability", "echo@1"), ("capability", "upper@1")], + })) + rt.run("build a widget report", Budget(200_000, 5.0, 400, 600.0), ROOT_AUTH, + run_id="r1", signature="any->str") + proj = rt.projection("r1") + children = [r for r, v in proj["tree"].items() if v["parent"]] + assert children, "expected at least one child run" + assert all(v["parent"] in proj["tree"] for v in proj["tree"].values() + if v["parent"]) + + +def test_a_supercritical_planner_never_reports_success(store, registry): + # every level emits two ambiguous children: m = 2 > 1, so the tree can only + # ever end at the depth cap. + rt = make_runtime(store, registry, max_depth=3, review_plans=False, + planner=ScriptedPlanner({ + 0: [("decompose", "left"), ("decompose", "right")]})) + res = rt.run("summarise the sales corpus for region north", + Budget(2_000_000, 50.0, 20_000, 600.0), ROOT_AUTH, run_id="h", + signature="any->str") + assert res.status == "escalated" and res.partial + + +def test_a_repeated_dead_end_costs_less_the_second_time(store, registry): + rt = make_runtime(store, registry, max_depth=3, review_plans=False, + planner=ScriptedPlanner({ + 0: [("decompose", "left"), ("decompose", "right")]})) + costs = [] + for i in range(3): + before = rt.stats.nodes + rt.run("summarise the sales corpus for region north", + Budget(400_000, 5.0, 2000, 600.0), ROOT_AUTH, run_id=f"h{i}", + signature="any->str") + costs.append(rt.stats.nodes - before) + assert costs[-1] < costs[0] / 2, costs + assert rt.stats.dead_end_hits > 0 + + +def test_an_escalating_sibling_does_not_cancel_the_others(store, registry): + rt = make_runtime(store, registry, max_depth=1, review_plans=False) + plan = Plan("p", "x", ( + Step("a", "capability", ref="echo@1", args={"text": "one"}, + output_schema="str"), + Step("b", "decompose", problem="unreachable", outputs=("y",), + output_schema="str"), + Step("c", "capability", ref="upper@1", args={"text": "three"}, + output_schema="str"), + )) + store.append_event("rr", "rr", "run_created", {}) + res = rt._execute_plan(plan, Budget(100_000, 1.0, 100, 60.0), ROOT_AUTH, + "rr", 1, {}) + done = {e["node_id"] for e in store.events("rr") + if e["type"] == "node_completed"} + assert res.status == "escalated" and "unresolved steps" in res.reason + assert {"a", "c"} <= done, "siblings after the escalation must still run" + + +def test_budget_exhaustion_is_loud_and_never_a_green_checkmark(store, registry): + rt = make_runtime(store, registry, max_depth=4, review_plans=False, + planner=ScriptedPlanner({ + 0: [("capability", "echo@1"), ("decompose", "deeper"), + ("decompose", "deeper still")]})) + res = rt.run("an expensive recursive problem", Budget(4_000, 0.05, 12, 60.0), + ROOT_AUTH, run_id="rb", signature="any->str") + kinds = [e["type"] for e in store.events("rb")] + assert res.status != "completed" and res.partial + assert "node_budget_exhausted" in kinds or "node_escalated" in kinds + + +def test_a_message_is_delivered_at_a_step_boundary_and_recorded(store, registry): + rt = make_runtime(store, registry, max_depth=1, review_plans=False) + plan = Plan("p", "x", (Step("s1", "capability", ref="upper@1", + args={"text": "original"}, output_schema="str"),)) + rt.bus.send("rm", "root", Message("m1", "rm/s1", "scope_change", + {"args": {"text": "redirected"}})) + store.append_event("rm", "rm", "run_created", {}) + res = rt._execute_plan(plan, Budget(100_000, 1.0, 50, 60.0), ROOT_AUTH, + "rm", 0, {}) + assert res.value == "REDIRECTED" + assert any(e["type"] == "message_delivered" for e in store.events("rm")) + + +def test_a_message_over_its_hop_budget_is_dead_lettered(store, registry): + rt = make_runtime(store, registry) + rt.bus.send("r", "root", Message("m", "r/x", "instruction", {}, hops=99, + max_hops=8)) + assert len(rt.bus.dead_letters) == 1 + assert rt.bus.take("r/x") == [] + + +def test_admission_control_demotes_an_overclaimed_atomic_step(store, registry): + registry.seed(Capability("guess", 1, lambda a: "PLAUSIBLE", "str", "str", + tests=[({}, "PLAUSIBLE")])) + + class Optimistic: + def decompose(self, problem, signature, capabilities, depth): + from minikernel.planner import PlanDraft + return PlanDraft(Plan("o", problem, tuple( + Step(f"s{i}", "capability", ref="guess@1", + args={"text": problem}, output_schema="str") + for i in range(3)), signature=signature)) + + unchecked = Runtime(store, registry, SolvedProblemLibrary(store), Optimistic(), + max_depth=1, review_plans=False) + res = unchecked.run("reproduce figure 3", Budget(200_000, 5.0, 400, 600.0), + ROOT_AUTH, run_id="u", signature="any->str") + assert res.status == "completed" and res.value == "PLAUSIBLE" + assert unchecked.stats.m_measured == 0.0 + + checked = Runtime(store, registry, SolvedProblemLibrary(store), Optimistic(), + max_depth=1, review_plans=False, + admission=lambda step, run: (False, "cannot reproduce a figure") + if step.ref == "guess@1" else (True, "")) + res2 = checked.run("reproduce figure 3", Budget(200_000, 5.0, 400, 600.0), + ROOT_AUTH, run_id="c", signature="any->str") + assert res2.status != "completed" + # each demoted step becomes a decomposition, whose own plan is demoted too, + # so the count compounds down the tree -- that is the point. + assert checked.stats.demoted >= 3 + assert checked.stats.m_declared == 0.0 and checked.stats.m_measured == 3.0 + + +def test_the_offspring_mean_is_not_the_product_of_the_means(store, registry): + """The estimator that got this wrong once already.""" + rt = make_runtime(store, registry, max_depth=2, review_plans=False, + planner=StubPlanner(seed=2, b=6, f=0.3, + capability_pool=("echo@1",))) + rt.run("x", Budget(400_000, 5.0, 4000, 600.0), ROOT_AUTH, run_id="e", + signature="any->str") + assert rt.stats.m_measured == pytest.approx( + rt.stats.ambiguous / len(rt.stats.fan_outs)) + + +# ------------------------------------------------------------- crash / resume + +def test_a_killed_run_resumes_to_an_identical_projection(tmp_path): + """Kills a real child process with a real exit code, then resumes.""" + script = os.path.join(PROTO, "run_scenarios.py") + env = dict(os.environ, MK_DB=str(tmp_path / "c.db"), + MK_CLEAN=str(tmp_path / "clean.db"), PYTHONPATH=PROTO) + clean = subprocess.run([sys.executable, script, "--child", "clean"], env=env, + capture_output=True, text=True, timeout=300) + assert clean.returncode == 0, clean.stderr + crashed = subprocess.run([sys.executable, script, "--child", "crash"], env=env, + capture_output=True, text=True, timeout=300) + assert crashed.returncode == 137, "expected a hard kill mid-run" + resumed = subprocess.run([sys.executable, script, "--child", "resume"], env=env, + capture_output=True, text=True, timeout=300) + assert resumed.returncode == 0, resumed.stderr + a = json.loads(resumed.stdout.strip().splitlines()[-1]) + b = json.loads(clean.stdout.strip().splitlines()[-1]) + assert a["skipped"] > 0, "resume must reuse completed work from the log" + assert a["projection"] == b["projection"] + + +# ------------------------------------------------------------------- planner + +def test_stub_planner_is_a_function_of_its_inputs(): + p = StubPlanner(seed=4, b=5, f=0.4, capability_pool=("echo@1",)) + a = p.decompose("the same problem", "any->str", {"echo@1": "trusted"}, 0) + b = p.decompose("the same problem", "any->str", {"echo@1": "trusted"}, 0) + assert [s.kind for s in a.plan.steps] == [s.kind for s in b.plan.steps] + + +@pytest.mark.parametrize("reply,steps", [ + ('{"steps": [], "rationale": "none"}', 0), + ('```json\n{"steps": [{"id":"s1","kind":"capability","ref":"echo@1"}]}\n```', 1), + ('here you go {"steps": [{"id":"s1","kind":"decompose","problem":"p"}]} ok', 1), +]) +def test_llm_reply_parsing_handles_the_shapes_models_actually_emit(reply, steps): + assert len(LLMPlanner._extract_json(reply).get("steps", [])) == steps + + +def test_llm_reply_parsing_refuses_to_guess_at_garbage(): + with pytest.raises(ValueError): + LLMPlanner._extract_json("I'm afraid I can't do that.") diff --git a/tests/test_model_routing.py b/tests/test_model_routing.py deleted file mode 100644 index 5efcf39f..00000000 --- a/tests/test_model_routing.py +++ /dev/null @@ -1,255 +0,0 @@ -"""Test model routing with real API calls.""" - -import pytest -import asyncio -from typing import Dict, Any, List -from orchestrator.tools.llm_tools import MultiModelRoutingTool -from orchestrator.models.registry import ModelRegistry -from tests.test_infrastructure import TestProvider - - -class TestModelRouting: - """Test model routing with real API calls.""" - - @pytest.fixture - async def model_registry(self): - """Create a model registry with test models.""" - registry = ModelRegistry() - test_provider = TestProvider() - registry.register_provider(test_provider) - return registry - - @pytest.fixture - async def routing_tool(self, model_registry): - """Create a MultiModelRoutingTool instance with test models.""" - tool = MultiModelRoutingTool() - tool.model_registry = model_registry - return tool - - @pytest.mark.asyncio - async def test_route_multiple_tasks(self, routing_tool): - """Test routing multiple tasks to appropriate models.""" - result = await routing_tool.execute( - action="route", - tasks=[ - {"task": "Summarize this text in 2 sentences", "context": "AI is transforming industries worldwide. Machine learning enables real-time processing."}, - {"task": "Write Python code", "context": "fibonacci function with type hints"}, - {"task": "Analyze sales data", "context": "Q4 2024 sales: $2.5M revenue, 15% growth"} - ], - routing_strategy="balanced", - constraints={"total_budget": 10.0, "max_latency": 30.0} - ) - - assert result["success"] - assert "result" in result - - routing_result = result["result"] - assert "recommendations" in routing_result - assert len(routing_result["recommendations"]) == 3 - assert "total_estimated_cost" in routing_result - assert routing_result["total_estimated_cost"] < 10.0 - - # Each recommendation has real model selection - for rec in routing_result["recommendations"]: - assert "model" in rec - assert "estimated_cost" in rec - assert rec["estimated_cost"] >= 0 - assert "reasons" in rec or "reasoning" in rec - - @pytest.mark.asyncio - async def test_optimize_batch_processing(self, routing_tool): - """Test batch optimization with real executions.""" - # Real translation tasks - result = await routing_tool.execute( - action="optimize_batch", - tasks=[ - "Translate 'Hello World' to Spanish", - "Translate 'Good morning' to French", - "Translate 'Thank you' to German", - "Translate 'Goodbye' to Italian" - ], - optimization_goal="minimize_cost", - constraints={"max_budget_per_task": 0.05} - ) - - assert result["success"] - assert "result" in result - - batch_result = result["result"] - assert "results" in batch_result - assert len(batch_result["results"]) == 4 - - # Check that translations were produced - translations = batch_result["results"] - assert any("translation" in str(t).lower() for t in translations) - - # Check cost tracking - assert "total_cost" in batch_result or "average_cost" in batch_result - if "total_cost" in batch_result: - assert batch_result["total_cost"] >= 0 # May be 0 for test models - assert "models_used" in batch_result - assert len(batch_result["models_used"]) > 0 - - @pytest.mark.asyncio - async def test_routing_strategies(self, routing_tool): - """Test different routing strategies.""" - strategies = { - "cost": "cost_optimized", - "balanced": "balanced", - "quality": "quality_optimized" - } - - results_by_strategy = {} - - for key, strategy in strategies.items(): - result = await routing_tool.execute( - action="route", - tasks=[ - {"task": "Write a complex analysis", "context": "Analyze market trends"}, - {"task": "Simple calculation", "context": "Add 2+2"} - ], - routing_strategy=strategy, - constraints={"total_budget": 10.0} - ) - - assert result["success"] - assert "result" in result - - strategy_result = result["result"] - assert "recommendations" in strategy_result - - # Store model selections for comparison - models = [rec["model"] for rec in strategy_result["recommendations"]] - costs = [rec["estimated_cost"] for rec in strategy_result["recommendations"]] - results_by_strategy[key] = {"models": models, "costs": costs} - - # Verify strategy affects model selection - # Cost-optimized should have lower total cost - cost_total = sum(results_by_strategy["cost"]["costs"]) - quality_total = sum(results_by_strategy["quality"]["costs"]) - - # Quality strategy should generally cost more - assert quality_total >= cost_total * 0.8 # Allow some variance - - @pytest.mark.asyncio - async def test_single_request_routing(self, routing_tool): - """Test backward compatibility with single request routing.""" - result = await routing_tool.execute( - request="Generate a haiku about artificial intelligence", - preferences={"quality": 0.8, "cost": 0.5, "speed": 0.3} - ) - - assert result["success"] - assert "result" in result - - # Check routing information is present - routing_result = result["result"] - assert "routing_reason" in routing_result - assert "all_loads" in routing_result - assert "current_load" in routing_result - - # Verify routing worked - assert isinstance(routing_result["routing_reason"], str) - assert len(routing_result["routing_reason"]) > 0 - - @pytest.mark.asyncio - async def test_error_handling(self, routing_tool): - """Test error handling for invalid inputs.""" - # Test with invalid action - result = await routing_tool.execute( - action="invalid_action", - tasks=[] - ) - assert result["success"] is False - assert "error" in result - - # Test with empty tasks - result = await routing_tool.execute( - action="route", - tasks=[], - routing_strategy="balanced" - ) - - # Empty tasks should be handled as success case (returning empty recommendations) - if result["success"]: - assert "result" in result - empty_result = result["result"] - assert empty_result["recommendations"] == [] - assert empty_result["total_estimated_cost"] == 0 - else: - # If tool treats empty tasks as error, that's also valid behavior - assert "error" in result - - @pytest.mark.asyncio - async def test_budget_constraints(self, routing_tool): - """Test that budget constraints are respected.""" - result = await routing_tool.execute( - action="route", - tasks=[ - {"task": "Write a 10,000 word essay", "context": "Complex topic"}, - {"task": "Translate entire book", "context": "500 pages"}, - {"task": "Generate comprehensive report", "context": "Annual report"} - ], - routing_strategy="cost_optimized", - constraints={"total_budget": 0.50} # Very low budget - ) - - assert result["success"] - assert "result" in result - - budget_result = result["result"] - assert budget_result["total_estimated_cost"] <= 0.50 * 1.1 # Allow 10% variance - - # Should select cheaper models - for rec in budget_result["recommendations"]: - model = rec["model"].lower() - # Check for budget-friendly models - assert any(cheap in model for cheap in ["nano", "mini", "1b", "gemma", "llama"]) - - @pytest.mark.asyncio - async def test_complex_task_routing(self, routing_tool): - """Test routing for complex multi-step tasks.""" - result = await routing_tool.execute( - action="route", - tasks=[ - { - "task": "Research and synthesize information", - "context": "Quantum computing applications in cryptography", - "requirements": ["deep_analysis", "citations", "technical_accuracy"] - }, - { - "task": "Generate marketing copy", - "context": "Product launch announcement", - "requirements": ["creativity", "engagement", "brand_voice"] - }, - { - "task": "Debug Python code", - "context": "AsyncIO race condition issue", - "requirements": ["code_understanding", "debugging", "solution"] - } - ], - routing_strategy="quality_optimized", - constraints={"total_budget": 20.0} - ) - - assert result["success"] - assert "result" in result - - complex_result = result["result"] - assert len(complex_result["recommendations"]) == 3 - - # Quality-optimized should select more capable models - for i, rec in enumerate(complex_result["recommendations"]): - model = rec["model"].lower() - - # Research task should get a strong reasoning model - if i == 0: - assert any(strong in model for strong in ["opus", "gpt-5", "pro", "sonnet"]) - - # Creative task should get a creative model - elif i == 1: - assert rec["estimated_cost"] > 0 - - # Code task should get a code-capable model - elif i == 2: - assert any(code in model for code in ["gpt", "claude", "codex", "sonnet"]) \ No newline at end of file diff --git a/tests/test_provider_abstractions.py b/tests/test_provider_abstractions.py deleted file mode 100644 index 2cb93c63..00000000 --- a/tests/test_provider_abstractions.py +++ /dev/null @@ -1,337 +0,0 @@ -"""Test provider abstractions with real API calls.""" - -import asyncio -import os -import pytest -from typing import Dict, Any - -from tests.test_infrastructure import create_test_orchestrator, TestModel, TestProvider -from orchestrator.models import ( - UnifiedModelRegistry, - create_registry_from_env, - create_default_configuration, - create_registry_from_config, - ProviderConfig, - AnthropicProvider, -) - -# Note: OpenAIProvider and LocalProvider removed in Claude Skills refactor (Issue #426) - - -class TestProviderAbstractions: - """Test provider abstractions and unified registry.""" - - @pytest.fixture - def registry(self): - """Create a test registry.""" - registry = create_registry_from_env() - return registry - - async def test_registry_creation(self): - """Test creating registry with different methods.""" - # Test creating from environment - registry_env = create_registry_from_env() - assert isinstance(registry_env, UnifiedModelRegistry) - - # Test creating from default config - config = create_default_configuration() - registry_config = create_registry_from_config(config) - assert isinstance(registry_config, UnifiedModelRegistry) - - # Test manual registry creation - registry_manual = UnifiedModelRegistry() - assert isinstance(registry_manual, UnifiedModelRegistry) - - await registry_env.cleanup() - await registry_config.cleanup() - await registry_manual.cleanup() - - async def test_provider_configuration(self): - """Test provider configuration.""" - registry = UnifiedModelRegistry() - - # Configure OpenAI provider - if os.getenv("OPENAI_API_KEY"): - registry.configure_provider( - provider_name="openai-test", - provider_type="openai", - config={ - "api_key": os.getenv("OPENAI_API_KEY"), - "timeout": 30.0, - } - ) - - # Configure Anthropic provider - if os.getenv("ANTHROPIC_API_KEY"): - registry.configure_provider( - provider_name="anthropic-test", - provider_type="anthropic", - config={ - "api_key": os.getenv("ANTHROPIC_API_KEY"), - "timeout": 30.0, - } - ) - - # Note: local provider no longer available in Claude Skills refactor - # Removed to focus on Anthropic-only - - # Check providers are registered - providers = registry.providers - assert len(providers) >= 0 # May have Anthropic if API key available - - await registry.cleanup() - - async def test_openai_provider(self): - """Test OpenAI provider directly.""" - pytest.skip("OpenAI provider removed in Claude Skills refactor (Issue #426)") - - # Test initialization - await provider.initialize() - assert provider.is_initialized - assert len(provider.available_models) > 0 - - # Test model discovery - models = await provider.discover_models() - assert isinstance(models, list) - assert len(models) > 0 - - # Test health check - is_healthy = await provider.health_check() - assert is_healthy is True - - # Test model capabilities - if "gpt-3.5-turbo" in provider.available_models: - capabilities = provider.get_model_capabilities("gpt-3.5-turbo") - assert capabilities.context_window > 0 - assert "generate" in capabilities.supported_tasks - - requirements = provider.get_model_requirements("gpt-3.5-turbo") - assert requirements.memory_gb > 0 - - cost = provider.get_model_cost("gpt-3.5-turbo") - assert cost.input_cost_per_1k_tokens >= 0 - assert not cost.is_free - - # Test model creation - if "gpt-3.5-turbo" in provider.available_models: - model = await provider.create_model("gpt-3.5-turbo") - assert model is not None - assert model.name == "gpt-3.5-turbo" - assert model.provider == "openai" - - # Test actual generation - try: - result = await model.generate("Say 'Hello, world!'", max_tokens=10) - assert isinstance(result, str) - assert len(result) > 0 - print(f"OpenAI model generated: {result}") - except Exception as e: - print(f"OpenAI generation test failed (may be rate limited): {e}") - - await provider.cleanup() - - async def test_anthropic_provider(self): - """Test Anthropic provider directly.""" - if not os.getenv("ANTHROPIC_API_KEY"): - pytest.skip("ANTHROPIC_API_KEY not available") - - config = ProviderConfig( - name="anthropic-test", - api_key=os.getenv("ANTHROPIC_API_KEY"), - timeout=30.0, - max_retries=2, - ) - - provider = AnthropicProvider(config) - - # Test initialization - await provider.initialize() - assert provider.is_initialized - assert len(provider.available_models) > 0 - - # Test model discovery - models = await provider.discover_models() - assert isinstance(models, list) - assert len(models) > 0 - - # Test health check - is_healthy = await provider.health_check() - assert is_healthy is True - - # Test model capabilities - if "claude-3-haiku" in provider.available_models: - capabilities = provider.get_model_capabilities("claude-3-haiku") - assert capabilities.context_window > 0 - assert "generate" in capabilities.supported_tasks - - requirements = provider.get_model_requirements("claude-3-haiku") - assert requirements.memory_gb > 0 - - cost = provider.get_model_cost("claude-3-haiku") - assert cost.input_cost_per_1k_tokens >= 0 - assert not cost.is_free - - # Test model creation - if "claude-3-haiku" in provider.available_models: - model = await provider.create_model("claude-3-haiku") - assert model is not None - assert model.name == "claude-3-haiku" - assert model.provider == "anthropic" - - # Test actual generation - try: - result = await model.generate("Say 'Hello, world!'", max_tokens=10) - assert isinstance(result, str) - assert len(result) > 0 - print(f"Anthropic model generated: {result}") - except Exception as e: - print(f"Anthropic generation test failed (may be rate limited): {e}") - - await provider.cleanup() - - async def test_local_provider(self): - """Test local provider (Ollama).""" - pytest.skip("Local provider removed in Claude Skills refactor (Issue #426)") - - # Test initialization (should work even if Ollama is not running) - await provider.initialize() - assert provider.is_initialized - - # Test model discovery - models = await provider.discover_models() - assert isinstance(models, list) - # Models list might be empty if Ollama is not running, that's OK - - # Test health check (might be False if Ollama not running) - is_healthy = await provider.health_check() - assert isinstance(is_healthy, bool) - - # Test model capabilities for known models - if provider.supports_model("gemma2:2b"): - capabilities = provider.get_model_capabilities("gemma2:2b") - assert capabilities.context_window > 0 - assert "generate" in capabilities.supported_tasks - - requirements = provider.get_model_requirements("gemma2:2b") - assert requirements.memory_gb > 0 - - cost = provider.get_model_cost("gemma2:2b") - assert cost.input_cost_per_1k_tokens == 0 # Local models are free - assert cost.is_free - - await provider.cleanup() - - async def test_unified_registry(self): - """Test unified registry with multiple providers.""" - registry = create_registry_from_env() - - # Initialize registry - await registry.initialize() - assert registry.is_initialized - - # Check providers - providers = registry.providers - assert isinstance(providers, dict) - assert len(providers) >= 1 # Should have at least local provider - - # Test model discovery - all_models = await registry.discover_all_models() - assert isinstance(all_models, dict) - - # Test health checks - health_status = await registry.health_check() - assert isinstance(health_status, dict) - - # Test registry info - info = registry.get_registry_info() - assert "provider_count" in info - assert "total_models" in info - assert "providers" in info - - # Test model listing - model_list = registry.list_models() - assert isinstance(model_list, dict) - - # Test finding a model (if any are available) - available_models = registry.available_models - if available_models: - first_model = next(iter(available_models.keys())) - provider_name = registry.find_model(first_model) - assert provider_name is not None - assert provider_name in providers - - # Test getting the model - try: - model = await registry.get_model(first_model) - assert model is not None - assert model.name == first_model - print(f"Successfully created model: {first_model} from provider: {provider_name}") - except Exception as e: - print(f"Failed to create model {first_model}: {e}") - - await registry.cleanup() - - async def test_model_generation_integration(self): - """Integration test with actual model generation.""" - registry = create_registry_from_env() - await registry.initialize() - - available_models = registry.available_models - test_prompt = "What is 2 + 2?" - - for model_name, provider_name in available_models.items(): - # Skip expensive models in testing - if any(expensive in model_name.lower() for expensive in ["gpt-4", "opus", "70b"]): - continue - - try: - print(f"\nTesting model: {model_name} from provider: {provider_name}") - model = await registry.get_model(model_name) - - # Test generation - result = await model.generate(test_prompt, max_tokens=20, temperature=0.1) - assert isinstance(result, str) - assert len(result) > 0 - print(f"Generated: {result}") - - # Test health check - is_healthy = await model.health_check() - print(f"Health check: {is_healthy}") - - # Only test one model to avoid API rate limits - break - - except Exception as e: - print(f"Failed to test model {model_name}: {e}") - continue - - await registry.cleanup() - - -def test_provider_abstractions_sync(): - """Synchronous wrapper for async tests.""" - async def run_tests(): - test_instance = TestProviderAbstractions() - - # Run basic tests - await test_instance.test_registry_creation() - await test_instance.test_provider_configuration() - - # Test individual providers (if API keys available) - # Note: OpenAI and Local providers removed in Claude Skills refactor - - if os.getenv("ANTHROPIC_API_KEY"): - await test_instance.test_anthropic_provider() - - # Test unified registry - await test_instance.test_unified_registry() - - # Integration test (if any models are available) - await test_instance.test_model_generation_integration() - - asyncio.run(run_tests()) - - -if __name__ == "__main__": - test_provider_abstractions_sync() - print("All provider abstraction tests passed!") \ No newline at end of file diff --git a/tests/test_provider_retirement.py b/tests/test_provider_retirement.py new file mode 100644 index 00000000..4be9299f --- /dev/null +++ b/tests/test_provider_retirement.py @@ -0,0 +1,140 @@ +"""Contract tests for the provider retirement (#430). + +The product's providers are Dartmouth Chat and the HuggingFace Inference API +(ADR 0001). The Anthropic / OpenAI / Google / Ollama / local-HuggingFace +adapters are retired: not importable, not exported, not offered by the +packaged default model pool, and never silently constructed as a fallback. + +A ``models.yaml`` written before the retirement may still name those sources; +population must skip each such entry with a warning that names the source, +and must never raise -- an old config file is not an error. +""" + +import logging +import os + +import pytest +import yaml + +pytestmark = pytest.mark.unit + + +_RETIRED_SOURCES = {"ollama", "openai", "anthropic", "google", "huggingface"} + + +def test_packaged_models_yaml_offers_no_retired_providers(): + """The shipped default model pool must not offer retired providers.""" + from orchestrator.install_configs import packaged_config_path + + path = packaged_config_path("models.yaml") + assert path.exists(), f"packaged models.yaml missing at {path}" + config = yaml.safe_load(path.read_text()) or {} + models = config.get("models") or [] + assert isinstance(models, list) + offered = {m.get("source") for m in models if isinstance(m, dict)} + assert not (offered & _RETIRED_SOURCES), ( + f"packaged models.yaml still offers retired providers: " + f"{sorted(offered & _RETIRED_SOURCES)}" + ) + + +def _seal_credentials(monkeypatch, tmp_path): + """Cut every provider credential source, including import-time paths. + + The credential modules compute their credential-file paths from + ``Path.home()`` at import time, so patching ``HOME`` afterwards changes + nothing -- the module constants must be redirected instead. + """ + from orchestrator.models import dartmouth_credentials, huggingface_credentials + + monkeypatch.delenv("DARTMOUTH_CHAT_API_KEY", raising=False) + monkeypatch.setattr( + dartmouth_credentials, "_ORCHESTRATOR_ENV_FILE", tmp_path / "nope.env" + ) + monkeypatch.setattr( + dartmouth_credentials, "_LLMXIVE_CREDENTIALS_FILE", tmp_path / "nope.toml" + ) + monkeypatch.delenv("HF_TOKEN", raising=False) + monkeypatch.setattr( + huggingface_credentials, "_ORCHESTRATOR_ENV_FILE", tmp_path / "nope.env" + ) + monkeypatch.setattr( + huggingface_credentials, "_HF_CLI_TOKEN_FILE", tmp_path / "nope-token" + ) + + +def test_populate_skips_retired_sources_with_a_warning(tmp_path, monkeypatch, caplog): + """A pre-retirement models.yaml is skipped entry-by-entry, never raised on.""" + from orchestrator._api import populate_model_registry + from orchestrator.models.model_registry import ModelRegistry + from orchestrator.utils import model_config_loader as loader_module + + config_file = tmp_path / "models.yaml" + config_file.write_text( + "models:\n" + + "".join( + f" - source: {source}\n name: some-model\n size: 1b\n" + for source in sorted(_RETIRED_SOURCES) + ) + ) + loader = loader_module.ModelConfigLoader(config_path=config_file) + monkeypatch.setattr(loader_module, "get_model_config_loader", lambda: loader) + _seal_credentials(monkeypatch, tmp_path) + + registry = ModelRegistry() + with caplog.at_level(logging.WARNING): + populate_model_registry(registry) + + assert registry.list_models() == [] + warned = [r.getMessage() for r in caplog.records if r.levelno >= logging.WARNING] + for source in sorted(_RETIRED_SOURCES): + assert any(source in message for message in warned), ( + f"no warning named retired source {source!r}: {warned}" + ) + + +def test_populate_without_any_credentials_registers_nothing(tmp_path, monkeypatch): + """No keys and an empty pool is a valid state, not an error.""" + from orchestrator._api import populate_model_registry + from orchestrator.models.model_registry import ModelRegistry + from orchestrator.utils import model_config_loader as loader_module + + config_file = tmp_path / "models.yaml" + config_file.write_text("models: []\n") + loader = loader_module.ModelConfigLoader(config_path=config_file) + monkeypatch.setattr(loader_module, "get_model_config_loader", lambda: loader) + _seal_credentials(monkeypatch, tmp_path) + + registry = ModelRegistry() + populate_model_registry(registry) + assert registry.list_models() == [] + + +def test_hybrid_control_system_without_registry_never_constructs_a_provider(): + """The retired 'no registry? build a gpt-4 client' fallback must stay gone.""" + import asyncio + + from orchestrator.control_systems.hybrid_control_system import ( + HybridControlSystem, + ) + from orchestrator.core.task import Task + + control_system = HybridControlSystem(model_registry=None) + task = Task(id="t1", name="t1", action="analyze_text", parameters={"text": "x"}) + result = asyncio.run(control_system._handle_analyze_text(task, {})) + assert result["success"] is False + assert "No suitable model" in result["error"] + + +def test_public_surface_drops_retired_model_exports(): + """Retired adapters are not reachable from the public package surface.""" + import orchestrator + from orchestrator.models import providers + + for name in ("HuggingFaceModel", "OllamaModel"): + assert not hasattr(orchestrator, name), ( + f"orchestrator.{name} still resolves after retirement" + ) + assert not hasattr(providers, "AnthropicProvider"), ( + "models.providers.AnthropicProvider still resolves after retirement" + ) diff --git a/tests/test_real_api_anthropic_integration.py b/tests/test_real_api_anthropic_integration.py deleted file mode 100644 index 4f77e344..00000000 --- a/tests/test_real_api_anthropic_integration.py +++ /dev/null @@ -1,479 +0,0 @@ -""" -Real API Integration Tests - Anthropic -Tests the enhanced model requirements specification with actual Anthropic API integration. -""" - -import pytest -import os -from unittest.mock import patch -from orchestrator.models.anthropic_model import AnthropicModel -from orchestrator.models.model_selector import ModelSelector, ModelSelectionCriteria -from orchestrator.models.model_registry import ModelRegistry -from orchestrator.core.model import ModelCapabilities, ModelCost - -from tests.test_infrastructure import create_test_orchestrator, TestModel, TestProvider - - -class TestRealAnthropicIntegration: - """Test enhanced model requirements with real Anthropic models.""" - - @pytest.fixture - def anthropic_registry(self): - """Create registry with real Anthropic models.""" - registry = ModelRegistry() - - # Only create models if API key is available (for CI/CD safety) - api_key = os.getenv("ANTHROPIC_API_KEY") - if not api_key: - pytest.skip("ANTHROPIC_API_KEY not available for real API testing") - - # Claude Sonnet 4 - Premium analysis model - claude_sonnet4 = AnthropicModel( - name="claude-sonnet-4-20250514", - api_key=api_key - ) - # Override for testing consistency - claude_sonnet4._size_billions = 200.0 # Estimated - claude_sonnet4._expertise = ["analysis", "research", "reasoning", "creative"] - registry.register_model(claude_sonnet4) - - # Claude Haiku - Fast and efficient - claude_haiku = AnthropicModel( - name="claude-3-haiku-20240307", - api_key=api_key - ) - # Override for testing - claude_haiku._size_billions = 13.0 # Estimated smaller model - claude_haiku._expertise = ["general", "fast", "chat"] - registry.register_model(claude_haiku) - - # Claude Opus - Highest capability - claude_opus = AnthropicModel( - name="claude-3-opus-20240229", - api_key=api_key - ) - # Override for testing - claude_opus._size_billions = 400.0 # Estimated largest - claude_opus._expertise = ["analysis", "research", "reasoning", "creative", "code"] - registry.register_model(claude_opus) - - return registry - - @pytest.fixture - def anthropic_selector(self, anthropic_registry): - """Create model selector with Anthropic registry.""" - return ModelSelector(anthropic_registry) - - def test_anthropic_model_initialization_with_enhanced_features(self): - """Test that Anthropic models initialize with enhanced Issue 194 features.""" - api_key = os.getenv("ANTHROPIC_API_KEY") - if not api_key: - pytest.skip("ANTHROPIC_API_KEY not available") - - model = AnthropicModel(name="claude-sonnet-4-20250514", api_key=api_key) - - # Test enhanced attributes from Issue 194 - assert hasattr(model, '_expertise') - assert hasattr(model, '_size_billions') - assert isinstance(model._expertise, list) - assert isinstance(model._size_billions, (int, float)) - assert model._size_billions > 0 - - # Test cost information - assert model.cost is not None - assert isinstance(model.cost, ModelCost) - assert not model.cost.is_free # Anthropic models are paid - - # Test enhanced cost methods from Issue 194 - task_cost = model.cost.estimate_cost_for_budget_period("per-task") - assert isinstance(task_cost, float) - assert task_cost > 0 - - efficiency = model.cost.get_cost_efficiency_score(0.95) - assert isinstance(efficiency, float) - assert efficiency > 0 - - @pytest.mark.asyncio - async def test_expertise_based_selection_with_anthropic(self, anthropic_selector): - """Test expertise-based selection with real Anthropic models.""" - # Low expertise - should prefer faster models - low_criteria = ModelSelectionCriteria(expertise="low") - model = await anthropic_selector.select_model(low_criteria) - - # Should select Haiku (fast model) or model that meets low requirements - assert model is not None - assert "fast" in model._expertise or "haiku" in model.name.lower() - - # Very high expertise - should prefer most capable models - very_high_criteria = ModelSelectionCriteria(expertise="very-high") - model = await anthropic_selector.select_model(very_high_criteria) - - # Should select Opus or Sonnet 4 for very high expertise - assert model is not None - assert "analysis" in model._expertise or "research" in model._expertise - assert model.name in ["claude-3-opus-20240229", "claude-sonnet-4-20250514"] - - @pytest.mark.asyncio - async def test_cost_constraint_selection_with_anthropic(self, anthropic_selector): - """Test cost constraint selection with real Anthropic pricing.""" - # Moderate budget - should prefer efficient models - budget_criteria = ModelSelectionCriteria( - cost_limit=1.0, # Moderate budget - budget_period="per-task" - ) - - model = await anthropic_selector.select_model(budget_criteria) - assert model is not None - - # Verify the selected model is within budget - estimated_cost = model.cost.estimate_cost_for_budget_period("per-task") - assert estimated_cost <= 1.0 - - # Higher budget - should allow premium models - premium_criteria = ModelSelectionCriteria( - cost_limit=10.0, # Higher budget - budget_period="per-task" - ) - - model = await anthropic_selector.select_model(premium_criteria) - assert model is not None - - # Should be within budget - estimated_cost = model.cost.estimate_cost_for_budget_period("per-task") - assert estimated_cost <= 10.0 - - @pytest.mark.asyncio - async def test_size_constraint_selection_with_anthropic(self, anthropic_selector): - """Test size constraint selection with Anthropic models.""" - # Small model preference - small_criteria = ModelSelectionCriteria(max_model_size=50.0) - model = await anthropic_selector.select_model(small_criteria) - - assert model is not None - assert model._size_billions <= 50.0 - # Should likely select Haiku - assert "haiku" in model.name.lower() - - # Large model preference - large_criteria = ModelSelectionCriteria(min_model_size=200.0) - model = await anthropic_selector.select_model(large_criteria) - - assert model is not None - assert model._size_billions >= 200.0 - # Should select Opus or Sonnet 4 - assert model.name in ["claude-3-opus-20240229", "claude-sonnet-4-20250514"] - - @pytest.mark.asyncio - async def test_modality_selection_with_anthropic(self, anthropic_selector): - """Test modality-based selection with Anthropic models.""" - # Analysis modality requirement (map to text for Anthropic) - analysis_criteria = ModelSelectionCriteria(modalities=["text"]) - model = await anthropic_selector.select_model(analysis_criteria) - - # Should select any Anthropic model (all support text) - assert model is not None - assert "claude" in model.name.lower() - - # Code modality requirement - code_criteria = ModelSelectionCriteria(modalities=["code"]) - model = await anthropic_selector.select_model(code_criteria) - - # Should select a code-capable model (Opus or Sonnet 4) - assert model is not None - assert "code" in model._expertise or model.capabilities.code_specialized - - @pytest.mark.asyncio - async def test_complex_criteria_with_anthropic(self, anthropic_selector): - """Test complex multi-criteria selection with Anthropic models.""" - complex_criteria = ModelSelectionCriteria( - expertise="high", - min_model_size=50.0, - max_model_size=500.0, - cost_limit=5.0, - budget_period="per-task", - selection_strategy="accuracy_optimized" - ) - - model = await anthropic_selector.select_model(complex_criteria) - assert model is not None - - # Verify it meets size constraints - assert 50.0 <= model._size_billions <= 500.0 - - # Verify cost constraint - estimated_cost = model.cost.estimate_cost_for_budget_period("per-task") - assert estimated_cost <= 5.0 - - # Should be high-accuracy model - assert model.capabilities.accuracy_score > 0.9 - - def test_anthropic_capability_detection(self, anthropic_registry): - """Test capability detection with real Anthropic models.""" - # Test capability analysis for each model - for model_key, model in anthropic_registry.models.items(): - analysis = anthropic_registry.detect_model_capabilities(model) - - # Should have complete analysis - assert "basic_capabilities" in analysis - assert "advanced_capabilities" in analysis - assert "performance_metrics" in analysis - assert "expertise_analysis" in analysis - assert "cost_analysis" in analysis - assert "suitability_scores" in analysis - - # Cost analysis should reflect Anthropic pricing - cost_analysis = analysis["cost_analysis"] - assert cost_analysis["type"] == "paid" - assert cost_analysis["cost_per_1k_avg"] > 0 - - # Expertise analysis should recognize Claude's strengths - expertise_analysis = analysis["expertise_analysis"] - if "opus" in model.name.lower(): - assert expertise_analysis["level"] == "very-high" - elif "haiku" in model.name.lower(): - assert expertise_analysis["level"] in ["low", "medium"] - - # Should have reasonable suitability scores - scores = analysis["suitability_scores"] - for capability, score in scores.items(): - assert 0.0 <= score <= 1.0 - - # Anthropic models should score well on analysis - assert scores["analysis"] > 0.7 - - def test_anthropic_task_recommendations(self, anthropic_registry): - """Test task-based recommendations with Anthropic models.""" - # Analysis-related task - analysis_recs = anthropic_registry.recommend_models_for_task( - "Analyze market research data and provide insights", - max_recommendations=2 - ) - - assert len(analysis_recs) > 0 - - for rec in analysis_recs: - assert "model" in rec - assert "reasoning" in rec - assert rec["suitability_score"] > 0 - - # Should recommend capable models for analysis - model = rec["model"] - assert "analysis" in model._expertise or model.capabilities.accuracy_score > 0.9 - - # Creative writing task - creative_recs = anthropic_registry.recommend_models_for_task( - "Write a creative story about space exploration", - max_recommendations=2 - ) - - # Should recommend creative-capable models - creative_capable_found = False - for rec in creative_recs: - if "creative" in rec["model"]._expertise: - creative_capable_found = True - break - - assert creative_capable_found - - def test_anthropic_cost_analysis_integration(self, anthropic_registry): - """Test cost analysis with real Anthropic pricing.""" - for model in anthropic_registry.models.values(): - # Test budget period estimates - task_cost = model.cost.estimate_cost_for_budget_period("per-task") - pipeline_cost = model.cost.estimate_cost_for_budget_period("per-pipeline") - hour_cost = model.cost.estimate_cost_for_budget_period("per-hour") - - # Costs should increase with usage - assert task_cost <= pipeline_cost <= hour_cost - - # Test cost breakdown - breakdown = model.cost.get_cost_breakdown(1000, 500) # 1000 input, 500 output - assert breakdown["total_cost"] > 0 - assert breakdown["input_cost"] > 0 - assert breakdown["output_cost"] > 0 - assert not breakdown["is_free"] - - # Test cost efficiency - Claude models should have good efficiency - efficiency = model.cost.get_cost_efficiency_score(model.capabilities.accuracy_score) - assert efficiency > 0 - - # Opus should be expensive but highly efficient due to quality - if "opus" in model.name.lower(): - assert model.capabilities.accuracy_score > 0.95 - - @pytest.mark.asyncio - async def test_anthropic_fallback_strategies(self, anthropic_selector): - """Test fallback strategies with Anthropic models.""" - # Impossible requirements with cheapest fallback - impossible_criteria = ModelSelectionCriteria( - min_model_size=10000.0, # Impossibly large - fallback_strategy="cheapest" - ) - - model = await anthropic_selector.select_model(impossible_criteria) - assert model is not None - - # Should fallback to most cost-effective model (likely Haiku) - assert "haiku" in model.name.lower() - - # Best available fallback - best_criteria = ModelSelectionCriteria( - min_model_size=10000.0, # Impossibly large - fallback_strategy="best_available" - ) - - model = await anthropic_selector.select_model(best_criteria) - assert model is not None - - # Should fallback to highest quality model (Opus or Sonnet 4) - assert model.capabilities.accuracy_score > 0.9 - assert model.name in ["claude-3-opus-20240229", "claude-sonnet-4-20250514"] - - @pytest.mark.asyncio - async def test_anthropic_yaml_integration(self, anthropic_selector): - """Test YAML requirements parsing with Anthropic models.""" - # Simulate YAML requirements for research task - yaml_requirements = { - "expertise": "very-high", - "modalities": ["text"], - "min_size": "100B", - "max_size": "1000B", - "cost_limit": 8.0, - "budget_period": "per-task", - "fallback_strategy": "best_available" - } - - # Parse and select - criteria = anthropic_selector.parse_requirements_from_yaml(yaml_requirements) - model = await anthropic_selector.select_model(criteria) - - assert model is not None - assert model._size_billions >= 100.0 - assert model._size_billions <= 1000.0 - - # Should be very high expertise (Opus or Sonnet 4) - registry = anthropic_selector.registry - assert registry._meets_expertise_level(model, "very-high") - - @pytest.mark.asyncio - async def test_anthropic_real_health_check(self, anthropic_registry): - """Test health checking with real Anthropic models.""" - # Get a model from registry - model = list(anthropic_registry.models.values())[0] - - # Mock the health check to avoid real API calls during testing - with patch.object(model, 'health_check', return_value=True) as mock_health: - is_healthy = await model.health_check() - assert is_healthy - mock_health.assert_called_once() - - # Test with registry health filtering - healthy_models = await anthropic_registry._filter_by_health([model]) - # Should return the model if health check passes - assert len(healthy_models) >= 0 # May be 0 if health check is mocked to fail - - def test_anthropic_expertise_hierarchy_integration(self, anthropic_registry): - """Test expertise hierarchy with Anthropic models.""" - # Test that models are properly classified in hierarchy - haiku_model = None - opus_model = None - sonnet_model = None - - for model in anthropic_registry.models.values(): - if "haiku" in model.name.lower(): - haiku_model = model - elif "opus" in model.name.lower(): - opus_model = model - elif "sonnet" in model.name.lower(): - sonnet_model = model - - if haiku_model: - # Haiku should meet low/medium requirements - assert anthropic_registry._meets_expertise_level(haiku_model, "low") - assert anthropic_registry._meets_expertise_level(haiku_model, "medium") - - if opus_model: - # Opus should meet all expertise levels (very high capability) - assert anthropic_registry._meets_expertise_level(opus_model, "low") - assert anthropic_registry._meets_expertise_level(opus_model, "medium") - assert anthropic_registry._meets_expertise_level(opus_model, "high") - assert anthropic_registry._meets_expertise_level(opus_model, "very-high") - - if sonnet_model: - # Sonnet 4 should meet very high requirements - assert anthropic_registry._meets_expertise_level(sonnet_model, "very-high") - - def test_anthropic_model_comparison(self, anthropic_registry): - """Test model comparison capabilities with Anthropic models.""" - models = list(anthropic_registry.models.values()) - if len(models) >= 2: - model1, model2 = models[0], models[1] - - # Test cost comparison - comparison = model1.cost.compare_cost_with(model2.cost) - assert "cost_ratio" in comparison - assert "savings" in comparison - assert "percent_savings" in comparison - - # Test capability matrix - matrix = anthropic_registry.get_capability_matrix() - assert len(matrix) == len(models) - - for model_key in matrix: - scores = matrix[model_key] - # All models should have analysis capability - assert "analysis" in scores - assert scores["analysis"] > 0.5 # Anthropic models excel at analysis - - -@pytest.mark.integration -class TestAnthropicLiveIntegration: - """ - Live integration tests that make actual API calls. - These tests are marked separately and should be run with caution. - """ - - @pytest.mark.skipif( - not os.getenv("ANTHROPIC_API_KEY") or not os.getenv("RUN_LIVE_TESTS"), - reason="Requires ANTHROPIC_API_KEY and RUN_LIVE_TESTS=1 to run live tests" - ) - @pytest.mark.asyncio - async def test_live_anthropic_generation_with_enhanced_selection(self): - """Test actual text generation with enhanced model selection.""" - registry = ModelRegistry() - - # Create real Anthropic model - model = AnthropicModel( - name="claude-sonnet-4-20250514", - api_key=os.getenv("ANTHROPIC_API_KEY") - ) - model._expertise = ["analysis", "research", "reasoning"] - model._size_billions = 200.0 - - registry.register_model(model) - selector = ModelSelector(registry) - - # Use enhanced selection - criteria = ModelSelectionCriteria( - expertise="very-high", - cost_limit=2.0, - budget_period="per-task" - ) - - selected_model = await selector.select_model(criteria) - assert selected_model is not None - - # Make actual API call - response = await selected_model.generate( - "What is the capital of France?", - temperature=0.1, - max_tokens=20 - ) - - assert isinstance(response, str) - assert len(response) > 0 - assert "Paris" in response # Should contain the answer - - -if __name__ == "__main__": - pytest.main([__file__, "-v", "-m", "not integration"]) \ No newline at end of file diff --git a/tests/test_real_api_ollama_integration.py b/tests/test_real_api_ollama_integration.py deleted file mode 100644 index 1203e041..00000000 --- a/tests/test_real_api_ollama_integration.py +++ /dev/null @@ -1,487 +0,0 @@ -""" -Real API Integration Tests - Ollama -Tests the enhanced model requirements specification with actual Ollama local models. -""" - -import pytest -import subprocess -import time -from unittest.mock import patch, MagicMock -from orchestrator.integrations.ollama_model import OllamaModel -from orchestrator.models.model_selector import ModelSelector, ModelSelectionCriteria -from orchestrator.models.model_registry import ModelRegistry -from orchestrator.core.model import ModelCapabilities, ModelCost - -from tests.test_infrastructure import create_test_orchestrator, TestModel, TestProvider - - -def is_ollama_running(): - """Check if Ollama is running locally.""" - try: - import requests - response = requests.get("http://localhost:11434/api/tags", timeout=5) - return response.status_code == 200 - except: - return False - - -def get_available_ollama_models(): - """Get list of locally available Ollama models.""" - if not is_ollama_running(): - return [] - - try: - import requests - response = requests.get("http://localhost:11434/api/tags", timeout=5) - if response.status_code == 200: - models = response.json().get("models", []) - return [model["name"] for model in models] - except: - pass - return [] - - -class TestRealOllamaIntegration: - """Test enhanced model requirements with real Ollama models.""" - - @pytest.fixture - def ollama_registry(self): - """Create registry with real Ollama models.""" - registry = ModelRegistry() - - # Check if Ollama is running - if not is_ollama_running(): - pytest.skip("Ollama is not running locally") - - available_models = get_available_ollama_models() - if not available_models: - pytest.skip("No Ollama models are locally available") - - # Create models based on what's available - # Try common model names - test_models = [ - ("gemma3:1b", 1.0, ["fast", "compact", "general"]), - ("gemma3:4b", 4.0, ["general", "chat", "reasoning"]), - ("llama3.2:3b", 3.0, ["fast", "compact", "general"]), - ("deepseek-r1:1.5b", 1.5, ["code", "reasoning", "programming"]), - ("deepseek-r1:8b", 8.0, ["general", "chat", "reasoning"]), - ] - - models_created = 0 - for model_name, size, expertise in test_models: - if model_name in available_models or models_created == 0: # Always create at least one for testing - try: - model = OllamaModel(model_name=model_name) - # Override for testing consistency - model._size_billions = size - model._expertise = expertise - registry.register_model(model) - models_created += 1 - except Exception as e: - print(f"Could not create model {model_name}: {e}") - continue - - if models_created == 0: - pytest.skip("Could not create any Ollama models for testing") - - return registry - - @pytest.fixture - def ollama_selector(self, ollama_registry): - """Create model selector with Ollama registry.""" - return ModelSelector(ollama_registry) - - def test_ollama_model_initialization_with_enhanced_features(self): - """Test that Ollama models initialize with enhanced Issue 194 features.""" - if not is_ollama_running(): - pytest.skip("Ollama is not running locally") - - model = OllamaModel(model_name="gemma3:1b") - - # Test enhanced attributes from Issue 194 - assert hasattr(model, '_expertise') - assert hasattr(model, '_size_billions') - assert isinstance(model._expertise, list) - assert isinstance(model._size_billions, (int, float)) - assert model._size_billions > 0 - - # Test cost information - Ollama models should be free - assert model.cost is not None - assert isinstance(model.cost, ModelCost) - assert model.cost.is_free # Ollama models are free - - # Test enhanced cost methods from Issue 194 - task_cost = model.cost.estimate_cost_for_budget_period("per-task") - assert task_cost == 0.0 # Should be free - - efficiency = model.cost.get_cost_efficiency_score(0.8) - assert efficiency == 100.0 # Free models have maximum efficiency - - @pytest.mark.asyncio - async def test_expertise_based_selection_with_ollama(self, ollama_selector): - """Test expertise-based selection with real Ollama models.""" - # Low expertise - should prefer fast/compact models - low_criteria = ModelSelectionCriteria(expertise="low") - model = await ollama_selector.select_model(low_criteria) - - # Should select a model that meets low expertise requirements - assert model is not None - registry = ollama_selector.registry - assert registry._meets_expertise_level(model, "low") - - # Should prefer models with "fast" or "compact" in expertise - if len(ollama_selector.registry.models) > 1: - fast_model_found = "fast" in model._expertise or "compact" in model._expertise - # If no fast model available, any model meeting low criteria is acceptable - assert fast_model_found or registry._meets_expertise_level(model, "low") - - @pytest.mark.asyncio - async def test_cost_constraint_selection_with_ollama(self, ollama_selector): - """Test cost constraint selection with Ollama models (all free).""" - # Any budget should work with free models - budget_criteria = ModelSelectionCriteria( - cost_limit=0.0, # Even zero budget - budget_period="per-task" - ) - - model = await ollama_selector.select_model(budget_criteria) - assert model is not None - - # Verify the selected model is within budget (free) - estimated_cost = model.cost.estimate_cost_for_budget_period("per-task") - assert estimated_cost == 0.0 - - # Higher budget - should still work with free models - premium_criteria = ModelSelectionCriteria( - cost_limit=100.0, # High budget - budget_period="per-task" - ) - - model = await ollama_selector.select_model(premium_criteria) - assert model is not None - assert model.cost.is_free - - @pytest.mark.asyncio - async def test_size_constraint_selection_with_ollama(self, ollama_selector): - """Test size constraint selection with Ollama models.""" - # Small model preference - small_criteria = ModelSelectionCriteria(max_model_size=5.0) - model = await ollama_selector.select_model(small_criteria) - - assert model is not None - assert model._size_billions <= 5.0 - - # If we have multiple models, test large model preference - if len(ollama_selector.registry.models) > 1: - large_criteria = ModelSelectionCriteria(min_model_size=3.0) - model = await ollama_selector.select_model(large_criteria) - - assert model is not None - assert model._size_billions >= 3.0 - - @pytest.mark.asyncio - async def test_modality_selection_with_ollama(self, ollama_selector): - """Test modality-based selection with Ollama models.""" - # Code modality requirement - code_criteria = ModelSelectionCriteria(modalities=["code"]) - model = await ollama_selector.select_model(code_criteria) - - # Should select a code-capable model if available - assert model is not None - - # Check if it's actually code-specialized - if "code" in model._expertise: - assert "code" in model._expertise - # Otherwise, any model should work as fallback - - @pytest.mark.asyncio - async def test_complex_criteria_with_ollama(self, ollama_selector): - """Test complex multi-criteria selection with Ollama models.""" - complex_criteria = ModelSelectionCriteria( - expertise="medium", - min_model_size=1.0, - max_model_size=10.0, - cost_limit=0.0, # Free only - budget_period="per-task", - selection_strategy="balanced" - ) - - model = await ollama_selector.select_model(complex_criteria) - assert model is not None - - # Verify it meets size constraints - assert 1.0 <= model._size_billions <= 10.0 - - # Verify cost constraint (should be free) - assert model.cost.is_free - - def test_ollama_capability_detection(self, ollama_registry): - """Test capability detection with real Ollama models.""" - # Test capability analysis for each model - for model_key, model in ollama_registry.models.items(): - analysis = ollama_registry.detect_model_capabilities(model) - - # Should have complete analysis - assert "basic_capabilities" in analysis - assert "advanced_capabilities" in analysis - assert "performance_metrics" in analysis - assert "expertise_analysis" in analysis - assert "cost_analysis" in analysis - assert "suitability_scores" in analysis - - # Cost analysis should reflect free pricing - cost_analysis = analysis["cost_analysis"] - assert cost_analysis["type"] == "free" - assert cost_analysis["cost_per_1k_avg"] == 0.0 - assert cost_analysis["budget_friendly"] == True - assert cost_analysis["efficiency_score"] == 100.0 - - # Should have reasonable suitability scores - scores = analysis["suitability_scores"] - for capability, score in scores.items(): - assert 0.0 <= score <= 1.0 - - # Free models should have maximum budget score - assert scores["budget_constrained"] == 1.0 - - def test_ollama_task_recommendations(self, ollama_registry): - """Test task-based recommendations with Ollama models.""" - # Code-related task - code_recs = ollama_registry.recommend_models_for_task( - "Help me write Python code", - max_recommendations=2 - ) - - assert len(code_recs) > 0 - - for rec in code_recs: - assert "model" in rec - assert "reasoning" in rec - assert rec["suitability_score"] > 0 - - # Should have reasonable reasoning - assert "free" in rec["reasoning"].lower() # Should mention it's free - - # General chat task - chat_recs = ollama_registry.recommend_models_for_task( - "Have a conversation with me", - max_recommendations=2 - ) - - assert len(chat_recs) > 0 - # Should recommend models suitable for chat - for rec in chat_recs: - model = rec["model"] - assert "general" in model._expertise or "chat" in model._expertise - - def test_ollama_cost_analysis_integration(self, ollama_registry): - """Test cost analysis with Ollama models (all free).""" - for model in ollama_registry.models.values(): - # Test budget period estimates - should all be 0 - task_cost = model.cost.estimate_cost_for_budget_period("per-task") - pipeline_cost = model.cost.estimate_cost_for_budget_period("per-pipeline") - hour_cost = model.cost.estimate_cost_for_budget_period("per-hour") - - assert task_cost == 0.0 - assert pipeline_cost == 0.0 - assert hour_cost == 0.0 - - # Test cost breakdown - breakdown = model.cost.get_cost_breakdown(1000, 500) - assert breakdown["total_cost"] == 0.0 - assert breakdown["input_cost"] == 0.0 - assert breakdown["output_cost"] == 0.0 - assert breakdown["is_free"] == True - - # Test cost efficiency - should be maximum - efficiency = model.cost.get_cost_efficiency_score(0.8) - assert efficiency == 100.0 - - # Test budget compliance - should always be within budget - assert model.cost.is_within_budget(0.0, "per-task") # Even zero budget - assert model.cost.is_within_budget(1000.0, "per-hour") - - @pytest.mark.asyncio - async def test_ollama_fallback_strategies(self, ollama_selector): - """Test fallback strategies with Ollama models.""" - # Impossible requirements with cheapest fallback - impossible_criteria = ModelSelectionCriteria( - min_model_size=1000.0, # Impossibly large - fallback_strategy="cheapest" - ) - - model = await ollama_selector.select_model(impossible_criteria) - assert model is not None - - # All Ollama models are free, so should fallback to any available - assert model.cost.is_free - - # Best available fallback - best_criteria = ModelSelectionCriteria( - min_model_size=1000.0, # Impossibly large - fallback_strategy="best_available" - ) - - model = await ollama_selector.select_model(best_criteria) - assert model is not None - - # Should fallback to best available model - assert model.cost.is_free - - @pytest.mark.asyncio - async def test_ollama_yaml_integration(self, ollama_selector): - """Test YAML requirements parsing with Ollama models.""" - # Simulate YAML requirements suitable for local models - yaml_requirements = { - "expertise": "medium", - "modalities": ["text"], - "min_size": "1B", - "max_size": "10B", - "cost_limit": 0.0, # Free only - "budget_period": "per-task", - "fallback_strategy": "best_available" - } - - # Parse and select - criteria = ollama_selector.parse_requirements_from_yaml(yaml_requirements) - model = await ollama_selector.select_model(criteria) - - assert model is not None - assert model._size_billions >= 1.0 - assert model._size_billions <= 10.0 - assert model.cost.is_free - - def test_ollama_model_size_parsing_integration(self): - """Test model size parsing with Ollama model names.""" - from orchestrator.utils.model_utils import parse_model_size - - # Test Ollama model names with size information - test_cases = [ - ("gemma3:1b", None, 1.0), - ("gemma3:4b", None, 4.0), - ("llama3.2:3b", None, 3.0), - ("deepseek-r1:1.5b", None, 1.5), - ("deepseek-r1:8b", None, 8.0), - ("deepseek-r1:32b", None, 32.0), - ] - - for model_name, size_str, expected in test_cases: - result = parse_model_size(model_name, size_str) - assert result == expected, f"Failed for {model_name}: expected {expected}, got {result}" - - def test_ollama_expertise_detection(self): - """Test expertise detection logic for Ollama models.""" - # Test with different model configurations - model1 = OllamaModel(model_name="gemma3:1b") - assert "fast" in model1._expertise or "compact" in model1._expertise - - model2 = OllamaModel(model_name="deepseek-r1:8b") - assert "code" in model2._expertise or "general" in model2._expertise - - model3 = OllamaModel(model_name="llama3.2:3b") - assert len(model3._expertise) > 0 - - @pytest.mark.asyncio - async def test_ollama_health_check_integration(self, ollama_registry): - """Test health checking with real Ollama models.""" - if not is_ollama_running(): - pytest.skip("Ollama is not running locally") - - # Get a model from registry - model = list(ollama_registry.models.values())[0] - - # Test health check - is_healthy = await model.health_check() - assert isinstance(is_healthy, bool) - - # If Ollama is running, model should be healthy - if is_ollama_running(): - assert is_healthy - - # Test with registry health filtering - healthy_models = await ollama_registry._filter_by_health([model]) - if is_healthy: - assert len(healthy_models) > 0 - assert model in healthy_models - - def test_ollama_performance_characteristics(self, ollama_registry): - """Test that Ollama models have appropriate performance characteristics.""" - for model in ollama_registry.models.values(): - # Free models should have maximum budget efficiency - analysis = ollama_registry.detect_model_capabilities(model) - scores = analysis["suitability_scores"] - - # Should excel at budget-constrained tasks - assert scores["budget_constrained"] == 1.0 - - # Fast models should have good speed scores - if "fast" in model._expertise: - # Should be suitable for speed-critical tasks - assert scores["speed_critical"] > 0.5 - - -@pytest.mark.skipif( - not is_ollama_running(), - reason="Requires Ollama to be running locally" -) -class TestOllamaLiveIntegration: - """ - Live integration tests that make actual calls to Ollama. - These tests require Ollama to be running locally. - """ - - @pytest.mark.asyncio - async def test_live_ollama_generation_with_enhanced_selection(self): - """Test actual text generation with enhanced model selection.""" - available_models = get_available_ollama_models() - if not available_models: - pytest.skip("No Ollama models available locally") - - registry = ModelRegistry() - - # Use the first available model - model_name = available_models[0] - model = OllamaModel(model_name=model_name) - - registry.register_model(model) - selector = ModelSelector(registry) - - # Use enhanced selection - criteria = ModelSelectionCriteria( - expertise="low", # Use low to be flexible - cost_limit=0.0, # Free only - budget_period="per-task" - ) - - selected_model = await selector.select_model(criteria) - assert selected_model is not None - - # Make actual API call with short response - try: - response = await selected_model.generate( - "Say hello in one word", - temperature=0.1, - max_tokens=5 - ) - - assert isinstance(response, str) - assert len(response) > 0 - print(f"Ollama response: {response}") - except Exception as e: - # Ollama might not have the model downloaded or running - pytest.skip(f"Ollama generation failed: {e}") - - def test_ollama_model_listing(self): - """Test listing of available Ollama models.""" - if not is_ollama_running(): - pytest.skip("Ollama is not running locally") - - available_models = get_available_ollama_models() - print(f"Available Ollama models: {available_models}") - - # Should have at least some indication of Ollama status - assert isinstance(available_models, list) - - -if __name__ == "__main__": - pytest.main([__file__, "-v", "-m", "not skipif"]) \ No newline at end of file diff --git a/tests/test_real_api_openai_integration.py b/tests/test_real_api_openai_integration.py deleted file mode 100644 index 857a960c..00000000 --- a/tests/test_real_api_openai_integration.py +++ /dev/null @@ -1,425 +0,0 @@ -""" -Real API Integration Tests - OpenAI -Tests the enhanced model requirements specification with actual OpenAI API calls. -""" - -import pytest -import os -from unittest.mock import patch -from orchestrator.models.openai_model import OpenAIModel -from orchestrator.models.model_selector import ModelSelector, ModelSelectionCriteria -from orchestrator.models.model_registry import ModelRegistry -from orchestrator.core.model import ModelCapabilities, ModelCost - -from tests.test_infrastructure import create_test_orchestrator, TestModel, TestProvider - - -class TestRealOpenAIIntegration: - """Test enhanced model requirements with real OpenAI models.""" - - @pytest.fixture - def openai_registry(self): - """Create registry with real OpenAI models.""" - registry = ModelRegistry() - - # Only create models if API key is available (for CI/CD safety) - api_key = os.getenv("OPENAI_API_KEY") - if not api_key: - pytest.skip("OPENAI_API_KEY not available for real API testing") - - # GPT-4o mini - balanced model - gpt4_mini = OpenAIModel( - name="gpt-4o-mini", - api_key=api_key - ) - # Override size estimate for testing - gpt4_mini._size_billions = 8.0 - gpt4_mini._expertise = ["general", "chat", "reasoning"] - registry.register_model(gpt4_mini) - - # GPT-4o - premium model with vision - gpt4o = OpenAIModel( - name="gpt-4o", - api_key=api_key - ) - # Override for testing - gpt4o._size_billions = 200.0 # Estimated - gpt4o._expertise = ["general", "reasoning", "code", "creative", "analysis"] - gpt4o.capabilities.vision_capable = True - registry.register_model(gpt4o) - - # GPT-3.5 Turbo - budget option - gpt35 = OpenAIModel( - name="gpt-3.5-turbo", - api_key=api_key - ) - # Override for testing - gpt35._size_billions = 175.0 - gpt35._expertise = ["general", "chat", "fast"] - registry.register_model(gpt35) - - return registry - - @pytest.fixture - def openai_selector(self, openai_registry): - """Create model selector with OpenAI registry.""" - return ModelSelector(openai_registry) - - def test_openai_model_initialization_with_enhanced_features(self): - """Test that OpenAI models initialize with enhanced Issue 194 features.""" - api_key = os.getenv("OPENAI_API_KEY") - if not api_key: - pytest.skip("OPENAI_API_KEY not available") - - model = OpenAIModel(name="gpt-4o-mini", api_key=api_key) - - # Test enhanced attributes from Issue 194 - assert hasattr(model, '_expertise') - assert hasattr(model, '_size_billions') - assert isinstance(model._expertise, list) - assert isinstance(model._size_billions, (int, float)) - assert model._size_billions > 0 - - # Test cost information - assert model.cost is not None - assert isinstance(model.cost, ModelCost) - assert not model.cost.is_free # OpenAI models are paid - - # Test enhanced cost methods from Issue 194 - task_cost = model.cost.estimate_cost_for_budget_period("per-task") - assert isinstance(task_cost, float) - assert task_cost > 0 - - efficiency = model.cost.get_cost_efficiency_score(0.9) - assert isinstance(efficiency, float) - assert efficiency > 0 - - @pytest.mark.asyncio - async def test_expertise_based_selection_with_openai(self, openai_selector): - """Test expertise-based selection with real OpenAI models.""" - # Low expertise - should prefer faster/cheaper models - low_criteria = ModelSelectionCriteria(expertise="low") - model = await openai_selector.select_model(low_criteria) - - # Should select a model that meets low expertise requirements - assert model is not None - assert "fast" in model._expertise or "general" in model._expertise - - # High expertise - should prefer more capable models - high_criteria = ModelSelectionCriteria(expertise="high") - model = await openai_selector.select_model(high_criteria) - - # Should select GPT-4 variant for high expertise - assert model is not None - assert "gpt-4" in model.name.lower() - - @pytest.mark.asyncio - async def test_cost_constraint_selection_with_openai(self, openai_selector): - """Test cost constraint selection with real OpenAI pricing.""" - # Very tight budget - should prefer cheaper models - budget_criteria = ModelSelectionCriteria( - cost_limit=0.01, # Very low budget - budget_period="per-task" - ) - - model = await openai_selector.select_model(budget_criteria) - assert model is not None - - # Verify the selected model is within budget - estimated_cost = model.cost.estimate_cost_for_budget_period("per-task") - assert estimated_cost <= 0.01 - - # Higher budget - should allow premium models - premium_criteria = ModelSelectionCriteria( - cost_limit=5.0, # Higher budget - budget_period="per-task" - ) - - model = await openai_selector.select_model(premium_criteria) - assert model is not None - - # Should be within budget - estimated_cost = model.cost.estimate_cost_for_budget_period("per-task") - assert estimated_cost <= 5.0 - - @pytest.mark.asyncio - async def test_size_constraint_selection_with_openai(self, openai_selector): - """Test size constraint selection with OpenAI models.""" - # Small model preference - small_criteria = ModelSelectionCriteria(max_model_size=50.0) - model = await openai_selector.select_model(small_criteria) - - assert model is not None - assert model._size_billions <= 50.0 - - # Large model preference - large_criteria = ModelSelectionCriteria(min_model_size=100.0) - model = await openai_selector.select_model(large_criteria) - - assert model is not None - assert model._size_billions >= 100.0 - - @pytest.mark.asyncio - async def test_modality_selection_with_openai(self, openai_selector): - """Test modality-based selection with OpenAI models.""" - # Vision modality requirement - vision_criteria = ModelSelectionCriteria(modalities=["vision"]) - model = await openai_selector.select_model(vision_criteria) - - # Should select a vision-capable model (GPT-4o) - assert model is not None - assert model.capabilities.vision_capable - assert "gpt-4o" in model.name.lower() - - # Code modality requirement - code_criteria = ModelSelectionCriteria(modalities=["code"]) - model = await openai_selector.select_model(code_criteria) - - # Should select a code-capable model - assert model is not None - assert model.capabilities.code_specialized or "code" in model._expertise - - @pytest.mark.asyncio - async def test_complex_criteria_with_openai(self, openai_selector): - """Test complex multi-criteria selection with OpenAI models.""" - complex_criteria = ModelSelectionCriteria( - expertise="medium", - min_model_size=5.0, - max_model_size=200.0, - cost_limit=2.0, - budget_period="per-task", - selection_strategy="balanced" - ) - - model = await openai_selector.select_model(complex_criteria) - assert model is not None - - # Verify it meets size constraints - assert 5.0 <= model._size_billions <= 200.0 - - # Verify cost constraint - estimated_cost = model.cost.estimate_cost_for_budget_period("per-task") - assert estimated_cost <= 2.0 - - def test_openai_capability_detection(self, openai_registry): - """Test capability detection with real OpenAI models.""" - # Test capability analysis for each model - for model_key, model in openai_registry.models.items(): - analysis = openai_registry.detect_model_capabilities(model) - - # Should have complete analysis - assert "basic_capabilities" in analysis - assert "advanced_capabilities" in analysis - assert "performance_metrics" in analysis - assert "expertise_analysis" in analysis - assert "cost_analysis" in analysis - assert "suitability_scores" in analysis - - # Cost analysis should reflect OpenAI pricing - cost_analysis = analysis["cost_analysis"] - assert cost_analysis["type"] == "paid" - assert cost_analysis["cost_per_1k_avg"] > 0 - - # Should have reasonable suitability scores - scores = analysis["suitability_scores"] - for capability, score in scores.items(): - assert 0.0 <= score <= 1.0 - - def test_openai_task_recommendations(self, openai_registry): - """Test task-based recommendations with OpenAI models.""" - # Code-related task - code_recs = openai_registry.recommend_models_for_task( - "Help me debug Python code", - max_recommendations=2 - ) - - assert len(code_recs) > 0 - - for rec in code_recs: - assert "model" in rec - assert "reasoning" in rec - assert rec["suitability_score"] > 0 - - # Should recommend capable models for coding - model = rec["model"] - assert model.capabilities.code_specialized or "code" in model._expertise - - # Vision-related task - vision_recs = openai_registry.recommend_models_for_task( - "Analyze this image for me", - max_recommendations=2 - ) - - # Should recommend vision-capable models - vision_capable_found = False - for rec in vision_recs: - if rec["model"].capabilities.vision_capable: - vision_capable_found = True - break - - assert vision_capable_found - - def test_openai_cost_analysis_integration(self, openai_registry): - """Test cost analysis with real OpenAI pricing.""" - for model in openai_registry.models.values(): - # Test budget period estimates - task_cost = model.cost.estimate_cost_for_budget_period("per-task") - pipeline_cost = model.cost.estimate_cost_for_budget_period("per-pipeline") - hour_cost = model.cost.estimate_cost_for_budget_period("per-hour") - - # Costs should increase with usage - assert task_cost <= pipeline_cost <= hour_cost - - # Test cost breakdown - breakdown = model.cost.get_cost_breakdown(1000, 500) # 1000 input, 500 output - assert breakdown["total_cost"] > 0 - assert breakdown["input_cost"] > 0 - assert breakdown["output_cost"] > 0 - assert not breakdown["is_free"] - - # Test cost comparison - other_model = list(openai_registry.models.values())[0] - if other_model != model: - comparison = model.cost.compare_cost_with(other_model.cost) - assert "cost_ratio" in comparison - assert "savings" in comparison - assert isinstance(comparison["cost_ratio"], (int, float)) - - @pytest.mark.asyncio - async def test_openai_fallback_strategies(self, openai_selector): - """Test fallback strategies with OpenAI models.""" - # Impossible requirements with cheapest fallback - impossible_criteria = ModelSelectionCriteria( - min_model_size=10000.0, # Impossibly large - fallback_strategy="cheapest" - ) - - model = await openai_selector.select_model(impossible_criteria) - assert model is not None - - # Should fallback to cheapest available model - # In OpenAI case, probably GPT-3.5 turbo - assert model.name in ["gpt-3.5-turbo", "gpt-4o-mini"] - - # Best available fallback - best_criteria = ModelSelectionCriteria( - min_model_size=10000.0, # Impossibly large - fallback_strategy="best_available" - ) - - model = await openai_selector.select_model(best_criteria) - assert model is not None - - # Should fallback to highest quality model - assert model.capabilities.accuracy_score > 0.8 - - @pytest.mark.asyncio - async def test_openai_yaml_integration(self, openai_selector): - """Test YAML requirements parsing with OpenAI models.""" - # Simulate YAML requirements - yaml_requirements = { - "expertise": "high", - "modalities": ["text", "code"], - "min_size": "10B", - "max_size": "500B", - "cost_limit": 3.0, - "budget_period": "per-task", - "fallback_strategy": "best_available" - } - - # Parse and select - criteria = openai_selector.parse_requirements_from_yaml(yaml_requirements) - model = await openai_selector.select_model(criteria) - - assert model is not None - assert model._size_billions >= 10.0 - assert model._size_billions <= 500.0 - - # Should be high expertise - registry = openai_selector.registry - assert registry._meets_expertise_level(model, "high") - - @pytest.mark.asyncio - async def test_openai_real_health_check(self, openai_registry): - """Test health checking with real OpenAI models.""" - # Get a model from registry - model = list(openai_registry.models.values())[0] - - # Mock the health check to avoid real API calls during testing - with patch.object(model, 'health_check', return_value=True) as mock_health: - is_healthy = await model.health_check() - assert is_healthy - mock_health.assert_called_once() - - # Test with registry health filtering - healthy_models = await openai_registry._filter_by_health([model]) - # Should return the model if health check passes - assert len(healthy_models) >= 0 # May be 0 if health check is mocked to fail - - def test_openai_model_size_parsing_integration(self): - """Test model size parsing with OpenAI model names.""" - from orchestrator.utils.model_utils import parse_model_size - - # Test OpenAI model names (which don't have explicit sizes) - test_cases = [ - ("gpt-4o", None, 1.0), # Should default to 1.0 when no size info - ("gpt-3.5-turbo", None, 1.0), - ("gpt-4-turbo", None, 1.0), - ] - - for model_name, size_str, expected in test_cases: - result = parse_model_size(model_name, size_str) - assert result == expected - - -@pytest.mark.integration -class TestOpenAILiveIntegration: - """ - Live integration tests that make actual API calls. - These tests are marked separately and should be run with caution. - """ - - @pytest.mark.skipif( - not os.getenv("OPENAI_API_KEY") or not os.getenv("RUN_LIVE_TESTS"), - reason="Requires OPENAI_API_KEY and RUN_LIVE_TESTS=1 to run live tests" - ) - @pytest.mark.asyncio - async def test_live_openai_generation_with_enhanced_selection(self): - """Test actual text generation with enhanced model selection.""" - registry = ModelRegistry() - - # Create real OpenAI model - model = OpenAIModel( - name="gpt-4o-mini", - api_key=os.getenv("OPENAI_API_KEY") - ) - model._expertise = ["general", "reasoning"] - model._size_billions = 8.0 - - registry.register_model(model) - selector = ModelSelector(registry) - - # Use enhanced selection - criteria = ModelSelectionCriteria( - expertise="medium", - cost_limit=0.1, - budget_period="per-task" - ) - - selected_model = await selector.select_model(criteria) - assert selected_model is not None - - # Make actual API call - response = await selected_model.generate( - "What is 2+2?", - temperature=0.1, - max_tokens=10 - ) - - assert isinstance(response, str) - assert len(response) > 0 - assert "4" in response # Should contain the answer - - -if __name__ == "__main__": - pytest.main([__file__, "-v", "-m", "not integration"]) \ No newline at end of file diff --git a/tests/test_research_assistant_example.py b/tests/test_research_assistant_example.py deleted file mode 100644 index 07f8888a..00000000 --- a/tests/test_research_assistant_example.py +++ /dev/null @@ -1,568 +0,0 @@ -""" -Comprehensive tests for the Research Assistant example. - -This test suite verifies that the research assistant example works correctly -with real API keys and produces high-quality outputs. -""" - -import asyncio -import os -import pytest -import yaml -from typing import Dict, Any - -from orchestrator import Orchestrator -from orchestrator.integrations.openai_model import OpenAIModel -from orchestrator.integrations.anthropic_model import AnthropicModel -from orchestrator.state.state_manager import StateManager -from orchestrator.tools.web_tools import WebSearchTool, HeadlessBrowserTool -from orchestrator.tools.data_tools import DataProcessingTool -from orchestrator.core.cache import MemoryCache - -from tests.test_infrastructure import create_test_orchestrator, TestModel, TestProvider - - -class ResearchAssistant: - """ - Research Assistant implementation for testing. - - This is a simplified version of the research assistant that can be tested - with real API keys and mock data. - """ - - def __init__(self, config: Dict[str, Any]): - self.config = config - self.orchestrator = None - self.state_manager = None - self.cache = None - # Load orchestrator configuration for web tools - self.orchestrator_config = self._load_orchestrator_config() - self._setup_orchestrator() - - def _setup_orchestrator(self): - """Initialize the orchestrator with models and tools.""" - # Initialize state manager for checkpointing - self.state_manager = StateManager( - backend_type="memory", - compression_enabled=False, # Use memory backend for testing - ) - - # Initialize caching for performance - self.cache = MemoryCache(max_size=1000, default_ttl=3600) # 1 hour - - # Import orchestrator module and initialize models first - import orchestrator as orc - - orc.init_models() - - # Initialize orchestrator - self.orchestrator = create_test_orchestrator() - - # Register additional models if needed - self._register_models() - - # Tools are handled by the control system - self.tools = self._get_tools() - - def _register_models(self): - """Register AI models with the orchestrator based on config/models.yaml.""" - # Register OpenAI models if API key is available - if self.config.get("openai_api_key"): - try: - # Use gpt-4.1 from config/models.yaml - gpt4 = OpenAIModel( - model_name="gpt-4.1", - api_key=self.config["openai_api_key"], - max_retries=3) - self.orchestrator.model_registry.register_model(gpt4) - except Exception as e: - print(f"Failed to register OpenAI model: {e}") - - # Register Anthropic models if API key is available - if self.config.get("anthropic_api_key"): - try: - # Use claude-4-sonnet from config/models.yaml - claude = AnthropicModel( - model_name="claude-sonnet-4-20250514", - api_key=self.config["anthropic_api_key"], - max_retries=3) - self.orchestrator.model_registry.register_model(claude) - except Exception as e: - print(f"Failed to register Anthropic model: {e}") - - def _load_orchestrator_config(self) -> Dict[str, Any]: - """Load orchestrator configuration for web tools.""" - # The default config is a packaged resource, not a path on one - # developer's machine. - from orchestrator.install_configs import packaged_config_path - - config_path = packaged_config_path("orchestrator.yaml") - with open(config_path, "r") as f: - return yaml.safe_load(f) - - def _get_tools(self): - """Get tools for web search and content extraction.""" - return { - "comprehensive_web_search": WebSearchTool(self.orchestrator_config), - "extract_web_content": HeadlessBrowserTool(self.orchestrator_config), - "analyze_source_credibility": DataProcessingTool(), - } - - async def conduct_research(self, query: str, context: str = "") -> Dict[str, Any]: - """ - Conduct comprehensive research on a given query. - - Args: - query: The research question or topic - context: Additional context to guide the research - - Returns: - Dictionary containing research results, report, and metadata - """ - # Conduct real research using actual web tools - try: - # Perform real web search - web_search_tool = self.tools["comprehensive_web_search"] - search_results = await web_search_tool.execute(query=query, max_results=5) - - # Extract content from first search result if available - browser_tool = self.tools["extract_web_content"] - extraction_url = "https://example.com" # Default fallback - - # Use actual search result URL if available - if search_results.get("results") and len(search_results["results"]) > 0: - extraction_url = search_results["results"][0].get("url", extraction_url) - - extraction_results = await browser_tool.execute( - action="scrape", url=extraction_url - ) - - return { - "query": query, - "context": context, - "search_results": search_results, - "extraction_results": extraction_results, - "quality_score": self._calculate_quality_score( - search_results, extraction_results - ), - "execution_time": 2.5, # Estimated execution time for real operations - "success": True, - } - - except Exception as e: - return { - "query": query, - "context": context, - "error": str(e), - "success": False, - } - - def _calculate_quality_score( - self, search_results: Dict, extraction_results: Dict - ) -> float: - """Calculate overall quality score for research results.""" - score = 0.0 - - # Search quality - more generous scoring - if search_results.get("results"): - search_score = min( - len(search_results["results"]) / 3.0, 1.0 - ) # Up to 3 results for max score - score += search_score * 0.4 - - # Extraction quality - more generous scoring - if extraction_results.get("content"): - content_length = len(extraction_results.get("content", "")) - extraction_score = min( - content_length / 500.0, 1.0 - ) # Up to 500 chars for max score - score += extraction_score * 0.4 - - # Base quality score for successful execution - score += 0.2 - - return min(score, 1.0) - - -class TestResearchAssistant: - """Comprehensive test suite for the Research Assistant example.""" - - @pytest.fixture - def config(self): - """Test configuration with API keys from environment.""" - return { - "openai_api_key": os.getenv("OPENAI_API_KEY"), - "anthropic_api_key": os.getenv("ANTHROPIC_API_KEY"), - "test_mode": True, - } - - @pytest.fixture - def assistant(self, config): - """Create research assistant instance.""" - return ResearchAssistant(config) - - @pytest.mark.asyncio - async def test_research_assistant_initialization(self, assistant): - """Test that the research assistant initializes correctly.""" - assert assistant.orchestrator is not None - assert assistant.state_manager is not None - assert assistant.cache is not None - - # Check that tools are available - assert "comprehensive_web_search" in assistant.tools - assert "extract_web_content" in assistant.tools - assert "analyze_source_credibility" in assistant.tools - - @pytest.mark.asyncio - async def test_basic_research_flow(self, assistant): - """Test basic research workflow.""" - query = "quantum computing applications" - context = "Focus on practical implementations" - - result = await assistant.conduct_research(query, context) - - # Basic result validation - assert result["query"] == query - assert result["context"] == context - assert result["success"] is True - assert "search_results" in result - assert "extraction_results" in result - assert "quality_score" in result - - # Quality assertions - assert result["quality_score"] >= 0.0 - assert result["quality_score"] <= 1.0 - - @pytest.mark.asyncio - async def test_web_search_functionality(self, assistant): - """Test web search tool functionality.""" - # Get the web search tool - web_search_tool = assistant.tools["comprehensive_web_search"] - - # Test search - search_result = await web_search_tool.execute( - query="machine learning", max_results=3 - ) - - # Validate search results - assert "query" in search_result - assert "results" in search_result - assert "total_results" in search_result - assert len(search_result["results"]) <= 3 - - # Check result structure - for result in search_result["results"]: - assert "title" in result - assert "url" in result - assert "snippet" in result - assert "relevance" in result - - @pytest.mark.asyncio - async def test_content_extraction_functionality(self, assistant): - """Test content extraction tool functionality.""" - # Get the browser tool - browser_tool = assistant.tools["extract_web_content"] - - # Test content extraction - extraction_result = await browser_tool.execute( - action="scrape", url="https://example.com" - ) - - # Validate extraction results - assert "url" in extraction_result - assert "title" in extraction_result - assert "text" in extraction_result or "content" in extraction_result - assert "word_count" in extraction_result - - # Check content quality - content = extraction_result.get("text", extraction_result.get("content", "")) - assert len(content) > 0 - assert extraction_result["word_count"] > 0 - - @pytest.mark.asyncio - async def test_multiple_search_terms(self, assistant): - """Test research with multiple search terms.""" - queries = [ - "artificial intelligence ethics", - "neural network optimization", - "deep learning applications", - ] - - results = [] - for query in queries: - result = await assistant.conduct_research(query) - results.append(result) - - # Validate all results - for i, result in enumerate(results): - assert result["query"] == queries[i] - assert result["success"] is True - assert result["quality_score"] > 0 - - @pytest.mark.asyncio - async def test_error_handling(self, assistant): - """Test error handling in research pipeline.""" - # Test with invalid query - result = await assistant.conduct_research("") - - # Should handle empty query gracefully - assert "query" in result - # May succeed with empty results or fail gracefully - assert "success" in result - - @pytest.mark.asyncio - async def test_caching_functionality(self, assistant): - """Test that caching works correctly.""" - query = "test caching query" - - # First request - result1 = await assistant.conduct_research(query) - - # Second request (should use cache) - result2 = await assistant.conduct_research(query) - - # Results should be consistent - assert result1["query"] == result2["query"] - assert result1["success"] == result2["success"] - - # Check cache statistics - cache_stats = assistant.cache.get_statistics() - assert cache_stats.get("entries", 0) >= 0 - - @pytest.mark.asyncio - async def test_state_management(self, assistant): - """Test state management and checkpointing.""" - # Create a test pipeline state - test_state = { - "pipeline_id": "test_pipeline", - "status": "running", - "current_task": "web_search", - "completed_tasks": [], - } - - # Save checkpoint - checkpoint_id = await assistant.state_manager.save_checkpoint( - execution_id="test_execution", - state=test_state, - metadata={"task_id": "web_search"}) - - # Load checkpoint - loaded_state = await assistant.state_manager.restore_checkpoint( - pipeline_id="test_pipeline", checkpoint_id=checkpoint_id - ) - - # Verify state was preserved - assert loaded_state is not None - assert isinstance(loaded_state, dict) - # Check that checkpoint save/restore functionality works - assert checkpoint_id is not None - - @pytest.mark.asyncio - async def test_quality_score_calculation(self, assistant): - """Test quality score calculation.""" - # Mock search results - search_results = { - "results": [ - {"title": "Test 1", "url": "http://example1.com", "relevance": 0.9}, - {"title": "Test 2", "url": "http://example2.com", "relevance": 0.8}, - {"title": "Test 3", "url": "http://example3.com", "relevance": 0.7}, - ] - } - - # Mock extraction results - extraction_results = { - "content": "This is test content " * 50, # 1000+ characters - "title": "Test Page", - "word_count": 100, - } - - # Calculate quality score - score = assistant._calculate_quality_score(search_results, extraction_results) - - # Validate score - assert 0.0 <= score <= 1.0 - assert score > 0.5 # Should be reasonably high with good mock data - - @pytest.mark.asyncio - async def test_real_api_integration(self, assistant, config): - """Test integration with real APIs (if keys are available).""" - # Require API keys for real testing - if not config.get("openai_api_key") and not config.get("anthropic_api_key"): - raise AssertionError( - "No API keys available for real API testing. " - "Please configure API keys in ~/.orchestrator/.env" - ) - - # Test with a simple query - query = "Python programming best practices" - result = await assistant.conduct_research(query) - - # Validate real API results - assert result["success"] is True - assert result["quality_score"] > 0 - assert "search_results" in result - - # Check that we got actual results - search_results = result["search_results"] - if "results" in search_results: - assert len(search_results["results"]) > 0 - - @pytest.mark.asyncio - async def test_performance_benchmarks(self, assistant): - """Test performance benchmarks for the research assistant.""" - import time - - query = "performance test query" - - # Measure execution time - start_time = time.time() - result = await assistant.conduct_research(query) - end_time = time.time() - - execution_time = end_time - start_time - - # Performance assertions - assert execution_time < 30.0 # Should complete within 30 seconds - assert result["success"] is True - - # Log performance metrics - print(f"Research execution time: {execution_time:.2f} seconds") - print(f"Quality score: {result['quality_score']:.2f}") - - @pytest.mark.asyncio - async def test_concurrent_research_requests(self, assistant): - """Test handling of concurrent research requests.""" - queries = ["concurrent test 1", "concurrent test 2", "concurrent test 3"] - - # Execute concurrent requests - tasks = [assistant.conduct_research(q) for q in queries] - results = await asyncio.gather(*tasks) - - # Validate all results - assert len(results) == len(queries) - for i, result in enumerate(results): - assert result["query"] == queries[i] - assert result["success"] is True - - def test_configuration_validation(self, config): - """Test configuration validation.""" - # Test with valid config - assistant = ResearchAssistant(config) - assert assistant.config == config - - # Test with minimal config - minimal_config = {"test_mode": True} - assistant = ResearchAssistant(minimal_config) - assert assistant.config["test_mode"] is True - - -class TestResearchAssistantIntegration: - """Integration tests for the complete research assistant system.""" - - @pytest.mark.asyncio - async def test_end_to_end_research_workflow(self): - """Test the complete end-to-end research workflow.""" - config = { - "openai_api_key": os.getenv("OPENAI_API_KEY"), - "anthropic_api_key": os.getenv("ANTHROPIC_API_KEY"), - "test_mode": True, - } - - assistant = ResearchAssistant(config) - - # Test complete workflow - query = "sustainable energy technologies" - context = "Focus on recent developments in solar and wind power" - - result = await assistant.conduct_research(query, context) - - # Comprehensive validation - assert result["success"] is True - assert result["query"] == query - assert result["context"] == context - assert result["quality_score"] > 0.0 - - # Validate search results structure - search_results = result["search_results"] - assert isinstance(search_results, dict) - - # Validate extraction results structure - extraction_results = result["extraction_results"] - assert isinstance(extraction_results, dict) - - print("End-to-end test completed successfully") - print(f"Query: {query}") - print(f"Quality Score: {result['quality_score']:.2f}") - print(f"Execution Time: {result['execution_time']:.2f}s") - - @pytest.mark.asyncio - async def test_research_quality_validation(self): - """Test that research results meet quality standards.""" - config = { - "openai_api_key": os.getenv("OPENAI_API_KEY"), - "anthropic_api_key": os.getenv("ANTHROPIC_API_KEY"), - "test_mode": True, - } - - assistant = ResearchAssistant(config) - - # Test with various query types - test_queries = [ - "machine learning algorithms", - "climate change impacts", - "blockchain technology applications", - "quantum computing principles", - ] - - quality_scores = [] - - for query in test_queries: - result = await assistant.conduct_research(query) - - # Validate quality standards - assert result["success"] is True - assert result["quality_score"] >= 0.3 # Minimum quality threshold - - quality_scores.append(result["quality_score"]) - - # Overall quality validation - avg_quality = sum(quality_scores) / len(quality_scores) - assert avg_quality >= 0.5 # Average quality should be reasonable - - print("Quality validation completed") - print(f"Average quality score: {avg_quality:.2f}") - print(f"Individual scores: {quality_scores}") - - -if __name__ == "__main__": - # Run a quick test if executed directly - async def quick_test(): - config = { - "openai_api_key": os.getenv("OPENAI_API_KEY"), - "anthropic_api_key": os.getenv("ANTHROPIC_API_KEY"), - "test_mode": True, - } - - if not config["openai_api_key"] and not config["anthropic_api_key"]: - print( - "No API keys found. Set OPENAI_API_KEY or ANTHROPIC_API_KEY environment variables." - ) - return - - assistant = ResearchAssistant(config) - - print("Testing Research Assistant...") - result = await assistant.conduct_research( - "artificial intelligence trends 2024", - "Focus on recent developments and applications") - - print(f"Test completed: {result['success']}") - print(f"Quality score: {result['quality_score']:.2f}") - - if result["success"]: - print("✅ Research Assistant example works correctly!") - else: - print("❌ Research Assistant example failed:") - print(f"Error: {result.get('error', 'Unknown error')}") - - asyncio.run(quick_test()) diff --git a/tests/test_update_models_no_hardcoded_ids.py b/tests/test_update_models_no_hardcoded_ids.py deleted file mode 100644 index 7d1a3d44..00000000 --- a/tests/test_update_models_no_hardcoded_ids.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Model ids must come from provider APIs, never from a list in the source. - -This repository has hardcoded Anthropic model ids three times, and all three -rotted the same way: - -1. ids frozen at 2024 dates, which the API had since retired; -2. invented ``-latest`` aliases, none of which existed (every call 404'd); -3. ``update_models.fetch_anthropic_models`` returning claude-2, claude-2.1 and - claude-instant-1.2, on the stated belief that "Anthropic doesn't have a - models.list() endpoint" -- it does. - -A list of model ids in source is a dated snapshot presented as fact. These -tests exist to make the fourth attempt fail loudly. -""" - -import inspect -import re - -import pytest - -from orchestrator.tools import update_models as update_models_module - -pytestmark = pytest.mark.unit - -#: Matches a concrete, dated or versioned Claude model id in source text. -_CLAUDE_ID = re.compile(r"claude[-\w.]*?(\d{8}|-\d+\.\d+|-latest)", re.IGNORECASE) - - -def test_anthropic_fetcher_contains_no_hardcoded_model_ids(): - source = inspect.getsource(update_models_module.ModelUpdater.fetch_anthropic_models) - # The docstring names the retired ids on purpose, as the explanation for - # why this rule exists. Only the executable body is checked. - body = source.split('"""')[-1] - found = _CLAUDE_ID.findall(body) - assert not found, ( - f"hardcoded Claude model ids are back in fetch_anthropic_models: " - f"{found}. Read ids from client.models.list() instead -- a static " - f"list here has rotted three times." - ) - - -def test_anthropic_fetcher_asks_the_api(): - source = inspect.getsource(update_models_module.ModelUpdater.fetch_anthropic_models) - assert "models.list()" in source, ( - "fetch_anthropic_models must query the live listing endpoint" - ) - - -@pytest.mark.asyncio -async def test_anthropic_fetcher_degrades_to_empty_without_a_key(monkeypatch): - """Skipping a provider is honest; inventing its catalogue is not.""" - monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) - updater = update_models_module.ModelUpdater() - assert await updater.fetch_anthropic_models() == [] - - -@pytest.mark.asyncio -async def test_dartmouth_fetcher_degrades_to_empty_without_a_credential(monkeypatch, tmp_path): - """Same contract for Dartmouth: no credential means no models, not guesses.""" - monkeypatch.delenv("DARTMOUTH_CHAT_API_KEY", raising=False) - # Point the on-disk credential lookups somewhere empty so a developer's - # real credential cannot make this test pass or fail by accident. - monkeypatch.setattr( - "orchestrator.models.dartmouth_credentials._ORCHESTRATOR_ENV_FILE", - tmp_path / "absent.env", - ) - monkeypatch.setattr( - "orchestrator.models.dartmouth_credentials._LLMXIVE_CREDENTIALS_FILE", - tmp_path / "absent.toml", - ) - updater = update_models_module.ModelUpdater() - assert await updater.fetch_dartmouth_models() == [] - - -def test_dartmouth_fetcher_registers_only_free_models(): - """Writing paid ids into models.yaml would make them selectable by default.""" - source = inspect.getsource(update_models_module.ModelUpdater.fetch_dartmouth_models) - assert "list_free_models()" in source - assert "list_paid_models" not in source