From 6339507a745c36a641a704421c62572562403dd7 Mon Sep 17 00:00:00 2001 From: JNK234 Date: Thu, 13 Aug 2026 16:50:09 -0500 Subject: [PATCH 1/3] fix: align stale provider defaults with bundled model registry Three registered defaults named models absent from models.yaml, so selecting those providers with no explicit model applied a default that failed the extension's own registry validation and printed a stderr warning -- the extension warning about its own default. - anthropic: claude-3-5-haiku-latest -> claude-haiku-4-5-20251001 (the old default resolved to claude-3-5-haiku-20241022, retired 19 Feb 2026; the replacement is the current registry entry that still supports extended thinking) - gemini: gemini-1.5-flash -> gemini-2.5-flash - ollama: llama3.2 -> llama3.2:3b (bare llama3.2 is not a pullable tag; the registry carries the :3b and :1b tags) Also corrects the Ollama help text, which told users to pull the same non-pullable bare tag. Adds ProviderDefaultsSpec as a deterministic drift guard asserting every registered descriptor's defaultModel resolves in the bundled registry, so this cannot silently regress. Verified the guard fails when a stale default is reintroduced. Fixes #62 --- .../providers/ProviderRegistrations.scala | 8 ++-- src/test/ProviderDefaultsSpec.scala | 45 +++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 src/test/ProviderDefaultsSpec.scala diff --git a/src/main/providers/ProviderRegistrations.scala b/src/main/providers/ProviderRegistrations.scala index 0294c96..b6d8b9b 100644 --- a/src/main/providers/ProviderRegistrations.scala +++ b/src/main/providers/ProviderRegistrations.scala @@ -48,7 +48,7 @@ object ProviderRegistrations { apiKeyConfigKey = "anthropic_api_key", baseUrlConfigKey = "anthropic_base_url", defaultBaseUrl = "https://api.anthropic.com/v1", - defaultModel = "claude-3-5-haiku-latest", + defaultModel = "claude-haiku-4-5-20251001", defaultMaxTokens = "4000", requiresApiKey = true, apiKeyPrefix = None, @@ -76,7 +76,7 @@ object ProviderRegistrations { apiKeyConfigKey = "gemini_api_key", baseUrlConfigKey = "gemini_base_url", defaultBaseUrl = "https://generativelanguage.googleapis.com/v1beta", - defaultModel = "gemini-1.5-flash", + defaultModel = "gemini-2.5-flash", defaultMaxTokens = "2048", requiresApiKey = true, apiKeyPrefix = None, @@ -104,7 +104,7 @@ object ProviderRegistrations { apiKeyConfigKey = "ollama_api_key", baseUrlConfigKey = "ollama_base_url", defaultBaseUrl = "http://localhost:11434", - defaultModel = "llama3.2", + defaultModel = "llama3.2:3b", defaultMaxTokens = "2048", requiresApiKey = false, apiKeyPrefix = None, @@ -122,7 +122,7 @@ object ProviderRegistrations { | - Or start Ollama app (it runs in background) | |3. Pull a model: - | - Run: ollama pull llama3.2 + | - Run: ollama pull llama3.2:3b | - Or try: ollama pull deepseek-r1:1.5b (smaller) | |4. Verify installation: diff --git a/src/test/ProviderDefaultsSpec.scala b/src/test/ProviderDefaultsSpec.scala new file mode 100644 index 0000000..db16dc4 --- /dev/null +++ b/src/test/ProviderDefaultsSpec.scala @@ -0,0 +1,45 @@ +// ABOUTME: Drift guard asserting every registered provider's defaultModel exists in the bundled registry +// ABOUTME: Prevents ProviderRegistrations.scala and models.yaml from silently falling out of sync +package org.nlogo.extensions.llm.providers + +import org.scalatest.funsuite.AnyFunSuite + +/** + * Guards the invariant that each provider's advertised default model is one the + * extension will actually accept. + * + * Without this, a stale default (e.g. a retired model, or an Ollama tag that is + * not pullable) only surfaces at runtime as a stderr warning from the + * extension's own validation -- the extension complaining about its own default. + */ +class ProviderDefaultsSpec extends AnyFunSuite { + + // Registrations are normally installed by LLMExtension.load(); do it here so + // the suite is self-contained and order-independent. + ProviderRegistry.reset() + ProviderRegistrations.registerAll() + + private val descriptors = ProviderRegistry.allNames.toSeq.sorted.flatMap(ProviderRegistry.get) + + test("providers are actually registered") { + assert(descriptors.nonEmpty, "ProviderRegistrations.registerAll() registered no providers") + } + + test("every provider default model is present in the bundled registry") { + val drifted = descriptors + .filterNot(d => ModelRegistry.isValidModel(d.name, d.defaultModel)) + .map { d => + s"provider '${d.name}' default '${d.defaultModel}' is not in the bundled model registry. " + + s"Known models: ${ModelRegistry.getModelListForDisplay(d.name)}" + } + + // Build the message as a String so a failure prints the drifted defaults + // rather than dumping whole ProviderDescriptor instances (helpText included). + assert(drifted.isEmpty, s"\n${drifted.mkString("\n")}") + } + + test("every provider has a non-empty model list in the bundled registry") { + val empty = descriptors.filter(d => ModelRegistry.getSupportedModels(d.name).isEmpty).map(_.name) + assert(empty.isEmpty, s"providers with no models in models.yaml: ${empty.mkString(", ")}") + } +} From f541a3751808043b25dfc1d9f4de4a1a92b0dd59 Mon Sep 17 00:00:00 2001 From: JNK234 Date: Thu, 13 Aug 2026 16:50:24 -0500 Subject: [PATCH 2/3] fix: branch Claude request shaping on model generation The Claude path sent one request shape to every model, which current Anthropic models reject with HTTP 400: - temperature was set unconditionally to 1.0 when thinking was on, and passed through otherwise. Non-default sampling parameters return 400 on Opus 4.7/4.8, Opus 5, Sonnet 5 and Fable/Mythos, on every request -- not only thinking ones -- so the non-thinking path is gated too. - thinking used {type: "enabled", budget_tokens: N} for all models. That shape is rejected on 4.7 and later, which take {type: "adaptive"} with depth steered by output_config.effort. Adds ClaudeModelCapabilities holding the per-generation table: thinkingMode (Extended for 4.5 and earlier, Adaptive otherwise, defaulting forward for unknown identifiers), supportsSamplingParams, and the reasoning_effort -> output_config.effort mapping. The extension's "none" has no Anthropic equivalent and is dropped so the API default applies. Placing this beside ClaudeProvider rather than in ReasoningModelDetector keeps the shared cross-provider matcher from growing another hardcoded family check (#31). A models.yaml capability flag was considered and rejected: the schema is a flat provider -> [model] list, so per-model attributes would require changing the parser, ProviderModels, merge semantics and every user's models-override.yaml. Also pins anthropic-version to 2023-06-01. The header was bumped to "2025-04-15" whenever thinking was on; Anthropic publishes no such version. Adds ClaudeRequestSpec covering extended-thinking, adaptive-thinking and non-thinking requests, asserting no temperature key is sent to 4.7+ models. Tests inspect the built request body directly, so they need no network or API key. Verified they fail when the temperature gate is removed. Fixes #63 --- .../providers/ClaudeModelCapabilities.scala | 103 +++++++++++ src/main/providers/ClaudeProvider.scala | 72 +++++--- src/test/ClaudeRequestSpec.scala | 164 ++++++++++++++++++ 3 files changed, 315 insertions(+), 24 deletions(-) create mode 100644 src/main/providers/ClaudeModelCapabilities.scala create mode 100644 src/test/ClaudeRequestSpec.scala diff --git a/src/main/providers/ClaudeModelCapabilities.scala b/src/main/providers/ClaudeModelCapabilities.scala new file mode 100644 index 0000000..258908d --- /dev/null +++ b/src/main/providers/ClaudeModelCapabilities.scala @@ -0,0 +1,103 @@ +// ABOUTME: Per-generation Anthropic API capabilities (thinking mode, sampling-parameter support) +// ABOUTME: Single place encoding which Claude models take extended thinking vs adaptive thinking + effort +package org.nlogo.extensions.llm.providers + +/** + * Which thinking request shape a Claude model accepts. + * + * Anthropic changed the thinking API across generations, and the two shapes are + * mutually exclusive -- sending the wrong one is an HTTP 400: + * + * - Extended: `thinking: {type: "enabled", budget_tokens: N}`. + * Claude 4.5 and earlier. Rejected by 4.7 and later. + * - Adaptive: `thinking: {type: "adaptive"}` with depth steered by + * `output_config.effort`. Claude 4.6 and later; the only mode on 4.7+. + * + * Claude 4.6 accepts both; adaptive is preferred there because extended + * thinking is deprecated on that generation. + */ +enum ClaudeThinkingMode: + case Extended + case Adaptive + +/** + * Capability lookup for Anthropic model identifiers. + * + * Kept separate from ReasoningModelDetector on purpose: that object answers the + * cross-provider question "should thinking be on at all", while these are + * Anthropic request-shape details that only ClaudeProvider needs. Folding them + * in would grow the shared cross-provider string-matching surface. + * + * Matching is on model-name substrings because Anthropic model IDs are + * versioned strings and the extension accepts user-supplied and override-config + * model names that are not in the bundled registry. + */ +object ClaudeModelCapabilities { + + /** + * Generations that predate adaptive thinking, and so must use the legacy + * `{type: "enabled", budget_tokens: N}` shape. + * + * Claude 3.x is included for completeness: those models are retired, but a + * user pinning one via override config should still get the shape their + * model expects rather than a guaranteed 400 from the adaptive shape. + */ + private val ExtendedThinkingMarkers = Seq( + "claude-3-5", "claude-3-7", "claude-3-opus", "claude-3-haiku", "claude-3-sonnet", + "claude-opus-4-0", "claude-opus-4-1", "claude-opus-4-5", + "claude-sonnet-4-0", "claude-sonnet-4-5", + "claude-haiku-4-5", + "claude-opus-4-20", "claude-sonnet-4-20" + ) + + /** + * Generations that reject non-default `temperature`/`top_p`/`top_k` with a 400 + * on EVERY request, thinking or not. + * + * Per Anthropic's docs this covers Claude 4.7 and later plus the Fable/Mythos + * line. Note this is not limited to thinking requests, which is why the + * non-thinking path has to honour it too. + */ + private val NoSamplingParamsMarkers = Seq( + "claude-opus-4-7", "claude-opus-4-8", "claude-opus-5", + "claude-sonnet-5", + "claude-fable-5", "claude-mythos-5", "claude-mythos-preview" + ) + + private def matches(model: String, markers: Seq[String]): Boolean = { + val m = model.toLowerCase + markers.exists(m.contains) + } + + /** + * Thinking request shape for a model. + * + * Defaults to Adaptive for unrecognized names: new Anthropic models move + * forward, not back, so an unknown identifier is far more likely to be a + * newer adaptive-only model than a pre-4.6 one. + */ + def thinkingMode(model: String): ClaudeThinkingMode = + if (matches(model, ExtendedThinkingMarkers)) ClaudeThinkingMode.Extended + else ClaudeThinkingMode.Adaptive + + /** + * Whether a `temperature` value may be sent for this model at all. + * + * False for 4.7+ regardless of thinking state. + */ + def supportsSamplingParams(model: String): Boolean = + !matches(model, NoSamplingParamsMarkers) + + /** + * Map the extension's reasoning_effort config onto Anthropic's + * `output_config.effort` value. + * + * The extension accepts none|low|medium|high|xhigh. Anthropic accepts + * low|medium|high|xhigh|max -- there is no "none", so it is treated as + * "no explicit effort" and the API default (high) applies. + */ + def effortValue(reasoningEffort: Option[String]): Option[String] = + reasoningEffort.map(_.toLowerCase.trim).collect { + case e @ ("low" | "medium" | "high" | "xhigh" | "max") => e + } +} diff --git a/src/main/providers/ClaudeProvider.scala b/src/main/providers/ClaudeProvider.scala index e845054..1db0180 100644 --- a/src/main/providers/ClaudeProvider.scala +++ b/src/main/providers/ClaudeProvider.scala @@ -39,14 +39,13 @@ class ClaudeProvider(implicit ec: ExecutionContext) extends BaseHttpProvider { } override protected def buildHeaders(apiKey: Option[String]): Map[String, String] = { - // Note: This reads ENABLE_THINKING from the provider's configStore, which stays in sync - // because LLMExtension invalidates the provider (currentProvider = None) on thinking config changes. - val thinkingEnabled = configStore.get(ConfigStore.ENABLE_THINKING).exists(_.toLowerCase == "true") - val version = if (thinkingEnabled) "2025-04-15" else "2023-06-01" + // 2023-06-01 is the only current Anthropic API version; thinking does not + // require a different one. (A previous version bumped this to "2025-04-15" + // when thinking was on, which is not a version Anthropic publishes.) Map( "x-api-key" -> apiKey.getOrElse(throw new IllegalStateException("API key required for Claude")), "content-type" -> "application/json", - "anthropic-version" -> version + "anthropic-version" -> ClaudeProvider.ApiVersion ) } @@ -78,28 +77,48 @@ class ClaudeProvider(implicit ec: ExecutionContext) extends BaseHttpProvider { baseRequest("system") = sysMsg.content } - if (isThinking) { - // Anthropic requires budget >= 1024 AND budget < max_tokens, so max_tokens must be > 1024 - if (maxTokens <= 1024) { - throw new RuntimeException( - s"Claude thinking requires max_tokens > 1024 (current: $maxTokens). " + - "The thinking budget must be at least 1024 and less than max_tokens." - ) - } + // Newer Claude generations (4.7+) reject non-default temperature/top_p/top_k + // with a 400 on EVERY request, thinking or not -- so this gate also applies + // to the non-thinking path below. + val allowsSampling = ClaudeModelCapabilities.supportsSamplingParams(request.model) - // Anthropic requires temperature=1.0 when thinking is enabled - baseRequest("temperature") = 1.0 + if (isThinking) { + ClaudeModelCapabilities.thinkingMode(request.model) match { + case ClaudeThinkingMode.Extended => + // Legacy shape: budget >= 1024 AND budget < max_tokens, so max_tokens must be > 1024 + if (maxTokens <= 1024) { + throw new RuntimeException( + s"Claude thinking requires max_tokens > 1024 (current: $maxTokens). " + + "The thinking budget must be at least 1024 and less than max_tokens." + ) + } - // Budget must be >= 1024 and < max_tokens - val budget = request.thinkingConfig.flatMap(_.budgetTokens) - .map(b => math.max(1024, math.min(b, maxTokens - 1))) - .getOrElse(math.max(1024, math.min(4096, maxTokens - 1))) + // These models require temperature=1.0 when thinking is enabled + if (allowsSampling) { + baseRequest("temperature") = 1.0 + } - baseRequest("thinking") = ujson.Obj( - "type" -> "enabled", - "budget_tokens" -> budget - ) - } else { + val budget = request.thinkingConfig.flatMap(_.budgetTokens) + .map(b => math.max(1024, math.min(b, maxTokens - 1))) + .getOrElse(math.max(1024, math.min(4096, maxTokens - 1))) + + baseRequest("thinking") = ujson.Obj( + "type" -> "enabled", + "budget_tokens" -> budget + ) + + case ClaudeThinkingMode.Adaptive => + // Adaptive models take no budget_tokens and no temperature; depth is + // steered by output_config.effort instead. + baseRequest("thinking") = ujson.Obj("type" -> "adaptive") + + ClaudeModelCapabilities + .effortValue(request.thinkingConfig.flatMap(_.reasoningEffort)) + .foreach { effort => + baseRequest("output_config") = ujson.Obj("effort" -> effort) + } + } + } else if (allowsSampling) { request.temperature.foreach { temp => baseRequest("temperature") = temp } @@ -166,3 +185,8 @@ class ClaudeProvider(implicit ec: ExecutionContext) extends BaseHttpProvider { } } } + +object ClaudeProvider { + /** The only Anthropic API version currently published. */ + val ApiVersion: String = "2023-06-01" +} diff --git a/src/test/ClaudeRequestSpec.scala b/src/test/ClaudeRequestSpec.scala new file mode 100644 index 0000000..58f92aa --- /dev/null +++ b/src/test/ClaudeRequestSpec.scala @@ -0,0 +1,164 @@ +// ABOUTME: Deterministic tests for ClaudeProvider request shaping across Claude generations +// ABOUTME: Asserts extended vs adaptive thinking and that temperature is omitted on 4.7+ models +package org.nlogo.extensions.llm.providers + +import org.nlogo.extensions.llm.models.{ChatMessage, ChatRequest, ThinkingConfig} +import org.scalatest.funsuite.AnyFunSuite + +/** + * Exposes the protected request builder so the JSON body can be asserted + * without performing any network I/O. + */ +class InspectableClaudeProvider extends ClaudeProvider()(using scala.concurrent.ExecutionContext.global) { + def buildRequest(request: ChatRequest): ujson.Value = createProviderRequest(request) +} + +class ClaudeRequestSpec extends AnyFunSuite { + + private val provider = new InspectableClaudeProvider + + private def request( + model: String, + thinking: Option[ThinkingConfig] = None, + temperature: Option[Double] = None, + maxTokens: Option[Int] = Some(4000) + ): ChatRequest = + ChatRequest( + model = model, + messages = Seq(ChatMessage.user("hi")), + maxTokens = maxTokens, + temperature = temperature, + thinkingConfig = thinking + ) + + private def keys(v: ujson.Value): Set[String] = v.obj.keys.toSet + + // --- Extended-thinking generation (Claude 4.5 and earlier) --- + + test("extended-thinking model uses enabled+budget_tokens and forces temperature 1.0") { + val body = provider.buildRequest( + request("claude-haiku-4-5-20251001", thinking = Some(ThinkingConfig(enabled = true))) + ) + + assert(body("thinking")("type").str == "enabled") + assert(body("thinking")("budget_tokens").num > 0) + assert(body("temperature").num == 1.0) + assert(!keys(body).contains("output_config"), "extended-thinking models must not receive output_config.effort") + } + + test("extended-thinking budget is clamped below max_tokens") { + val body = provider.buildRequest( + request( + "claude-haiku-4-5-20251001", + thinking = Some(ThinkingConfig(enabled = true, budgetTokens = Some(99999))), + maxTokens = Some(2000) + ) + ) + assert(body("thinking")("budget_tokens").num == 1999) + } + + test("extended-thinking rejects max_tokens at or below 1024") { + val ex = intercept[RuntimeException] { + provider.buildRequest( + request("claude-haiku-4-5-20251001", thinking = Some(ThinkingConfig(enabled = true)), maxTokens = Some(1024)) + ) + } + assert(ex.getMessage.contains("max_tokens > 1024")) + } + + // --- Adaptive-thinking generation (Claude 4.7+) --- + + test("adaptive-thinking model uses type adaptive with no budget_tokens and no temperature") { + val body = provider.buildRequest( + request("claude-opus-4-7", thinking = Some(ThinkingConfig(enabled = true))) + ) + + assert(body("thinking")("type").str == "adaptive") + assert(!keys(body("thinking")).contains("budget_tokens"), "adaptive thinking must not send budget_tokens") + assert(!keys(body).contains("temperature"), "adaptive-thinking models reject temperature") + } + + test("reasoning effort maps onto output_config.effort for adaptive models") { + val body = provider.buildRequest( + request("claude-opus-5", thinking = Some(ThinkingConfig(enabled = true, reasoningEffort = Some("xhigh")))) + ) + assert(body("output_config")("effort").str == "xhigh") + } + + test("reasoning effort 'none' sends no output_config so the API default applies") { + val body = provider.buildRequest( + request("claude-opus-5", thinking = Some(ThinkingConfig(enabled = true, reasoningEffort = Some("none")))) + ) + assert(!keys(body).contains("output_config")) + } + + test("adaptive model ignores budget_tokens config rather than sending it") { + val body = provider.buildRequest( + request("claude-sonnet-5", thinking = Some(ThinkingConfig(enabled = true, budgetTokens = Some(2048)))) + ) + assert(body("thinking")("type").str == "adaptive") + assert(!keys(body("thinking")).contains("budget_tokens")) + } + + // --- Non-thinking requests --- + + test("non-thinking request to a 4.7+ model sends NO temperature key") { + val body = provider.buildRequest(request("claude-opus-4-7", temperature = Some(0.7))) + + assert(!keys(body).contains("temperature"), s"temperature must be suppressed on 4.7+, got: $body") + assert(!keys(body).contains("thinking")) + } + + test("non-thinking request to Opus 5 and Sonnet 5 sends NO temperature key") { + Seq("claude-opus-5", "claude-sonnet-5", "claude-opus-4-8", "claude-fable-5").foreach { model => + val body = provider.buildRequest(request(model, temperature = Some(0.3))) + assert(!keys(body).contains("temperature"), s"temperature must be suppressed on $model") + } + } + + test("non-thinking request to an older model still honors temperature") { + val body = provider.buildRequest(request("claude-haiku-4-5-20251001", temperature = Some(0.7))) + assert(body("temperature").num == 0.7) + } + + // --- Capability table --- + + test("thinking mode classification matches Anthropic generations") { + import ClaudeThinkingMode._ + assert(ClaudeModelCapabilities.thinkingMode("claude-haiku-4-5-20251001") == Extended) + assert(ClaudeModelCapabilities.thinkingMode("claude-sonnet-4-5-20250929") == Extended) + assert(ClaudeModelCapabilities.thinkingMode("claude-opus-4-5-20251101") == Extended) + assert(ClaudeModelCapabilities.thinkingMode("claude-3-7-sonnet-20250219") == Extended) + + assert(ClaudeModelCapabilities.thinkingMode("claude-opus-4-7") == Adaptive) + assert(ClaudeModelCapabilities.thinkingMode("claude-opus-4-8") == Adaptive) + assert(ClaudeModelCapabilities.thinkingMode("claude-opus-5") == Adaptive) + assert(ClaudeModelCapabilities.thinkingMode("claude-sonnet-5") == Adaptive) + assert(ClaudeModelCapabilities.thinkingMode("claude-fable-5") == Adaptive) + // Unknown/newer identifiers default forward to adaptive. + assert(ClaudeModelCapabilities.thinkingMode("claude-something-new") == Adaptive) + } + + test("sampling-parameter support matches Anthropic generations") { + assert(ClaudeModelCapabilities.supportsSamplingParams("claude-haiku-4-5-20251001")) + assert(ClaudeModelCapabilities.supportsSamplingParams("claude-opus-4-6")) + + assert(!ClaudeModelCapabilities.supportsSamplingParams("claude-opus-4-7")) + assert(!ClaudeModelCapabilities.supportsSamplingParams("claude-opus-4-8")) + assert(!ClaudeModelCapabilities.supportsSamplingParams("claude-opus-5")) + assert(!ClaudeModelCapabilities.supportsSamplingParams("claude-sonnet-5")) + assert(!ClaudeModelCapabilities.supportsSamplingParams("claude-fable-5")) + } + + test("effort values outside Anthropic's accepted set are dropped") { + assert(ClaudeModelCapabilities.effortValue(Some("high")).contains("high")) + assert(ClaudeModelCapabilities.effortValue(Some("HIGH")).contains("high")) + assert(ClaudeModelCapabilities.effortValue(Some("none")).isEmpty) + assert(ClaudeModelCapabilities.effortValue(Some("bogus")).isEmpty) + assert(ClaudeModelCapabilities.effortValue(None).isEmpty) + } + + test("api version header is the published one and does not vary with thinking") { + assert(ClaudeProvider.ApiVersion == "2023-06-01") + } +} From f491760297e75700198d7447595c62b0a196d5d0 Mon Sep 17 00:00:00 2001 From: JNK234 Date: Thu, 13 Aug 2026 18:46:25 -0500 Subject: [PATCH 3/3] fix: make the #63 fix reachable and default forward on unknown models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three follow-ups from review of this branch. 1. models.yaml carried no model that takes the adaptive-thinking path, so the new request shaping was unreachable out of the box — every bundled Anthropic entry was 4.5-or-earlier. Adds claude-fable-5, claude-opus-5, claude-sonnet-5, claude-opus-4-8/4-7 and the 4.6 pair, with IDs taken from Anthropic's live model overview (dateless from 4.6 on, still pinned snapshots). Drops claude-3-7-sonnet, retired 19 Feb 2026 — listing a retired model is the defect #62 fixed. 2. supportsSamplingParams was a denylist of models that reject temperature, so claude-sonnet-4-7 or claude-opus-4-9 matched nothing, fell through as permitted, and would be sent a temperature they 400 on. It is now an allowlist of generations known to ACCEPT sampling params, so unknown names default forward — the same direction thinkingMode already defaulted. The two checks are now consistent. 3. FALLBACK_CONFIG still held claude-3-5-haiku-latest, gemini-1.5-flash and bare llama3.2 — the exact retired models #62 removed elsewhere. The existing drift guard reads the loaded registry, so it could not see them. Updated, and a second guard now covers the fallback map. The new guard was mutation-tested: reintroducing a stale Ollama default fails it with the drifted name, and passes once restored. 73 tests pass. --- .../providers/ClaudeModelCapabilities.scala | 28 ++++++++++++------- src/main/providers/ModelRegistry.scala | 17 +++++++---- src/main/resources/config/models.yaml | 25 +++++++++++------ src/test/ClaudeRequestSpec.scala | 13 +++++++++ src/test/ProviderDefaultsSpec.scala | 18 ++++++++++++ 5 files changed, 77 insertions(+), 24 deletions(-) diff --git a/src/main/providers/ClaudeModelCapabilities.scala b/src/main/providers/ClaudeModelCapabilities.scala index 258908d..83b4b77 100644 --- a/src/main/providers/ClaudeModelCapabilities.scala +++ b/src/main/providers/ClaudeModelCapabilities.scala @@ -51,17 +51,25 @@ object ClaudeModelCapabilities { ) /** - * Generations that reject non-default `temperature`/`top_p`/`top_k` with a 400 - * on EVERY request, thinking or not. + * Generations that still ACCEPT `temperature`/`top_p`/`top_k`. * - * Per Anthropic's docs this covers Claude 4.7 and later plus the Fable/Mythos - * line. Note this is not limited to thinking requests, which is why the - * non-thinking path has to honour it too. + * Claude 4.7 and later, plus the Fable/Mythos line, reject a non-default + * `temperature` with a 400 on EVERY request, thinking or not — which is why + * the non-thinking path has to honour this too. + * + * This is deliberately an allowlist of older generations rather than a denylist + * of newer ones. A denylist has to enumerate every future model ID, so + * `claude-sonnet-4-7` or `claude-opus-4-9` would match nothing, fall through as + * permitted, and be sent a temperature they reject. Listing what is known to + * accept sampling params instead makes the unknown case default FORWARD — the + * same direction `thinkingMode` already defaults, so the two stay consistent. */ - private val NoSamplingParamsMarkers = Seq( - "claude-opus-4-7", "claude-opus-4-8", "claude-opus-5", - "claude-sonnet-5", - "claude-fable-5", "claude-mythos-5", "claude-mythos-preview" + private val SamplingParamsMarkers = Seq( + "claude-3-5", "claude-3-7", "claude-3-opus", "claude-3-haiku", "claude-3-sonnet", + "claude-opus-4-0", "claude-opus-4-1", "claude-opus-4-5", "claude-opus-4-6", + "claude-sonnet-4-0", "claude-sonnet-4-5", "claude-sonnet-4-6", + "claude-haiku-4-5", + "claude-opus-4-20", "claude-sonnet-4-20" ) private def matches(model: String, markers: Seq[String]): Boolean = { @@ -86,7 +94,7 @@ object ClaudeModelCapabilities { * False for 4.7+ regardless of thinking state. */ def supportsSamplingParams(model: String): Boolean = - !matches(model, NoSamplingParamsMarkers) + matches(model, SamplingParamsMarkers) /** * Map the extension's reasoning_effort config onto Anthropic's diff --git a/src/main/providers/ModelRegistry.scala b/src/main/providers/ModelRegistry.scala index cb11a52..387f1b8 100644 --- a/src/main/providers/ModelRegistry.scala +++ b/src/main/providers/ModelRegistry.scala @@ -27,15 +27,20 @@ object ModelRegistry { private var modelDirLoaded: Option[String] = None private var overrideLoadMessage: Option[String] = None - // Fallback config in case YAML loading fails (minimal set for stability) - private val FALLBACK_CONFIG: Map[String, ProviderModels] = Map( + // Fallback config in case YAML loading fails (minimal set for stability). + // + // Every provider's defaultModel must appear here as well as in models.yaml. + // The drift guard in ProviderDefaultsSpec checks descriptors against the + // LOADED registry, so a stale entry here survives it — which is how the + // retired models #62 removed elsewhere lingered in this map. + private[llm] val FALLBACK_CONFIG: Map[String, ProviderModels] = Map( "openai" -> ProviderModels(Set("gpt-4o", "gpt-4o-mini", "gpt-4", "gpt-3.5-turbo"), isCustom = false), "anthropic" -> ProviderModels(Set( - "claude-3-5-sonnet-20241022", "claude-3-5-sonnet-latest", - "claude-3-5-haiku-20241022", "claude-3-5-haiku-latest" + "claude-opus-5", "claude-sonnet-5", + "claude-haiku-4-5-20251001", "claude-opus-4-5-20251101" ), isCustom = false), - "gemini" -> ProviderModels(Set("gemini-1.5-pro", "gemini-1.5-flash", "gemini-2.0-flash-exp"), isCustom = false), - "ollama" -> ProviderModels(Set("llama3.2", "llama3.1", "mistral", "phi4"), isCustom = false), + "gemini" -> ProviderModels(Set("gemini-2.5-pro", "gemini-2.5-flash", "gemini-3-pro-preview"), isCustom = false), + "ollama" -> ProviderModels(Set("llama3.2:3b", "llama3.2:1b", "mistral", "phi4"), isCustom = false), "openrouter" -> ProviderModels(Set("openai/gpt-4o", "openai/gpt-4o-mini", "anthropic/claude-3.5-sonnet", "deepseek/deepseek-r1"), isCustom = false), "together" -> ProviderModels(Set("meta-llama/Llama-3.3-70B-Instruct-Turbo", "deepseek-ai/DeepSeek-R1", "Qwen/Qwen2.5-72B-Instruct-Turbo"), isCustom = false) ) diff --git a/src/main/resources/config/models.yaml b/src/main/resources/config/models.yaml index ca4c077..478fa11 100644 --- a/src/main/resources/config/models.yaml +++ b/src/main/resources/config/models.yaml @@ -37,20 +37,29 @@ openai: - gpt-3.5-turbo anthropic: - # Claude 4.5 (latest production) + # Adaptive thinking (thinking.type "adaptive" + output_config.effort). + # These reject thinking.type "enabled" and any non-default temperature. + # IDs are dateless from the 4.6 generation on, and still pinned snapshots. + - claude-fable-5 + - claude-opus-5 + - claude-sonnet-5 + - claude-opus-4-8 + - claude-opus-4-7 + + # Adaptive thinking, extended thinking deprecated but still accepted + - claude-opus-4-6 + - claude-sonnet-4-6 + + # Extended thinking only (thinking.type "enabled" + budget_tokens). + # These reject "adaptive". - claude-opus-4-5-20251101 - claude-sonnet-4-5-20250929 - claude-haiku-4-5-20251001 - - # Claude 4.1 (current) + + # Claude 4.1 / 4 (extended thinking) - claude-opus-4-1-20250805 - - # Claude 4 (stable) - claude-sonnet-4-20250514 - claude-opus-4-20250514 - - # Claude 3.7 (deprecated, retiring Feb 19, 2026) - - claude-3-7-sonnet-20250219 # deprecated gemini: # Gemini 3 (latest preview) diff --git a/src/test/ClaudeRequestSpec.scala b/src/test/ClaudeRequestSpec.scala index 58f92aa..78e0bc0 100644 --- a/src/test/ClaudeRequestSpec.scala +++ b/src/test/ClaudeRequestSpec.scala @@ -142,6 +142,7 @@ class ClaudeRequestSpec extends AnyFunSuite { test("sampling-parameter support matches Anthropic generations") { assert(ClaudeModelCapabilities.supportsSamplingParams("claude-haiku-4-5-20251001")) assert(ClaudeModelCapabilities.supportsSamplingParams("claude-opus-4-6")) + assert(ClaudeModelCapabilities.supportsSamplingParams("claude-sonnet-4-6")) assert(!ClaudeModelCapabilities.supportsSamplingParams("claude-opus-4-7")) assert(!ClaudeModelCapabilities.supportsSamplingParams("claude-opus-4-8")) @@ -150,6 +151,18 @@ class ClaudeRequestSpec extends AnyFunSuite { assert(!ClaudeModelCapabilities.supportsSamplingParams("claude-fable-5")) } + test("unknown models default forward on BOTH capability checks") { + import ClaudeThinkingMode._ + // A denylist of "no sampling params" models had to enumerate every future + // ID, so these fell through as permitted and were sent a temperature they + // reject with a 400. Both checks must default the same direction. + for (m <- Seq("claude-sonnet-4-7", "claude-haiku-4-7", "claude-opus-4-9", + "claude-something-new")) { + assert(ClaudeModelCapabilities.thinkingMode(m) == Adaptive, s"$m thinking mode") + assert(!ClaudeModelCapabilities.supportsSamplingParams(m), s"$m sampling params") + } + } + test("effort values outside Anthropic's accepted set are dropped") { assert(ClaudeModelCapabilities.effortValue(Some("high")).contains("high")) assert(ClaudeModelCapabilities.effortValue(Some("HIGH")).contains("high")) diff --git a/src/test/ProviderDefaultsSpec.scala b/src/test/ProviderDefaultsSpec.scala index db16dc4..7e7c286 100644 --- a/src/test/ProviderDefaultsSpec.scala +++ b/src/test/ProviderDefaultsSpec.scala @@ -42,4 +42,22 @@ class ProviderDefaultsSpec extends AnyFunSuite { val empty = descriptors.filter(d => ModelRegistry.getSupportedModels(d.name).isEmpty).map(_.name) assert(empty.isEmpty, s"providers with no models in models.yaml: ${empty.mkString(", ")}") } + + test("every provider default model is present in the YAML-load fallback too") { + // The check above reads the LOADED registry, so a stale entry in + // FALLBACK_CONFIG survives it. That map is what the extension falls back on + // when models.yaml cannot be read, and it kept retired models long after + // they were removed from the YAML. + val drifted = descriptors.flatMap { d => + ModelRegistry.FALLBACK_CONFIG.get(d.name) match { + case None => + Some(s"provider '${d.name}' has no FALLBACK_CONFIG entry") + case Some(pm) if !pm.models.contains(d.defaultModel) => + Some(s"provider '${d.name}' default '${d.defaultModel}' missing from FALLBACK_CONFIG " + + s"(has: ${pm.models.toSeq.sorted.mkString(", ")})") + case _ => None + } + } + assert(drifted.isEmpty, s"\n${drifted.mkString("\n")}") + } }