diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39007d4..fad13a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,13 @@ on: push: branches: [main] pull_request: - branches: [main] + # Also run on PRs stacked onto a feature branch, not just those targeting + # main, so every PR in a stack is verified before its base lands. + branches: + - main + - "feat/**" + - "fix/**" + - "chore/**" jobs: build: diff --git a/.github/workflows/param-guard.yml b/.github/workflows/param-guard.yml index 23871ad..71c86b4 100644 --- a/.github/workflows/param-guard.yml +++ b/.github/workflows/param-guard.yml @@ -10,7 +10,13 @@ name: Param guard on: pull_request: - branches: [main] + # Also run on PRs stacked onto a feature branch, not just those targeting + # main, so every PR in a stack is verified before its base lands. + branches: + - main + - "feat/**" + - "fix/**" + - "chore/**" types: [opened, synchronize, reopened, labeled, unlabeled] jobs: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 32bec9d..f4363f5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,6 +48,13 @@ You don't need to know the schema to file one. A link to the official docs is th - You can also use `{ not: }` to say "any value except this one". - See the [schema doc](docs/model-parameters-schema.md#applicability) for the exact rule syntax and evaluation semantics. + These rules are **enforced at runtime**, not just rendered on the site. The + `modelparams` package rejects parameter combinations your rules forbid, so a rule + that's too strict makes someone's valid request fail validation. Check it against + the provider's docs. When a rule references a parameter the request didn't set, + evaluation falls back to that parameter's `default`, because that is what the + provider applies in its place. + 6. **Auth-type rules of thumb:** - **`api_key`:** list parameters from the official API reference. Don't invent ones the API doesn't accept. - **`subscription`:** list user-facing toggles and presets the consumer can actually set. Skip implementation details. diff --git a/packages/modelparams/README.md b/packages/modelparams/README.md index c629368..7ec3221 100644 --- a/packages/modelparams/README.md +++ b/packages/modelparams/README.md @@ -96,6 +96,41 @@ import { paramsSchema } from "modelparams"; app.post("/chat", validator("json", paramsSchema("openai/gpt-4.1")), handler); ``` +### Conflicting parameters + +Some parameters are only accepted for certain values of _other_ parameters. Anthropic rejects `top_p` unless `temperature` is 1; a thinking budget does nothing unless thinking is on. `parseParams` enforces these rules, and `checkApplicability` reports them on their own: + +```ts +import { checkApplicability } from "modelparams"; + +checkApplicability("anthropic/claude-3-opus-20240229", { temperature: 0.5, top_p: 0.9 }); +// [{ path: "top_p", +// message: "top_p does not apply when temperature ≠ 1", +// conflictsWith: ["temperature"] }] +``` + +A parameter you never set still counts, because the provider applies its own default in your place. + +### Drop what won't fly + +`dropUnsupported` is the catalog-driven equivalent of LiteLLM's `drop_params`, extended to conditional conflicts. It returns a payload that is always safe to spread into a provider call, plus what it removed and why: + +```ts +import { dropUnsupported } from "modelparams"; + +const { params, dropped } = dropUnsupported("openai/gpt-5.5", { + temperature: 0.7, // gpt-5.5 exposes no temperature + reasoning_effort: "low", +}); + +// params → { reasoning_effort: "low" } +// dropped → [{ path: "temperature", code: "unknown_parameter", reason: "…" }] + +await openai.chat.completions.create({ model: "gpt-5.5", messages, ...params }); +``` + +Each dropped entry carries a `code`: `unknown_parameter`, `invalid_value`, or `not_applicable`. + ## API ### Types @@ -114,15 +149,19 @@ app.post("/chat", validator("json", paramsSchema("openai/gpt-4.1")), handler); ### Functions -| Function | Description | -| -------------------------- | ----------------------------------------------------------- | -| `getModel(id)` | The full catalog entry for a model id. | -| `getDefaults(id)` | The catalog-declared defaults. | -| `getParam(id, path)` | A single parameter's definition (range, enum values, etc.). | -| `listModels({ provider })` | List model ids, optionally filtered by provider. | -| `listAllModels()` | The full `CATALOG` array. | -| `parseParams(id, input)` | Validate an untrusted params object against the catalog. | -| `paramsSchema(id)` | A Standard Schema that validates a params object for `id`. | +| Function | Description | +| -------------------------------- | ----------------------------------------------------------- | +| `getModel(id)` | The full catalog entry for a model id. | +| `getDefaults(id)` | The catalog-declared defaults. | +| `getParam(id, path)` | A single parameter's definition (range, enum values, etc.). | +| `listModels({ provider })` | List model ids, optionally filtered by provider. | +| `listAllModels()` | The full `CATALOG` array. | +| `parseParams(id, input)` | Validate an untrusted params object against the catalog. | +| `paramsSchema(id)` | A Standard Schema that validates a params object for `id`. | +| `checkApplicability(id, params)` | Report parameters that conflict with others in the request. | +| `isApplicable(id, path, params)` | Is one parameter accepted alongside the rest? | +| `dropUnsupported(id, params)` | Strip everything the model won't accept. | +| `resolveModelId(input)` | Resolve a bare model slug or full id to a catalog id. | ### Constants diff --git a/packages/modelparams/src/applicability.ts b/packages/modelparams/src/applicability.ts new file mode 100644 index 0000000..65a3643 --- /dev/null +++ b/packages/modelparams/src/applicability.ts @@ -0,0 +1,269 @@ +import { checkValue } from "./check-value.js"; +import type { ModelId } from "./generated/model-ids.js"; +import { getModel } from "./helpers.js"; +import type { + ApplicabilityCondition, + ApplicabilityRule, + ApplicabilityValue, + JsonPrimitive, + Param, +} from "./types.js"; + +/** A parameter that was supplied but isn't accepted alongside the other values in the request. */ +export interface ApplicabilityIssue { + /** Dot path of the parameter that doesn't apply. */ + readonly path: string; + /** Human-readable cause, e.g. `top_p does not apply when temperature ≠ 1`. */ + readonly message: string; + /** Dot paths of the parameters whose values caused the conflict. */ + readonly conflictsWith: readonly string[]; +} + +function toRules( + input: ApplicabilityRule | readonly ApplicabilityRule[] | undefined, +): readonly ApplicabilityRule[] { + if (!input) return []; + return Array.isArray(input) + ? (input as readonly ApplicabilityRule[]) + : [input as ApplicabilityRule]; +} + +function isCondition(value: ApplicabilityValue): value is ApplicabilityCondition { + return typeof value === "object" && value !== null && !Array.isArray(value) && "not" in value; +} + +/** + * Does the observed value satisfy one entry of a rule? An absent value never + * satisfies a condition — we only report a conflict on positive evidence, so a + * parameter the request never mentions and the model never defaults can't + * trigger one. + */ +function conditionHolds( + observed: JsonPrimitive | undefined, + expected: ApplicabilityValue, +): boolean { + if (observed === undefined) return false; + if (isCondition(expected)) { + const { not } = expected; + return Array.isArray(not) ? !not.includes(observed) : observed !== not; + } + if (Array.isArray(expected)) return expected.includes(observed); + return observed === expected; +} + +function formatPrimitive(value: JsonPrimitive): string { + return typeof value === "string" ? `"${value}"` : String(value); +} + +function formatExpectation(path: string, expected: ApplicabilityValue): string { + if (isCondition(expected)) { + const { not } = expected; + return Array.isArray(not) + ? `${path} is not one of ${not.map(formatPrimitive).join(", ")}` + : `${path} ≠ ${formatPrimitive(not as JsonPrimitive)}`; + } + if (Array.isArray(expected)) { + return expected.length === 1 + ? `${path} = ${formatPrimitive(expected[0]!)}` + : `${path} is one of ${expected.map(formatPrimitive).join(", ")}`; + } + return `${path} = ${formatPrimitive(expected as JsonPrimitive)}`; +} + +function describeRule(rule: ApplicabilityRule): string { + return Object.entries(rule) + .map(([path, expected]) => formatExpectation(path, expected)) + .join(" and "); +} + +/** Every key must hold for the rule to match. */ +function ruleMatches( + rule: ApplicabilityRule, + resolve: (path: string) => JsonPrimitive | undefined, +): boolean { + return Object.entries(rule).every(([path, expected]) => conditionHolds(resolve(path), expected)); +} + +function anyRuleMatches( + rules: readonly ApplicabilityRule[], + resolve: (path: string) => JsonPrimitive | undefined, +): ApplicabilityRule | undefined { + return rules.find((rule) => ruleMatches(rule, resolve)); +} + +/** + * Build the effective view of a request: what the provider will actually see. + * An explicitly supplied value wins; otherwise the catalog default stands in, + * because that is what the provider applies when the key is omitted. + */ +function effectiveValues( + params: readonly Param[], + supplied: Readonly>, +): (path: string) => JsonPrimitive | undefined { + const defaults = new Map(); + for (const param of params) { + if (param.default !== undefined) defaults.set(param.path, param.default); + } + return (path) => { + if (Object.prototype.hasOwnProperty.call(supplied, path)) { + return supplied[path] as JsonPrimitive; + } + return defaults.get(path); + }; +} + +/** + * Is `path` accepted for this model given the rest of the request? + * + * Returns `true` for parameters with no applicability rules, and for parameters + * the model doesn't declare at all (that's `parseParams`' job to report, not + * this one's). + * + * @example + * isApplicable("anthropic/claude-3-opus-20240229", "top_p", { temperature: 0.5 }); + * // false — Anthropic rejects top_p unless temperature is 1 + */ +export function isApplicable( + id: ModelId, + path: string, + params: Readonly> = {}, +): boolean { + const all = getModel(id).params as readonly Param[]; + const param = all.find((p) => p.path === path); + if (!param?.applicability) return true; + const resolve = effectiveValues(all, params); + const { only, except } = param.applicability; + if (only && !anyRuleMatches(toRules(only), resolve)) return false; + if (except && anyRuleMatches(toRules(except), resolve)) return false; + return true; +} + +/** + * Report every supplied parameter that the model won't accept alongside the + * others. This is the check that catches the errors a per-parameter validator + * cannot see — `temperature and top_p cannot both be specified`, thinking-mode + * knobs on a non-thinking request, and so on. + * + * Only parameters present in `params` are reported; unknown parameters are left + * to {@link parseParams}. + * + * @example + * checkApplicability("anthropic/claude-3-opus-20240229", { temperature: 0.5, top_p: 0.9 }); + * // [{ path: "top_p", message: "top_p does not apply when temperature ≠ 1", conflictsWith: ["temperature"] }] + */ +export function checkApplicability( + id: ModelId, + params: Readonly>, +): readonly ApplicabilityIssue[] { + const all = getModel(id).params as readonly Param[]; + const resolve = effectiveValues(all, params); + const issues: ApplicabilityIssue[] = []; + + for (const param of all) { + if (!param.applicability) continue; + if (!Object.prototype.hasOwnProperty.call(params, param.path)) continue; + + const { only, except } = param.applicability; + + const onlyRules = toRules(only); + if (onlyRules.length > 0 && !anyRuleMatches(onlyRules, resolve)) { + issues.push({ + path: param.path, + message: `${param.path} only applies when ${onlyRules.map(describeRule).join(", or when ")}`, + conflictsWith: [...new Set(onlyRules.flatMap((rule) => Object.keys(rule)))], + }); + continue; + } + + const matchedExcept = anyRuleMatches(toRules(except), resolve); + if (matchedExcept) { + issues.push({ + path: param.path, + message: `${param.path} does not apply when ${describeRule(matchedExcept)}`, + conflictsWith: Object.keys(matchedExcept), + }); + } + } + + return issues; +} + +/** Why a parameter was removed: not in the catalog, bad value, or a conflict. */ +export type DropCode = "unknown_parameter" | "invalid_value" | "not_applicable"; + +/** One parameter removed by {@link dropUnsupported}, with the reason why. */ +export interface DroppedParam { + readonly path: string; + readonly value: unknown; + readonly code: DropCode; + readonly reason: string; + /** For `not_applicable`: the parameters whose values caused the conflict. */ + readonly conflictsWith?: readonly string[]; +} + +/** The result of {@link dropUnsupported}: what's safe to send, and what was removed. */ +export interface DropResult { + readonly params: Record; + readonly dropped: readonly DroppedParam[]; +} + +/** + * Strip everything the model won't accept and return a params object that is + * safe to spread into a provider call — the catalog-driven equivalent of + * LiteLLM's `drop_params`, extended to conditional conflicts. + * + * Drops unknown parameters, values that fail their type/range/enum constraint, + * and parameters that don't apply alongside the rest of the request. + * + * @example + * const { params, dropped } = dropUnsupported("openai/gpt-5.5", { + * temperature: 0.7, // dropped: gpt-5.5 exposes no temperature + * reasoning_effort: "low", + * }); + * await openai.chat.completions.create({ model: "gpt-5.5", messages, ...params }); + */ +export function dropUnsupported(id: ModelId, input: Readonly>): DropResult { + const all = getModel(id).params as readonly Param[]; + const defs = new Map(all.map((param) => [param.path, param])); + const dropped: DroppedParam[] = []; + const kept: Record = {}; + + for (const [path, value] of Object.entries(input)) { + const def = defs.get(path); + if (!def) { + dropped.push({ + path, + value, + code: "unknown_parameter", + reason: `${id} does not accept ${path}`, + }); + continue; + } + const problem = checkValue(def, value); + if (problem) { + dropped.push({ path, value, code: "invalid_value", reason: `${path} ${problem}` }); + continue; + } + kept[path] = value as JsonPrimitive; + } + + // Applicability is judged against the surviving request, then re-judged after + // each removal: dropping one parameter can open a gate another parameter's + // rule depended on. Each pass removes at least one key, so this terminates. + for (;;) { + const issues = checkApplicability(id, kept); + if (issues.length === 0) break; + for (const issue of issues) { + dropped.push({ + path: issue.path, + value: kept[issue.path], + code: "not_applicable", + reason: issue.message, + conflictsWith: issue.conflictsWith, + }); + delete kept[issue.path]; + } + } + + return { params: kept, dropped }; +} diff --git a/packages/modelparams/src/check-value.ts b/packages/modelparams/src/check-value.ts new file mode 100644 index 0000000..e1b1040 --- /dev/null +++ b/packages/modelparams/src/check-value.ts @@ -0,0 +1,30 @@ +import type { JsonPrimitive, Param } from "./types.js"; + +/** + * Validate one value against a single parameter definition — type, numeric + * range, and enum membership. Returns an error message, or null if the value is + * acceptable in isolation. + * + * Cross-parameter conflicts are not this function's concern; see + * `checkApplicability`. + */ +export function checkValue(def: Param, value: unknown): string | null { + if (def.type === "boolean") { + return typeof value === "boolean" ? null : "must be a boolean"; + } + if (def.type === "string") { + return typeof value === "string" ? null : "must be a string"; + } + if (def.type === "enum") { + const values = def.values ?? []; + if (values.includes(value as JsonPrimitive)) return null; + return `must be one of ${values.map((v) => JSON.stringify(v)).join(", ")}`; + } + // "integer" | "number" + if (typeof value !== "number" || Number.isNaN(value)) return "must be a number"; + if (def.type === "integer" && !Number.isInteger(value)) return "must be an integer"; + const { min, max } = def.range ?? {}; + if (min !== undefined && value < min) return `must be >= ${min}`; + if (max !== undefined && value > max) return `must be <= ${max}`; + return null; +} diff --git a/packages/modelparams/src/index.ts b/packages/modelparams/src/index.ts index f25a433..6bc8823 100644 --- a/packages/modelparams/src/index.ts +++ b/packages/modelparams/src/index.ts @@ -6,7 +6,12 @@ export type { ParamGroup, ParamRange, JsonPrimitive, + Applicability, + ApplicabilityCondition, + ApplicabilityRule, + ApplicabilityValue, } from "./types.js"; +export type { ApplicabilityIssue, DropCode, DroppedParam, DropResult } from "./applicability.js"; export type { ModelId, Provider } from "./generated/model-ids.js"; export type { ParamsById } from "./generated/params-by-id.js"; export type { CatalogEntry } from "./generated/data.js"; @@ -23,3 +28,6 @@ export { CATALOG, BY_ID } from "./generated/data.js"; export { getModel, getDefaults, listModels, getParam, listAllModels } from "./helpers.js"; export { parseParams, paramsSchema } from "./parse.js"; +export { checkApplicability, dropUnsupported, isApplicable } from "./applicability.js"; +export { resolveModelId } from "./resolve.js"; +export type { ResolveResult } from "./resolve.js"; diff --git a/packages/modelparams/src/parse.ts b/packages/modelparams/src/parse.ts index 854bccb..3f09d7c 100644 --- a/packages/modelparams/src/parse.ts +++ b/packages/modelparams/src/parse.ts @@ -1,3 +1,5 @@ +import { checkApplicability } from "./applicability.js"; +import { checkValue } from "./check-value.js"; import type { ModelId } from "./generated/model-ids.js"; import { getModel } from "./helpers.js"; import type { JsonPrimitive, Param } from "./types.js"; @@ -15,36 +17,17 @@ export type ParseParamsResult = | { readonly success: true; readonly value: Record } | { readonly success: false; readonly issues: readonly ParamIssue[] }; -/** Validate one value against a parameter definition. Returns an error message, or null if ok. */ -function checkValue(def: Param, value: unknown): string | null { - if (def.type === "boolean") { - return typeof value === "boolean" ? null : "must be a boolean"; - } - if (def.type === "string") { - return typeof value === "string" ? null : "must be a string"; - } - if (def.type === "enum") { - const values = def.values ?? []; - if (values.includes(value as JsonPrimitive)) return null; - return `must be one of ${values.map((v) => JSON.stringify(v)).join(", ")}`; - } - // "integer" | "number" - if (typeof value !== "number" || Number.isNaN(value)) return "must be a number"; - if (def.type === "integer" && !Number.isInteger(value)) return "must be an integer"; - const { min, max } = def.range ?? {}; - if (min !== undefined && value < min) return `must be >= ${min}`; - if (max !== undefined && value > max) return `must be <= ${max}`; - return null; -} - /** * Validate an untrusted params object (e.g. an HTTP request body) against a * model's catalog. Unknown keys, wrong types, out-of-range numbers and invalid * enum values are reported. This is the runtime complement to `ParamsOf`, * which only constrains params known at compile time. * - * Note: parameters are validated independently; cross-parameter `applicability` - * rules (e.g. a knob that only applies when another is set) are not yet enforced. + * Cross-parameter `applicability` rules are enforced too, so combinations the + * provider rejects at runtime — `temperature` alongside a thinking budget, say — + * surface here rather than as a 400 from the provider. Every parameter is + * checked in isolation first; conflicts are only reported once each value is + * individually valid. * * @example * const result = parseParams("openai/gpt-4.1", req.body.params); @@ -77,7 +60,23 @@ export function parseParams(id: ModelId, input: unknown): ParseParamsResult { value[key] = raw as JsonPrimitive; } - return issues.length > 0 ? { success: false, issues } : { success: true, value }; + if (issues.length > 0) return { success: false, issues }; + + // Only worth asking once every value is individually sound — an out-of-range + // temperature would otherwise produce a confusing conflict report on top of + // the range error. + const conflicts = checkApplicability(id, value); + if (conflicts.length > 0) { + return { + success: false, + issues: conflicts.map((conflict) => ({ + message: conflict.message, + path: [conflict.path], + })), + }; + } + + return { success: true, value }; } /** diff --git a/packages/modelparams/src/resolve.ts b/packages/modelparams/src/resolve.ts new file mode 100644 index 0000000..62a0b7e --- /dev/null +++ b/packages/modelparams/src/resolve.ts @@ -0,0 +1,77 @@ +import { MODEL_IDS, type ModelId } from "./generated/model-ids.js"; + +/** The outcome of {@link resolveModelId}. */ +export type ResolveResult = + | { readonly ok: true; readonly id: ModelId } + | { readonly ok: false; readonly reason: "not_found"; readonly suggestions: readonly ModelId[] } + | { readonly ok: false; readonly reason: "ambiguous"; readonly matches: readonly ModelId[] }; + +function sharedPrefixLength(a: string, b: string): number { + let i = 0; + while (i < a.length && i < b.length && a[i] === b[i]) i += 1; + return i; +} + +/** + * Rank a catalog id against an unresolvable query so the caller gets usable + * near-misses instead of an empty list. + * + * Scored against the bare model slug as well as the full id. A query like + * `gpt-5.5-turbo-ultra` shares no prefix with `openai/gpt-5.5` because the + * provider gets in the way, but it does contain that model's slug, and that is + * the match worth surfacing. + */ +function scoreCandidate(id: string, query: string): number { + const slug = id.slice(id.indexOf("/") + 1); + // A slash means the caller is already thinking in qualified ids, so don't let + // a bare slug outrank the full form. + const targets = query.includes("/") ? [id] : [slug, id]; + + let best = 0; + for (const target of targets) { + if (target === query) { + best = Math.max(best, 4); + } else if (target.includes(query) || (target.length >= 4 && query.includes(target))) { + best = Math.max(best, 3); + } else if (sharedPrefixLength(target, query) >= 4) { + best = Math.max(best, 1); + } + } + return best; +} + +/** + * Resolve a user-supplied model reference to a catalog id. + * + * Accepts a full `provider/model` id, or a bare provider-native model slug + * (`claude-opus-4-7`) when exactly one provider publishes it. A bare slug that + * several providers publish is reported as ambiguous rather than guessed at. + * + * @example + * resolveModelId("gpt-5.5"); // → { ok: true, id: "openai/gpt-5.5" } + * resolveModelId("anthropic/claude-opus-4-7"); // → { ok: true, id: "anthropic/claude-opus-4-7" } + */ +export function resolveModelId(input: string): ResolveResult { + const query = input.trim(); + if (MODEL_IDS.includes(query as ModelId)) { + return { ok: true, id: query as ModelId }; + } + + if (!query.includes("/")) { + const matches = MODEL_IDS.filter((id) => id.slice(id.indexOf("/") + 1) === query); + if (matches.length === 1) return { ok: true, id: matches[0]! }; + if (matches.length > 1) return { ok: false, reason: "ambiguous", matches }; + } + + const needle = query.toLowerCase(); + const suggestions = MODEL_IDS.map((id) => ({ + id, + score: scoreCandidate(id.toLowerCase(), needle), + })) + .filter((c) => c.score > 0) + .sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)) + .slice(0, 5) + .map((c) => c.id); + + return { ok: false, reason: "not_found", suggestions }; +} diff --git a/packages/modelparams/src/types.ts b/packages/modelparams/src/types.ts index e856285..b739cab 100644 --- a/packages/modelparams/src/types.ts +++ b/packages/modelparams/src/types.ts @@ -47,6 +47,35 @@ export interface ParamRange { readonly step?: number; } +/** Negated match: the referenced parameter must not equal (or be among) `not`. */ +export interface ApplicabilityCondition { + readonly not: JsonPrimitive | readonly JsonPrimitive[]; +} + +/** + * What another parameter must be for a rule to match: an exact value, one of a + * set of values, or anything but a value (`{ not: ... }`). + */ +export type ApplicabilityValue = JsonPrimitive | readonly JsonPrimitive[] | ApplicabilityCondition; + +/** + * One condition set, keyed by the dot path of the parameter it constrains. + * Every entry must hold for the rule to match (AND). + */ +export type ApplicabilityRule = { readonly [path: string]: ApplicabilityValue }; + +/** + * When a parameter is accepted, expressed in terms of the other parameters in + * the same request. A list of rules matches if any one of them matches (OR). + * + * `only` — the parameter applies *only* while a rule matches. + * `except` — the parameter does *not* apply while a rule matches. + */ +export interface Applicability { + readonly only?: ApplicabilityRule | readonly ApplicabilityRule[]; + readonly except?: ApplicabilityRule | readonly ApplicabilityRule[]; +} + /** * A single parameter definition in a loose, easy-to-iterate shape — the runtime * counterpart to the precise per-model `ParamsOf` types. @@ -71,4 +100,6 @@ export interface Param { readonly range?: ParamRange; /** Present on `enum` params. */ readonly values?: readonly JsonPrimitive[]; + /** When set, the parameter is only accepted for some values of its siblings. */ + readonly applicability?: Applicability; } diff --git a/packages/modelparams/tests/applicability.test.ts b/packages/modelparams/tests/applicability.test.ts new file mode 100644 index 0000000..5b37f24 --- /dev/null +++ b/packages/modelparams/tests/applicability.test.ts @@ -0,0 +1,222 @@ +import { describe, expect, it } from "vitest"; +import { + checkApplicability, + dropUnsupported, + getDefaults, + getModel, + getParam, + isApplicable, + MODEL_IDS, + parseParams, + type Param, +} from "../src/index.js"; + +// These fixtures are real catalog entries, picked because they carry the rule +// shapes the engine has to handle. If the catalog changes shape under them the +// assertions below should fail loudly rather than silently stop testing +// anything — hence the guards. +const OPUS_3 = "anthropic/claude-3-opus-20240229"; +const OPUS_47 = "anthropic/claude-opus-4-7"; + +describe("catalog fixtures still carry the rules under test", () => { + it("claude-3-opus top_p is gated on temperature", () => { + const top_p = getParam(OPUS_3, "top_p") as Param | undefined; + expect(top_p?.applicability?.except).toBeDefined(); + }); + + it("claude-opus-4-7 thinking.display is gated on thinking.type", () => { + const display = getParam(OPUS_47, "thinking.display") as Param | undefined; + expect(display?.applicability?.only).toBeDefined(); + }); +}); + +describe("checkApplicability", () => { + it("flags top_p when temperature is moved off its default", () => { + const issues = checkApplicability(OPUS_3, { temperature: 0.5, top_p: 0.9 }); + expect(issues.map((i) => i.path)).toContain("top_p"); + const conflict = issues.find((i) => i.path === "top_p"); + expect(conflict?.conflictsWith).toContain("temperature"); + expect(conflict?.message).toMatch(/temperature/); + }); + + it("allows top_p when temperature is left at its default", () => { + expect(checkApplicability(OPUS_3, { top_p: 0.9 })).toEqual([]); + }); + + it("allows top_p when temperature is explicitly set to the permitted value", () => { + expect(checkApplicability(OPUS_3, { temperature: 1, top_p: 0.9 })).toEqual([]); + }); + + it("ignores parameters the request never supplied", () => { + // temperature conflicts with top_p, but top_p isn't in the request. + expect(checkApplicability(OPUS_3, { temperature: 0.5 })).toEqual([]); + }); + + it("reports an `only` rule when the gating parameter has the wrong value", () => { + const display = getParam(OPUS_47, "thinking.display") as Param; + const allowed = (display.applicability?.only as { "thinking.type": string[] })["thinking.type"]; + const thinkingType = getParam(OPUS_47, "thinking.type") as Param; + const disallowed = (thinkingType.values ?? []).find((v) => !allowed.includes(v as string)); + expect(disallowed).toBeDefined(); + + const issues = checkApplicability(OPUS_47, { + "thinking.type": disallowed, + "thinking.display": display.values?.[0], + }); + expect(issues.map((i) => i.path)).toContain("thinking.display"); + expect(issues.find((i) => i.path === "thinking.display")?.message).toMatch(/only applies when/); + }); + + it("accepts an `only` rule when the gating parameter matches", () => { + const display = getParam(OPUS_47, "thinking.display") as Param; + const allowed = (display.applicability?.only as { "thinking.type": string[] })["thinking.type"]; + const issues = checkApplicability(OPUS_47, { + "thinking.type": allowed[0], + "thinking.display": display.values?.[0], + }); + expect(issues.map((i) => i.path)).not.toContain("thinking.display"); + }); + + it("returns nothing for a model with no applicability rules", () => { + const plain = getModel("openai/gpt-5.5").params as readonly Param[]; + expect(plain.every((p) => !p.applicability)).toBe(true); + expect(checkApplicability("openai/gpt-5.5", { reasoning_effort: "low" })).toEqual([]); + }); +}); + +describe("isApplicable", () => { + it("is true for a parameter with no rules", () => { + expect(isApplicable(OPUS_3, "max_tokens", {})).toBe(true); + }); + + it("is true for a parameter the model doesn't declare", () => { + expect(isApplicable(OPUS_3, "frequency_penalty", {})).toBe(true); + }); + + it("tracks the surrounding request", () => { + expect(isApplicable(OPUS_3, "top_p", { temperature: 1 })).toBe(true); + expect(isApplicable(OPUS_3, "top_p", { temperature: 0.2 })).toBe(false); + }); + + it("defaults to the catalog value when the gating parameter is absent", () => { + // claude-3-opus defaults temperature to 1, which is what the rule permits. + expect(getParam(OPUS_3, "temperature")?.default).toBe(1); + expect(isApplicable(OPUS_3, "top_p", {})).toBe(true); + }); +}); + +describe("parseParams enforces applicability", () => { + it("rejects a conflicting combination", () => { + const result = parseParams(OPUS_3, { temperature: 0.5, top_p: 0.9 }); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.issues.some((i) => i.path[0] === "top_p")).toBe(true); + }); + + it("still accepts a valid combination", () => { + const result = parseParams(OPUS_3, { temperature: 1, top_p: 0.9 }); + expect(result.success).toBe(true); + }); + + it("reports the range error rather than a conflict when a value is out of range", () => { + const result = parseParams(OPUS_3, { temperature: 99, top_p: 0.9 }); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.issues).toHaveLength(1); + expect(result.issues[0]?.path[0]).toBe("temperature"); + }); +}); + +describe("dropUnsupported", () => { + it("drops a parameter the model doesn't accept", () => { + const { params, dropped } = dropUnsupported("openai/gpt-5.5", { + temperature: 0.7, + reasoning_effort: "low", + }); + expect(params).toEqual({ reasoning_effort: "low" }); + expect(dropped.map((d) => d.path)).toEqual(["temperature"]); + }); + + it("drops the conflicting parameter, keeping the rest of the request", () => { + const { params, dropped } = dropUnsupported(OPUS_3, { + temperature: 0.5, + top_p: 0.9, + max_tokens: 1024, + }); + expect(params).toEqual({ temperature: 0.5, max_tokens: 1024 }); + expect(dropped.map((d) => d.path)).toEqual(["top_p"]); + expect(dropped[0]?.value).toBe(0.9); + expect(dropped[0]?.reason).toMatch(/temperature/); + }); + + it("drops values that fail their own constraint", () => { + const { params, dropped } = dropUnsupported(OPUS_3, { temperature: 99, max_tokens: 512 }); + expect(params).toEqual({ max_tokens: 512 }); + expect(dropped.map((d) => d.path)).toEqual(["temperature"]); + }); + + it("leaves a already-valid request untouched", () => { + const input = { max_tokens: 1024, temperature: 1 }; + const { params, dropped } = dropUnsupported(OPUS_3, input); + expect(params).toEqual(input); + expect(dropped).toEqual([]); + }); + + it("always returns a request that parses clean", () => { + const { params } = dropUnsupported(OPUS_3, { + temperature: 0.5, + top_p: 0.9, + top_k: 40, + nonsense: true, + }); + expect(parseParams(OPUS_3, params).success).toBe(true); + }); +}); + +describe("a gated parameter set without opening its gate", () => { + // The catalog gives thinking.budget_tokens a default *and* gates it on + // thinking.type — the default describes the budget used once thinking is on, + // not a value that is always live. Passing the budget without enabling + // thinking is a silent no-op at the provider, so it should be reported. + const GATED = "anthropic/claude-3-7-sonnet-20250219"; + + it("is reported", () => { + const issues = checkApplicability(GATED, { "thinking.budget_tokens": 8000 }); + expect(issues.map((i) => i.path)).toContain("thinking.budget_tokens"); + }); + + it("is accepted once the gate is open", () => { + const issues = checkApplicability(GATED, { + "thinking.type": "enabled", + "thinking.budget_tokens": 8000, + }); + expect(issues).toEqual([]); + }); +}); + +describe("the whole catalog", () => { + it("converges: dropUnsupported always yields a request that parses clean", () => { + // Guards the fixed-point loop in dropUnsupported against both + // non-termination and emitting output it would itself reject. + for (const id of MODEL_IDS) { + const { params } = dropUnsupported(id, { ...getDefaults(id) }); + const result = parseParams(id, params); + expect( + result.success, + `${id}: ${result.success ? "" : result.issues.map((i) => i.message).join("; ")}`, + ).toBe(true); + } + }); + + it("never drops a parameter it considers applicable", () => { + for (const id of MODEL_IDS) { + const { params, dropped } = dropUnsupported(id, { ...getDefaults(id) }); + for (const path of Object.keys(params)) { + expect(isApplicable(id, path, params), `${id} kept inapplicable ${path}`).toBe(true); + } + for (const d of dropped) { + expect(Object.keys(params)).not.toContain(d.path); + } + } + }); +}); diff --git a/packages/modelparams/tests/resolve.test.ts b/packages/modelparams/tests/resolve.test.ts new file mode 100644 index 0000000..9a4cfd4 --- /dev/null +++ b/packages/modelparams/tests/resolve.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "vitest"; +import { MODEL_IDS, resolveModelId } from "../src/index.js"; + +describe("resolveModelId", () => { + it("passes through a full catalog id", () => { + const result = resolveModelId("openai/gpt-5.5"); + expect(result).toEqual({ ok: true, id: "openai/gpt-5.5" }); + }); + + it("resolves a bare slug published by exactly one provider", () => { + const result = resolveModelId("gpt-5.5"); + expect(result).toEqual({ ok: true, id: "openai/gpt-5.5" }); + }); + + it("resolves a subscription variant", () => { + const result = resolveModelId("claude-opus-4-7-subscription"); + expect(result).toEqual({ ok: true, id: "anthropic/claude-opus-4-7-subscription" }); + }); + + it("trims surrounding whitespace", () => { + expect(resolveModelId(" gpt-5.5 ")).toEqual({ ok: true, id: "openai/gpt-5.5" }); + }); + + it("reports an unknown model with suggestions rather than guessing", () => { + const result = resolveModelId("gpt-5.5-turbo-ultra"); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.reason).toBe("not_found"); + expect(result.suggestions.length).toBeGreaterThan(0); + // Suggestions must be real ids the caller can retry with. + for (const id of result.suggestions) expect(MODEL_IDS).toContain(id); + }); + + it("returns no suggestions for input with nothing in common", () => { + const result = resolveModelId("zzzzzzzzzz"); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.reason).toBe("not_found"); + expect(result.suggestions).toEqual([]); + }); + + it("never silently picks a winner when a bare slug is ambiguous", () => { + // Find a slug that more than one provider publishes, if the catalog has one. + const bySlug = new Map(); + for (const id of MODEL_IDS) { + const slug = id.slice(id.indexOf("/") + 1); + bySlug.set(slug, [...(bySlug.get(slug) ?? []), id]); + } + const ambiguous = [...bySlug.entries()].find(([, ids]) => ids.length > 1); + if (!ambiguous) return; // no collisions in the catalog today + + const [slug, ids] = ambiguous; + const result = resolveModelId(slug); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.reason).toBe("ambiguous"); + expect([...result.matches].sort()).toEqual([...ids].sort()); + }); + + it("resolves every catalog id back to itself", () => { + for (const id of MODEL_IDS) { + expect(resolveModelId(id), id).toEqual({ ok: true, id }); + } + }); +});