diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f4363f5..1e3f4ac 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -180,12 +180,18 @@ The website code lives under `src/`: - `src/build/` — SSG pipeline (renders pages, compiles assets, emits JSON API, generates a social card per page). - `src/server/` — Express dev server. +Everything the catalog ships beyond the static site: + +- `api/` — Vercel Functions, currently `POST /api/v1/validate`. These serve paths the + static build doesn't emit; the rest of `/api/v1/*` stays static JSON from `dist/`. + Conventions: - TypeScript, ES modules, strict mode. - No file over 300 lines, no function over 50 lines. - Format with Prettier, lint with ESLint. `npm run format` and `npm run lint` will set you straight. - Tests live under `tests/` and run with Vitest. +- `npm run typecheck` covers the site and `api/` (via `tsconfig.api.json`). ## Pull requests diff --git a/README.md b/README.md index 016d4c3..a35ae92 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,31 @@ curl https://modelparams.dev/api/v1/params/gpt-5.5.json Schema at `https://modelparams.dev/api/v1/schema.json`, per the [Model Parameters convention](docs/model-parameters-schema.md). +### Validate a request + +POST the parameters you're about to send. You get back what's wrong — including combinations the provider rejects — and a corrected payload. + +```bash +curl -s https://modelparams.dev/api/v1/validate \ + -H 'Content-Type: application/json' \ + -d '{"model":"claude-3-opus-20240229","params":{"temperature":0.5,"top_p":0.9}}' +``` + +```json +{ + "valid": false, + "issues": [ + { + "path": "top_p", + "code": "not_applicable", + "message": "top_p does not apply when temperature ≠ 1", + "conflictsWith": ["temperature"] + } + ], + "safeParams": { "temperature": 0.5 } +} +``` + ## Adding a model Drop a YAML file in `models//`, open a PR, and CI validates it against the schema. Details in [CONTRIBUTING.md](CONTRIBUTING.md). Can't open a PR? [File an issue](https://github.com/mnfst/modelparams.dev/issues/new/choose) with a link to the docs. diff --git a/api/v1/validate.ts b/api/v1/validate.ts new file mode 100644 index 0000000..47beae5 --- /dev/null +++ b/api/v1/validate.ts @@ -0,0 +1,162 @@ +// POST /api/v1/validate — check a params object against a model's catalog entry. +// +// Stateless: the catalog is compiled into the bundle at build time from the same +// generated data the npm package ships, so there is no filesystem read, no +// network call, and nothing to keep in sync at runtime. +import { + dropUnsupported, + getModel, + resolveModelId, + type DroppedParam, +} from "../../packages/modelparams/src/index.js"; + +const MAX_BODY_BYTES = 64 * 1024; + +const CORS = { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type", + "Access-Control-Max-Age": "86400", +} as const; + +function json(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body, null, 2) + "\n", { + status, + headers: { + "Content-Type": "application/json; charset=utf-8", + // A validation result is a pure function of the request body, but bodies + // vary per caller — caching it at the edge would serve one caller's + // verdict to another. + "Cache-Control": "no-store", + ...CORS, + }, + }); +} + +function fail(error: string, detail: Record, status: number): Response { + return json({ error, ...detail }, status); +} + +const USAGE = { + endpoint: "POST /api/v1/validate", + description: + "Validate a params object against a model. Reports unknown parameters, out-of-range values, " + + "and combinations the provider rejects, and returns a corrected payload.", + request: { + model: "provider/model id, or a bare model slug when it is unambiguous", + params: "object of provider-native parameter paths to values", + }, + example: { + model: "anthropic/claude-3-opus-20240229", + params: { temperature: 0.5, top_p: 0.9, max_tokens: 1024 }, + }, + docs: "https://modelparams.dev/api", +} as const; + +function toIssue(dropped: DroppedParam) { + return { + path: dropped.path, + code: dropped.code, + message: dropped.reason, + ...(dropped.conflictsWith ? { conflictsWith: dropped.conflictsWith } : {}), + }; +} + +async function readBody(request: Request): Promise> { + const raw = await request.text(); + if (raw.length > MAX_BODY_BYTES) { + throw new RangeError(`request body exceeds ${MAX_BODY_BYTES} bytes`); + } + if (raw.trim() === "") throw new SyntaxError("request body is empty"); + const parsed: unknown = JSON.parse(raw); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + throw new SyntaxError("request body must be a JSON object"); + } + return parsed as Record; +} + +export default { + async fetch(request: Request): Promise { + if (request.method === "OPTIONS") { + return new Response(null, { status: 204, headers: CORS }); + } + + // A GET here is almost always someone exploring the API by hand or an agent + // probing the URL, so answer with the contract instead of a bare 405. + if (request.method === "GET") return json(USAGE); + + if (request.method !== "POST") { + return fail("method_not_allowed", { allowed: ["POST", "GET", "OPTIONS"], usage: USAGE }, 405); + } + + let body: Record; + try { + body = await readBody(request); + } catch (err) { + const status = err instanceof RangeError ? 413 : 400; + return fail( + "invalid_request_body", + { message: (err as Error).message, usage: USAGE }, + status, + ); + } + + const { model, params } = body; + if (typeof model !== "string" || model.trim() === "") { + return fail( + "invalid_request_body", + { message: "`model` must be a non-empty string", usage: USAGE }, + 400, + ); + } + + const resolved = resolveModelId(model); + if (!resolved.ok) { + return resolved.reason === "ambiguous" + ? fail( + "ambiguous_model", + { + message: `"${model}" is published by more than one provider; qualify it with a provider prefix`, + matches: resolved.matches, + }, + 400, + ) + : fail( + "unknown_model", + { + message: `"${model}" is not in the catalog`, + suggestions: resolved.suggestions, + catalog: "https://modelparams.dev/api/v1/models.json", + }, + 404, + ); + } + + if ( + params !== undefined && + (typeof params !== "object" || params === null || Array.isArray(params)) + ) { + return fail( + "invalid_request_body", + { message: "`params` must be an object", usage: USAGE }, + 400, + ); + } + + const supplied = (params ?? {}) as Record; + const { params: safeParams, dropped } = dropUnsupported(resolved.id, supplied); + const entry = getModel(resolved.id); + + return json({ + model: resolved.id, + provider: entry.provider, + authType: entry.authType, + valid: dropped.length === 0, + issues: dropped.map(toIssue), + // Always a payload that would pass validation — callers that just want to + // make the call work can spread this and move on. + safeParams, + docs: `https://modelparams.dev/models/${resolved.id}`, + }); + }, +}; diff --git a/package.json b/package.json index ba8c062..22a597d 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "dev": "tsx watch src/server/dev.ts", "validate": "tsx src/data/validate.ts", "guard:params": "tsx src/data/check-removals.ts", - "typecheck": "tsc --noEmit", + "typecheck": "tsc --noEmit && tsc -p tsconfig.api.json", "lint": "eslint . --ext .ts,.tsx", "format": "prettier --write .", "test": "vitest run", diff --git a/src/build/build.ts b/src/build/build.ts index ffb4ad5..c984c41 100644 --- a/src/build/build.ts +++ b/src/build/build.ts @@ -156,6 +156,7 @@ async function writeApiIndex(modelCount: number): Promise { modelByIdSubscription: "/api/v1/models/{provider}/{model}-subscription.json", paramsByModelApiKey: "/api/v1/params/{model}.json", paramsByModelSubscription: "/api/v1/params/{model}-subscription.json", + validate: "POST /api/v1/validate", }, modelCount, docs: "https://github.com/mnfst/modelparams.dev#api", diff --git a/src/data/llms.ts b/src/data/llms.ts index 63cb02d..4616760 100644 --- a/src/data/llms.ts +++ b/src/data/llms.ts @@ -63,6 +63,23 @@ function guideApi(siteUrl: string): string[] { `curl ${siteUrl}/api/v1/models/anthropic/claude-opus-4-7-subscription.json`, "```", "", + "## Validate a request before you send it", + "", + "POST a model and a params object; the response reports unknown parameters, values", + "outside their range, and combinations the provider rejects — plus a corrected payload", + "under `safeParams` that is always safe to send as-is:", + "", + "```bash", + `curl -s ${siteUrl}/api/v1/validate \\`, + ` -H 'Content-Type: application/json' \\`, + ` -d '{"model":"claude-3-opus-20240229","params":{"temperature":0.5,"top_p":0.9}}'`, + "```", + "", + "Each issue carries a `code`: `unknown_parameter` (no such knob on this model),", + "`invalid_value` (out of range or not in the enum), or `not_applicable` (the knob", + "exists but conflicts with another value in the same request, listed in", + "`conflictsWith`).", + "", "## JSON Schema", "", "Every entry validates against a JSON Schema you can use in your editor or pipeline:", diff --git a/src/views/api.ejs b/src/views/api.ejs index 047b6ca..4eb8c0a 100644 --- a/src/views/api.ejs +++ b/src/views/api.ejs @@ -70,6 +70,45 @@ + +
+

Validate a request

+

+ POST a model and the parameters you intend to send. The response reports unknown parameters, values outside + their range, and combinations the provider rejects — plus a corrected payload you can send as-is. +

+
+
+ POST +
+ 
curl -s https://modelparams.dev/api/v1/validate \
+  -H 'Content-Type: application/json' \
+  -d '{"model":"claude-3-opus-20240229","params":{"temperature":0.5,"top_p":0.9}}'
+
+
+ 
{
+  "model": "anthropic/claude-3-opus-20240229",
+  "valid": false,
+  "issues": [
+    {
+      "path": "top_p",
+      "code": "not_applicable",
+      "message": "top_p does not apply when temperature ≠ 1",
+      "conflictsWith": ["temperature"]
+    }
+  ],
+  "safeParams": { "temperature": 0.5 }
+}
+
+

+ Every issue carries a + code: + unknown_parameter, + invalid_value, or + not_applicable for a conflict with another value in the same request. +

+
+

JSON Schema

diff --git a/tests/validate-endpoint.test.ts b/tests/validate-endpoint.test.ts new file mode 100644 index 0000000..b7e5539 --- /dev/null +++ b/tests/validate-endpoint.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from "vitest"; +import handler from "../api/v1/validate.js"; + +interface ValidateBody { + model: string; + provider: string; + authType: string; + valid: boolean; + issues: { path: string; code: string; message: string; conflictsWith?: string[] }[]; + safeParams: Record; + error?: string; + suggestions?: string[]; + matches?: string[]; +} + +async function post(body: unknown): Promise<{ status: number; body: ValidateBody }> { + const response = await handler.fetch( + new Request("https://modelparams.dev/api/v1/validate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: typeof body === "string" ? body : JSON.stringify(body), + }), + ); + return { status: response.status, body: (await response.json()) as ValidateBody }; +} + +describe("POST /api/v1/validate", () => { + it("accepts a valid params object", async () => { + const { status, body } = await post({ + model: "anthropic/claude-3-opus-20240229", + params: { max_tokens: 1024, temperature: 1 }, + }); + expect(status).toBe(200); + expect(body.valid).toBe(true); + expect(body.issues).toEqual([]); + expect(body.safeParams).toEqual({ max_tokens: 1024, temperature: 1 }); + }); + + it("reports a conditional conflict and returns a corrected payload", async () => { + const { status, body } = await post({ + model: "anthropic/claude-3-opus-20240229", + params: { temperature: 0.5, top_p: 0.9, max_tokens: 1024 }, + }); + expect(status).toBe(200); + expect(body.valid).toBe(false); + expect(body.issues).toHaveLength(1); + expect(body.issues[0]!.path).toBe("top_p"); + expect(body.issues[0]!.code).toBe("not_applicable"); + expect(body.issues[0]!.conflictsWith).toContain("temperature"); + expect(body.safeParams).toEqual({ temperature: 0.5, max_tokens: 1024 }); + }); + + it("reports an unknown parameter", async () => { + const { body } = await post({ + model: "openai/gpt-5.5", + params: { temperature: 0.7 }, + }); + expect(body.valid).toBe(false); + expect(body.issues[0]!).toMatchObject({ path: "temperature", code: "unknown_parameter" }); + expect(body.safeParams).toEqual({}); + }); + + it("reports an out-of-range value", async () => { + const { body } = await post({ + model: "anthropic/claude-3-opus-20240229", + params: { temperature: 42 }, + }); + expect(body.valid).toBe(false); + expect(body.issues[0]!).toMatchObject({ path: "temperature", code: "invalid_value" }); + }); + + it("resolves a bare model slug", async () => { + const { status, body } = await post({ model: "gpt-5.5", params: {} }); + expect(status).toBe(200); + expect(body.model).toBe("openai/gpt-5.5"); + expect(body.provider).toBe("openai"); + }); + + it("treats a missing params object as an empty request", async () => { + const { status, body } = await post({ model: "openai/gpt-5.5" }); + expect(status).toBe(200); + expect(body.valid).toBe(true); + expect(body.safeParams).toEqual({}); + }); + + it("404s an unknown model with usable suggestions", async () => { + const { status, body } = await post({ model: "gpt-5.5-turbo-ultra", params: {} }); + expect(status).toBe(404); + expect(body.error).toBe("unknown_model"); + // Near-misses, not an empty array: the caller needs something to retry with. + expect(body.suggestions?.length).toBeGreaterThan(0); + expect(body.suggestions).toContain("openai/gpt-5.5"); + }); + + it("400s a malformed body", async () => { + const { status, body } = await post("{not json"); + expect(status).toBe(400); + expect(body.error).toBe("invalid_request_body"); + }); + + it("400s a missing model", async () => { + const { status, body } = await post({ params: { temperature: 1 } }); + expect(status).toBe(400); + expect(body.error).toBe("invalid_request_body"); + }); + + it("400s a non-object params", async () => { + const { status } = await post({ model: "openai/gpt-5.5", params: [1, 2] }); + expect(status).toBe(400); + }); + + it("never caches a verdict at the edge", async () => { + const response = await handler.fetch( + new Request("https://modelparams.dev/api/v1/validate", { + method: "POST", + body: JSON.stringify({ model: "openai/gpt-5.5", params: {} }), + }), + ); + expect(response.headers.get("Cache-Control")).toBe("no-store"); + }); +}); + +describe("other methods on /api/v1/validate", () => { + it("answers GET with the endpoint contract", async () => { + const response = await handler.fetch( + new Request("https://modelparams.dev/api/v1/validate", { method: "GET" }), + ); + expect(response.status).toBe(200); + expect(((await response.json()) as { endpoint: string }).endpoint).toBe( + "POST /api/v1/validate", + ); + }); + + it("answers a CORS preflight", async () => { + const response = await handler.fetch( + new Request("https://modelparams.dev/api/v1/validate", { method: "OPTIONS" }), + ); + expect(response.status).toBe(204); + expect(response.headers.get("Access-Control-Allow-Origin")).toBe("*"); + expect(response.headers.get("Access-Control-Allow-Methods")).toContain("POST"); + }); + + it("405s anything else", async () => { + const response = await handler.fetch( + new Request("https://modelparams.dev/api/v1/validate", { method: "DELETE" }), + ); + expect(response.status).toBe(405); + }); +}); diff --git a/tsconfig.api.json b/tsconfig.api.json new file mode 100644 index 0000000..9679d27 --- /dev/null +++ b/tsconfig.api.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": ".", + "noEmit": true + }, + "include": ["api/**/*.ts", "packages/modelparams/src/**/*.ts"] +} diff --git a/vercel.json b/vercel.json index c22c3bf..653f28c 100644 --- a/vercel.json +++ b/vercel.json @@ -18,6 +18,14 @@ "source": "/api/v1/(.*)", "headers": [{ "key": "X-Robots-Tag", "value": "noindex" }] }, + { + "source": "/api/v1/validate", + "headers": [ + { "key": "Cache-Control", "value": "no-store" }, + { "key": "Access-Control-Allow-Methods", "value": "POST, GET, OPTIONS" }, + { "key": "Access-Control-Allow-Headers", "value": "Content-Type" } + ] + }, { "source": "/(llms.txt|llms-full.txt)", "headers": [