From b93c9d29e63c263c316b022501b52afe9d547ade Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sat, 7 Nov 2026 02:02:04 +0000 Subject: [PATCH] Keep a model's answer when it comes back in the shape of the schema Small models shown a JSON Schema often nest correct values under "properties" next to $schema/type/required, so validation reported every field undefined. Unwrap that and a wrapper named after the object, and stop sending the meta keys that invite the copy. Co-Authored-By: Claude Opus 5.5 --- index.d.ts | 1 + package.json | 2 +- src/json.js | 32 +++++++++++++++++++++++++++- src/router.js | 7 +++--- test/router.test.js | 52 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 5 deletions(-) diff --git a/index.d.ts b/index.d.ts index 6c350a7..2461e7a 100644 --- a/index.d.ts +++ b/index.d.ts @@ -179,6 +179,7 @@ export declare class AnthropicProvider implements TextProvider { export declare function extractJson(text: string): unknown; export declare function validate(schema: SchemaLike | undefined, value: unknown): T; +export declare function unwrapEcho(value: unknown, jsonSchema?: Record, name?: string): unknown; export declare function jsonInstructions(name?: string, jsonSchema?: Record): string; export interface CreateOptions { diff --git a/package.json b/package.json index 7f9fdb6..eae50f1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/ai-providers", - "version": "0.1.0", + "version": "0.1.1", "type": "module", "description": "One text-model call for every Profullstack app: c0mpute first, OpenAI and Anthropic as alternating fallbacks, dead keys parked. Zero dependencies.", "keywords": [ diff --git a/src/json.js b/src/json.js index d3133f1..dbaa48f 100644 --- a/src/json.js +++ b/src/json.js @@ -57,9 +57,39 @@ export function validate(schema, value) { return value; } +const isObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v); + +/** + * Small models shown a JSON Schema often answer in the shape of the schema, with + * correct values one level down: {"$schema":..., "type":"object", "properties": + * {"headline":"Real copy", ...}, "required":[...]}. Nothing is wrong with the + * values, but validating that object reports every field as undefined. Same for + * {"ad_copy":{...}}, where the object is wrapped in its own name. + * + * Only unwraps when the top level has none of the schema's fields and the inner + * object has at least one, so a schema with a field really called `properties` + * is left alone. + */ +export function unwrapEcho(value, jsonSchema, name) { + const keys = Object.keys(jsonSchema?.properties ?? {}); + if (!keys.length || !isObject(value) || keys.some((k) => k in value)) return value; + const entries = Object.values(value); + const candidates = [value.properties, name ? value[name] : undefined, entries.length === 1 ? entries[0] : undefined]; + return candidates.find((c) => isObject(c) && keys.some((k) => k in c)) ?? value; +} + export function jsonInstructions(name, jsonSchema) { + // The meta keys add nothing to the answer and are what start a model copying + // the schema instead of filling it in. + const { $schema, $id, ...shape } = jsonSchema ?? {}; return [ `Return only one JSON object${name ? ` (${name})` : ''}. No Markdown, no code fences, no commentary.`, - ...(jsonSchema ? ['It must match this JSON schema:', JSON.stringify(jsonSchema)] : []), + ...(jsonSchema + ? [ + 'It must match this JSON schema:', + JSON.stringify(shape), + 'The schema describes your answer, it is not part of it. Put each value at the top level of the object, and never write the words "type", "properties" or "required" as keys.', + ] + : []), ].join('\n'); } diff --git a/src/router.js b/src/router.js index dd0c3d9..6bb3738 100644 --- a/src/router.js +++ b/src/router.js @@ -13,7 +13,7 @@ * A success clears it. */ import { ProviderError, toProviderError } from './errors.js'; -import { extractJson, validate, jsonInstructions } from './json.js'; +import { extractJson, validate, jsonInstructions, unwrapEcho } from './json.js'; export class AllProvidersFailedError extends Error { constructor(attempts, skipped = []) { @@ -163,10 +163,11 @@ export class AiRouter { export async function generateObjectOn(provider, req) { const system = [req.system, jsonInstructions(req.name, req.jsonSchema)].filter(Boolean).join('\n\n'); const base = { ...req, system, json: true }; + const read = (text) => validate(req.schema, unwrapEcho(extractJson(text), req.jsonSchema, req.name)); let out = await provider.generate(base); let object; try { - object = validate(req.schema, extractJson(out.text)); + object = read(out.text); } catch (first) { const repairs = req.repairs ?? 1; if (repairs < 1) throw new ProviderError(provider.name, first.message, { kind: 'output' }); @@ -187,7 +188,7 @@ export async function generateObjectOn(provider, req) { ].join('\n'), }); try { - object = validate(req.schema, extractJson(out.text)); + object = read(out.text); } catch (second) { throw new ProviderError(provider.name, second.message, { kind: 'output' }); } diff --git a/test/router.test.js b/test/router.test.js index bf89da0..29b3aba 100644 --- a/test/router.test.js +++ b/test/router.test.js @@ -6,6 +6,8 @@ import { classify, createAiProviders, extractJson, + jsonInstructions, + unwrapEcho, C0mputeProvider, OpenAIProvider, AnthropicProvider, @@ -193,3 +195,53 @@ test('createAiProviders: c0mpute always, paid only with keys, AI_PAID_FALLBACK=o const r = createAiProviders({ env: { C0MPUTE_MODELS: 'a, b' } }); assert.deepEqual(r.primary[0].models, ['a', 'b']); }); + +// The answers below are the shapes huihui_ai/qwen3.5-abliterated:9b really gave +// for crawlproof ad copy, 2026-11-07. +const adSchema = { + $schema: 'https://json-schema.org/draft/2020-12/schema', + type: 'object', + properties: { headline: { type: 'string' }, body: { type: 'string' } }, + required: ['headline', 'body'], + additionalProperties: false, +}; +const needsAd = (v) => { + if (typeof v?.headline !== 'string' || typeof v?.body !== 'string') throw new Error('headline and body required'); + return v; +}; + +test('a model that answers in the shape of the schema keeps its values', async () => { + const echo = JSON.stringify({ ...adSchema, properties: { headline: 'See your site', body: 'Ask 9 models.' } }); + const provider = ok('c0mpute', echo); + const { object } = await new AiRouter({ primary: [provider] }).generateObject({ + name: 'ad_copy', + schema: needsAd, + jsonSchema: adSchema, + user: 'x', + }); + assert.deepEqual(object, { headline: 'See your site', body: 'Ask 9 models.' }); + assert.equal(provider.calls, 1, 'no repair round for an answer that was already right'); +}); + +test('unwrapEcho takes a wrapper named after the object, or a lone wrapper', () => { + const ad = { headline: 'a', body: 'b' }; + assert.deepEqual(unwrapEcho({ ad_copy: ad }, adSchema, 'ad_copy'), ad); + assert.deepEqual(unwrapEcho({ result: ad }, adSchema), ad); +}); + +test('unwrapEcho leaves a real answer alone', () => { + const flat = { $schema: 'x', headline: 'a', body: 'b' }; + assert.equal(unwrapEcho(flat, adSchema), flat); + const own = { type: 'object', properties: { properties: { type: 'object' } } }; + const value = { properties: { properties: { a: 1 } } }; + assert.equal(unwrapEcho(value, own), value, 'a field really called properties'); + const junk = { properties: { other: 1 } }; + assert.equal(unwrapEcho(junk, adSchema), junk); +}); + +test('the prompt leaves out the schema meta keys and says not to echo it', () => { + const text = jsonInstructions('ad_copy', adSchema); + assert.ok(!text.includes('$schema')); + assert.ok(text.includes('"headline"')); + assert.match(text, /not part of it/); +});