Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,7 @@ export declare class AnthropicProvider implements TextProvider {

export declare function extractJson(text: string): unknown;
export declare function validate<T>(schema: SchemaLike<T> | undefined, value: unknown): T;
export declare function unwrapEcho(value: unknown, jsonSchema?: Record<string, unknown>, name?: string): unknown;
export declare function jsonInstructions(name?: string, jsonSchema?: Record<string, unknown>): string;

export interface CreateOptions {
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@profullstack/ai-providers",
"version": "0.1.0",
"version": "0.1.1",
"type": "module",
"description": "One text-model call for every Profullstack app: c0mpute first, OpenAI and Anthropic as alternating fallbacks, dead keys parked. Zero dependencies.",
"keywords": [
Expand Down
32 changes: 31 additions & 1 deletion src/json.js
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,39 @@ export function validate(schema, value) {
return value;
}

const isObject = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);

/**
* Small models shown a JSON Schema often answer in the shape of the schema, with
* correct values one level down: {"$schema":..., "type":"object", "properties":
* {"headline":"Real copy", ...}, "required":[...]}. Nothing is wrong with the
* values, but validating that object reports every field as undefined. Same for
* {"ad_copy":{...}}, where the object is wrapped in its own name.
*
* Only unwraps when the top level has none of the schema's fields and the inner
* object has at least one, so a schema with a field really called `properties`
* is left alone.
*/
export function unwrapEcho(value, jsonSchema, name) {
const keys = Object.keys(jsonSchema?.properties ?? {});
if (!keys.length || !isObject(value) || keys.some((k) => k in value)) return value;
const entries = Object.values(value);
const candidates = [value.properties, name ? value[name] : undefined, entries.length === 1 ? entries[0] : undefined];
return candidates.find((c) => isObject(c) && keys.some((k) => k in c)) ?? value;
}

export function jsonInstructions(name, jsonSchema) {
// The meta keys add nothing to the answer and are what start a model copying
// the schema instead of filling it in.
const { $schema, $id, ...shape } = jsonSchema ?? {};
return [
`Return only one JSON object${name ? ` (${name})` : ''}. No Markdown, no code fences, no commentary.`,
...(jsonSchema ? ['It must match this JSON schema:', JSON.stringify(jsonSchema)] : []),
...(jsonSchema
? [
'It must match this JSON schema:',
JSON.stringify(shape),
'The schema describes your answer, it is not part of it. Put each value at the top level of the object, and never write the words "type", "properties" or "required" as keys.',
]
: []),
].join('\n');
}
7 changes: 4 additions & 3 deletions src/router.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
* A success clears it.
*/
import { ProviderError, toProviderError } from './errors.js';
import { extractJson, validate, jsonInstructions } from './json.js';
import { extractJson, validate, jsonInstructions, unwrapEcho } from './json.js';

export class AllProvidersFailedError extends Error {
constructor(attempts, skipped = []) {
Expand Down Expand Up @@ -163,10 +163,11 @@ export class AiRouter {
export async function generateObjectOn(provider, req) {
const system = [req.system, jsonInstructions(req.name, req.jsonSchema)].filter(Boolean).join('\n\n');
const base = { ...req, system, json: true };
const read = (text) => validate(req.schema, unwrapEcho(extractJson(text), req.jsonSchema, req.name));
let out = await provider.generate(base);
let object;
try {
object = validate(req.schema, extractJson(out.text));
object = read(out.text);
} catch (first) {
const repairs = req.repairs ?? 1;
if (repairs < 1) throw new ProviderError(provider.name, first.message, { kind: 'output' });
Expand All @@ -187,7 +188,7 @@ export async function generateObjectOn(provider, req) {
].join('\n'),
});
try {
object = validate(req.schema, extractJson(out.text));
object = read(out.text);
} catch (second) {
throw new ProviderError(provider.name, second.message, { kind: 'output' });
}
Expand Down
52 changes: 52 additions & 0 deletions test/router.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import {
classify,
createAiProviders,
extractJson,
jsonInstructions,
unwrapEcho,
C0mputeProvider,
OpenAIProvider,
AnthropicProvider,
Expand Down Expand Up @@ -193,3 +195,53 @@ test('createAiProviders: c0mpute always, paid only with keys, AI_PAID_FALLBACK=o
const r = createAiProviders({ env: { C0MPUTE_MODELS: 'a, b' } });
assert.deepEqual(r.primary[0].models, ['a', 'b']);
});

// The answers below are the shapes huihui_ai/qwen3.5-abliterated:9b really gave
// for crawlproof ad copy, 2026-11-07.
const adSchema = {
$schema: 'https://json-schema.org/draft/2020-12/schema',
type: 'object',
properties: { headline: { type: 'string' }, body: { type: 'string' } },
required: ['headline', 'body'],
additionalProperties: false,
};
const needsAd = (v) => {
if (typeof v?.headline !== 'string' || typeof v?.body !== 'string') throw new Error('headline and body required');
return v;
};

test('a model that answers in the shape of the schema keeps its values', async () => {
const echo = JSON.stringify({ ...adSchema, properties: { headline: 'See your site', body: 'Ask 9 models.' } });
const provider = ok('c0mpute', echo);
const { object } = await new AiRouter({ primary: [provider] }).generateObject({
name: 'ad_copy',
schema: needsAd,
jsonSchema: adSchema,
user: 'x',
});
assert.deepEqual(object, { headline: 'See your site', body: 'Ask 9 models.' });
assert.equal(provider.calls, 1, 'no repair round for an answer that was already right');
});

test('unwrapEcho takes a wrapper named after the object, or a lone wrapper', () => {
const ad = { headline: 'a', body: 'b' };
assert.deepEqual(unwrapEcho({ ad_copy: ad }, adSchema, 'ad_copy'), ad);
assert.deepEqual(unwrapEcho({ result: ad }, adSchema), ad);
});

test('unwrapEcho leaves a real answer alone', () => {
const flat = { $schema: 'x', headline: 'a', body: 'b' };
assert.equal(unwrapEcho(flat, adSchema), flat);
const own = { type: 'object', properties: { properties: { type: 'object' } } };
const value = { properties: { properties: { a: 1 } } };
assert.equal(unwrapEcho(value, own), value, 'a field really called properties');
const junk = { properties: { other: 1 } };
assert.equal(unwrapEcho(junk, adSchema), junk);
});

test('the prompt leaves out the schema meta keys and says not to echo it', () => {
const text = jsonInstructions('ad_copy', adSchema);
assert.ok(!text.includes('$schema'));
assert.ok(text.includes('"headline"'));
assert.match(text, /not part of it/);
});
Loading