From 6b00c314c1c53f52f1598a864a4f6054c970ff8c Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 16:03:13 -0700 Subject: [PATCH 1/3] feat: add a keyed decisions API with image assets Callers can ask named choice, noul, score, and box questions and attach PNG or JPEG images. The existing question list and adapter stay in place. --- package.json | 5 + src/adapter.test.ts | 22 +++ src/adapter.ts | 27 +++- src/assets.ts | 223 +++++++++++++++++++++++++++++++ src/client.ts | 77 ++++++++++- src/decisions.test.ts | 270 +++++++++++++++++++++++++++++++++++++ src/decisions.ts | 304 ++++++++++++++++++++++++++++++++++++++++++ src/evaluate.ts | 34 ++++- src/index.ts | 5 + src/schemas.ts | 98 +++++++++++++- 10 files changed, 1052 insertions(+), 13 deletions(-) create mode 100644 src/assets.ts create mode 100644 src/decisions.test.ts create mode 100644 src/decisions.ts diff --git a/package.json b/package.json index d181084..22b619a 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,11 @@ "corbits-system-one-src": "./src/index.ts", "types": "./dist/index.d.ts", "default": "./dist/index.js" + }, + "./decisions": { + "corbits-system-one-src": "./src/decisions.ts", + "types": "./dist/decisions.d.ts", + "default": "./dist/decisions.js" } }, "publishConfig": { diff --git a/src/adapter.test.ts b/src/adapter.test.ts index 41fefdb..c04d8b4 100644 --- a/src/adapter.test.ts +++ b/src/adapter.test.ts @@ -183,4 +183,26 @@ describe("adapter factory", () => { /model must not be empty/, ); }); + + test("providerOptions.systemOne accepts keyed questions", () => { + const adapter = createSystemOneAdapter(); + const request = adapter.buildRequest([], "jev-latest", { + providerOptions: { + systemOne: { + state: "ticket", + questions: { + category: { + type: "choice", + instructions: "What?", + criteria: { a: null }, + }, + }, + }, + }, + }); + const body = JSON.parse(request.body) as { + questions: Record; + }; + expect(body.questions["category"]?.type).toBe("choice"); + }); }); diff --git a/src/adapter.ts b/src/adapter.ts index ea67599..94364b5 100644 --- a/src/adapter.ts +++ b/src/adapter.ts @@ -19,6 +19,7 @@ import type { import { resolveEndpoint } from "./config.js"; import { Decision, + DecisionQuestionRecord, toDecision, toWireQuestions, WireResponseBody, @@ -51,9 +52,11 @@ export const SYSTEM_ONE_PROVIDER = "corbits-system-one"; // `state` replaces the transcript-derived default; `questions` replaces the // single-question default. Both are validated, never cast — `QuestionList` // carries the same non-empty/unique-id contract as `EvaluateInput`. +const QuestionsInput = QuestionList.or(DecisionQuestionRecord); + const SystemOneProviderOptions = type({ "state?": State, - "questions?": QuestionList, + "questions?": QuestionsInput, "+": "reject", }); @@ -85,6 +88,24 @@ const DEFAULT_QUESTIONS: QuestionList = [ }, ]; +function asQuestionList( + questions: QuestionList | DecisionQuestionRecord, +): QuestionList { + if (Array.isArray(questions)) return questions; + const compiled = Object.entries(questions).map(([id, question]) => ({ + id, + ...question, + })); + const parsed = QuestionList(compiled); + if (parsed instanceof type.errors) { + throw new ProtocolMismatchError( + `${SYSTEM_ONE_PROVIDER} buildRequest: invalid providerOptions.systemOne questions: ${parsed.summary}`, + questions, + ); + } + return parsed; +} + function transcriptText(turns: ConversationTurn[]): string { const texts: string[] = []; for (const turn of turns) { @@ -187,7 +208,9 @@ function createSystemOneProviderAdapter( ); } if (overrides.state !== undefined) state = overrides.state; - if (overrides.questions !== undefined) questions = overrides.questions; + if (overrides.questions !== undefined) { + questions = asQuestionList(overrides.questions); + } } const requestModel = model || endpoint.model; const headers: Record = { diff --git a/src/assets.ts b/src/assets.ts new file mode 100644 index 0000000..2298da1 --- /dev/null +++ b/src/assets.ts @@ -0,0 +1,223 @@ +import { SystemOneError } from "./errors.js"; + +const CONTENT_TYPES = [ + ["image/png", [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]], + ["image/jpeg", [0xff, 0xd8, 0xff]], +] as const; + +const REFERENCE = //g; + +const IMAGE = Symbol.for("@typesafe-ai/sdk/image"); +const TEXT_WITH_ASSETS = Symbol.for("@typesafe-ai/sdk/text-with-assets"); + +const randomHex = (): string => + Array.from(crypto.getRandomValues(new Uint8Array(16)), (byte) => + byte.toString(16).padStart(2, "0"), + ).join(""); + +const isImage = (value: object): value is Image => IMAGE in value; + +const isTextWithAssets = (value: object): value is TextWithAssets => + TEXT_WITH_ASSETS in value; + +function fail(message: string): never { + throw new SystemOneError("parse-error", message); +} + +const toBytes = (data: ArrayBuffer | ArrayBufferView): Uint8Array => { + if ( + !ArrayBuffer.isView(data) && + Object.prototype.toString.call(data) !== "[object ArrayBuffer]" + ) { + fail("Image data must be file bytes or a Blob."); + } + return ArrayBuffer.isView(data) + ? new Uint8Array(data.buffer, data.byteOffset, data.byteLength) + : new Uint8Array(data); +}; + +export const contentTypeOf = (bytes: Uint8Array): string => { + for (const [contentType, signature] of CONTENT_TYPES) { + if (signature.every((byte, index) => bytes[index] === byte)) { + return contentType; + } + } + return "application/octet-stream"; +}; + +/** A PNG or JPEG image. Create one with `image` or `Image.fromBase64`. */ +export class Image { + readonly type = "image"; + readonly data: Uint8Array | Blob; + readonly #reference = ``; + + constructor(data: Blob | ArrayBuffer | ArrayBufferView) { + this.data = data instanceof Blob ? data : toBytes(data); + const size = + this.data instanceof Blob ? this.data.size : this.data.byteLength; + if (size === 0) fail("Image data is empty."); + } + + /** Decode base64 file bytes. Do not include a `data:` URL prefix. */ + static fromBase64(text: string): Image { + let binary: string; + try { + binary = atob(text); + } catch (cause) { + const error = new SystemOneError( + "parse-error", + "Image data must contain valid base64 text.", + ); + error.cause = cause; + throw error; + } + return new Image(Uint8Array.from(binary, (char) => char.charCodeAt(0))); + } + + toString(): string { + return this.#reference; + } + + toJSON(): string { + return this.#reference; + } +} + +Object.defineProperty(Image.prototype, IMAGE, { value: true }); + +class TextWithAssets { + readonly text: string; + readonly assets: Image[]; + + constructor(text: string, assets: Image[]) { + this.text = text; + this.assets = assets; + } + + toString(): string { + return this.text; + } + + toJSON(): string { + return this.text; + } +} + +Object.defineProperty(TextWithAssets.prototype, TEXT_WITH_ASSETS, { + value: true, +}); + +export type TextWithAssetsValue = TextWithAssets; + +/** Create an image from file bytes or a Blob. PNG and JPEG only, checked before send. */ +export const image = (data: Blob | ArrayBuffer | ArrayBufferView): Image => + new Image(data); + +const collectAssets = ( + value: unknown, + found: Map, + visited: Set, +): void => { + if (typeof value !== "object" || value === null || visited.has(value)) return; + visited.add(value); + if (isImage(value)) found.set(String(value), value); + else if (isTextWithAssets(value)) { + for (const asset of value.assets) found.set(String(asset), asset); + } else { + for (const item of Object.values(value)) + collectAssets(item, found, visited); + } +}; + +const textWithAssets = (text: string, values: unknown): TextWithAssets => { + const interpolated = new Map(); + collectAssets(values, interpolated, new Set()); + const assets = new Map(); + for (const [reference] of text.matchAll(REFERENCE)) { + const asset = interpolated.get(reference); + if (asset !== undefined) assets.set(reference, asset); + } + return new TextWithAssets(text, [...assets.values()]); +}; + +/** + * Tag a template so interpolated images are sent with the text. + * An image in an untagged template is rejected before the request is sent. + */ +export const withAssets = Object.assign( + (strings: TemplateStringsArray, ...values: unknown[]): TextWithAssets => + textWithAssets( + strings.reduce( + (joined, piece, index) => joined + String(values[index - 1]) + piece, + ), + values, + ), + { + join: (values: readonly unknown[], separator: string): TextWithAssets => + textWithAssets(values.join(separator), values), + }, +); + +export type SerializedRequest = { + json: string; + assets: Record | undefined; +}; + +/** Replace images with references and reject dangling or unused asset names. */ +export function serializeRequest( + payload: unknown, + assets: Record = {}, +): SerializedRequest { + const names = new Map( + Object.entries(assets).map(([name, asset]) => [asset, name]), + ); + const known = new Map( + Object.values(assets).map((asset) => [String(asset), asset]), + ); + const json = JSON.stringify(payload, function (key, value: unknown) { + if (typeof value === "number" && !Number.isFinite(value)) + return String(value); + const holder = this as Record; + const original = holder[key]; + if (typeof original !== "object" || original === null) return value; + if (isImage(original)) { + const name = names.get(original); + if (name !== undefined) return name; + known.set(String(original), original); + } else if (isTextWithAssets(original)) { + let text = original.text; + for (const asset of original.assets) { + const name = names.get(asset); + if (name === undefined) known.set(String(asset), asset); + else text = text.replaceAll(String(asset), name); + } + return text; + } + return value; + }); + const unreferenced = Object.keys(assets).filter( + (name) => !json.includes(JSON.stringify(name).slice(1, -1)), + ); + if (unreferenced.length > 0) { + fail( + `Assets not referenced in the request: ${unreferenced.join(", ")}. Write each name in \`assets\` into state, question instructions, or criteria.`, + ); + } + const sent = new Map(Object.entries(assets)); + const missing = new Set(); + for (const [reference] of json.matchAll(REFERENCE)) { + if (sent.has(reference)) continue; + const asset = known.get(reference); + if (asset === undefined) missing.add(reference); + else sent.set(reference, asset); + } + if (missing.size > 0) { + fail( + `Image references without an attached image: ${[...missing].join(", ")}. Tag template literals that interpolate images with \`withAssets\`.`, + ); + } + return { + json, + assets: sent.size === 0 ? undefined : Object.fromEntries(sent), + }; +} diff --git a/src/client.ts b/src/client.ts index 0d0e1eb..07b0c56 100644 --- a/src/client.ts +++ b/src/client.ts @@ -23,6 +23,11 @@ import { SystemOneError, TransportError } from "./errors.js"; export type PostEvaluateOptions = { timeoutMs?: number; apiKey?: string; + imageParts?: { + name: string; + bytes: Uint8Array; + contentType: "image/png" | "image/jpeg"; + }[]; }; /** Transport dependencies: the `fetch` to POST with and the timeout scheduler. */ @@ -35,6 +40,61 @@ export type PostEvaluateResponse = { httpStatus: number; }; +function encodeMultipart( + payload: unknown, + parts: NonNullable, +): { body: Uint8Array; contentType: string } { + const boundary = Array.from( + crypto.getRandomValues(new Uint8Array(16)), + (byte) => byte.toString(16).padStart(2, "0"), + ).join(""); + const encoder = new TextEncoder(); + const chunks: Uint8Array[] = []; + const fields: { + name: string; + contentType: string; + content: Uint8Array; + filename?: string; + }[] = [ + { + name: "request", + contentType: "application/json", + content: encoder.encode(JSON.stringify(payload)), + }, + ...parts.map((part) => ({ + name: part.name, + contentType: part.contentType, + content: part.bytes, + filename: part.name, + })), + ]; + for (const field of fields) { + const disposition = + field.filename === undefined + ? `name="${field.name}"` + : `name="${field.name}"; filename="${field.filename}"`; + chunks.push( + encoder.encode( + `--${boundary}\r\nContent-Disposition: form-data; ${disposition}\r\nContent-Type: ${field.contentType}\r\n\r\n`, + ), + field.content, + encoder.encode("\r\n"), + ); + } + chunks.push(encoder.encode(`--${boundary}--\r\n`)); + const size = chunks.reduce((total, chunk) => total + chunk.byteLength, 0); + const body = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return { + body, + contentType: `multipart/form-data; boundary=${boundary}`, + }; +} + /** * POSTs an evaluation payload and returns the decoded JSON body. Sends * `Authorization: Bearer ` only when a key is present; aborts the @@ -52,9 +112,17 @@ export async function postEvaluate( options: PostEvaluateOptions, deps: EvaluateDeps, ): Promise { - let serialized: string; + let serialized: string | Uint8Array; + const imageParts = options.imageParts; + let contentType = "application/json"; try { - serialized = JSON.stringify(body); + if (imageParts !== undefined && imageParts.length > 0) { + const encoded = encodeMultipart(body, imageParts); + serialized = encoded.body; + contentType = encoded.contentType; + } else { + serialized = JSON.stringify(body); + } } catch { throw new SystemOneError( "parse-error", @@ -81,9 +149,12 @@ export async function postEvaluate( }; try { const headers: Record = { - "content-type": "application/json", + "content-type": contentType, accept: "application/json", }; + if (imageParts !== undefined && imageParts.length > 0) { + headers["TypeSafe-Danger-Zone-Images"] = "true"; + } if (options.apiKey !== undefined && options.apiKey !== "") { headers["authorization"] = `Bearer ${options.apiKey}`; } diff --git a/src/decisions.test.ts b/src/decisions.test.ts new file mode 100644 index 0000000..a8a537a --- /dev/null +++ b/src/decisions.test.ts @@ -0,0 +1,270 @@ +import type { EvaluateDeps } from "./client"; +import { describe, expect, test } from "bun:test"; +import { createDefaultScheduler } from "@intx/inference"; + +import { Image, image, withAssets } from "./assets"; +import { box, choice, noul, score, systemOne } from "./decisions"; +import { SystemOneError } from "./errors"; + +const png = Uint8Array.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, +]); +const jpeg = Uint8Array.from([0xff, 0xd8, 0xff, 0x00]); +const gif = Uint8Array.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61]); + +function deps(fetchImpl: EvaluateDeps["fetch"]): EvaluateDeps { + return { fetch: fetchImpl, scheduler: createDefaultScheduler() }; +} + +function okBody() { + return { + model: "jev-latest", + answers: { + category: { + type: "choice", + choice: "billing", + probabilities: { billing: 0.8, technical: 0.2 }, + confidence: 0.7, + }, + escalate: { type: "noul", noul: 0.2 }, + severity: { + type: "score", + score: 1, + legend: { "0": "none", "1": "cosmetic", "2": "unusable" }, + probabilities: { "0": 0.2, "1": 0.6, "2": 0.2 }, + confidence: 0.5, + }, + }, + }; +} + +describe("systemOne", () => { + test("returns named answers and falls back without throwing", async () => { + let calls = 0; + const result = await systemOne( + { + state: { document: "charged twice" }, + questions: { + category: choice("What is this?", { billing: null, technical: null }), + escalate: noul("Must this be escalated?"), + severity: score("How severe?", ["none", "cosmetic", "unusable"]), + }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async () => { + calls += 1; + return new Response(JSON.stringify(okBody()), { status: 200 }); + }), + }, + ); + expect(calls).toBe(1); + if (result.fallback) throw new Error("expected answers"); + expect(result.answers.category.choice).toBe("billing"); + expect(result.answers.category.probabilities.billing).toBe(0.8); + expect(result.answers.category.confidence).toBe(0.7); + expect(result.answers.escalate.noul).toBe(0.2); + expect(result.answers.severity.score).toBe(1); + + const fallback = await systemOne( + { + state: { document: "x" }, + questions: { escalate: noul("Must this be escalated?") }, + }, + { + deps: deps(() => Promise.reject(new Error("fetch must not be called"))), + }, + ); + expect(fallback.fallback).toBe(true); + if (!fallback.fallback) throw new Error("expected fallback"); + expect(fallback.reason).toBe("no-key"); + + const http = await systemOne( + { + state: { document: "x" }, + questions: { escalate: noul("Must this be escalated?") }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async () => new Response("no", { status: 500 })), + }, + ); + expect(http.fallback).toBe(true); + if (!http.fallback) throw new Error("expected fallback"); + expect(http.reason).toBe("http-error"); + }); + + test("sends images as multipart and rejects other formats first", async () => { + const bodies: { headers: Headers; text: string }[] = []; + const photo = image(png); + const result = await systemOne( + { + state: { photo }, + questions: { + region: box("The damaged area"), + damaged: noul("Is it damaged?"), + }, + model: "jev-preview", + config: { apiKey: "test-key" }, + }, + { + deps: deps(async (_url, init) => { + const headers = new Headers(init?.headers); + const body = init?.body; + const text = + typeof body === "string" + ? body + : await new Response(body as Blob).text(); + bodies.push({ headers, text }); + return new Response( + JSON.stringify({ + answers: { + region: { + type: "box", + box: { top: 0.1, bottom: 0.4, left: 0.2, right: 0.5 }, + confidence: null, + }, + damaged: { type: "noul", noul: 0.9 }, + }, + }), + { status: 200 }, + ); + }), + }, + ); + if (result.fallback) throw new Error(result.reason); + expect(result.answers.region.box).toEqual({ + top: 0.1, + bottom: 0.4, + left: 0.2, + right: 0.5, + }); + expect(result.answers.region.confidence).toBeNull(); + const sent = bodies[0]; + if (sent === undefined) throw new Error("expected a request"); + expect(sent.headers.get("TypeSafe-Danger-Zone-Images")).toBe("true"); + expect(sent.headers.get("content-type")).toStartWith("multipart/form-data"); + expect(sent.text).toContain('name="request"'); + expect(sent.text).toContain('name="asset-0"'); + expect(sent.text).toContain("jev-preview"); + + let fetched = false; + let thrown: unknown; + try { + await systemOne( + { + state: image(gif), + questions: { damaged: noul("Is it damaged?") }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async () => { + fetched = true; + return new Response("{}", { status: 200 }); + }), + }, + ); + } catch (cause) { + thrown = cause; + } + expect(fetched).toBe(false); + expect(thrown).toBeInstanceOf(SystemOneError); + }); + + test("rejects a plain template image and requires every asset name", async () => { + const photo = image(jpeg); + let fetched = false; + const transport = deps(async () => { + fetched = true; + return new Response("{}", { status: 200 }); + }); + await expect( + systemOne( + { + state: `see ${photo}`, + questions: { damaged: noul("Is it damaged?") }, + config: { apiKey: "test-key" }, + }, + { deps: transport }, + ), + ).rejects.toBeInstanceOf(SystemOneError); + + await expect( + systemOne( + { + state: "no photo here", + questions: { damaged: noul("Is it damaged?") }, + assets: { "": image(png) }, + config: { apiKey: "test-key" }, + }, + { deps: transport }, + ), + ).rejects.toBeInstanceOf(SystemOneError); + expect(fetched).toBe(false); + + const tagged = await systemOne( + { + state: withAssets`compare ${photo}`, + questions: { + same: noul(withAssets`Does ${photo} match?`), + }, + config: { apiKey: "test-key" }, + }, + { + deps: deps( + async () => + new Response( + JSON.stringify({ + answers: { same: { type: "noul", noul: 0.4 } }, + }), + { status: 200 }, + ), + ), + }, + ); + if (tagged.fallback) throw new Error(tagged.reason); + expect(tagged.answers.same.noul).toBe(0.4); + + const joined = withAssets.join([image(png), image(jpeg)], ", "); + expect(String(joined)).toContain(" please", + questions: { damaged: noul("Is damaged?") }, + assets: { + "": Image.fromBase64(Buffer.from(png).toString("base64")), + }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async (_url, init) => { + const text = await new Response(init?.body as Blob).text(); + expect(text).toContain(""); + expect(text).toContain("image/png"); + return new Response( + JSON.stringify({ + answers: { damaged: { type: "noul", noul: 0.1 } }, + }), + { status: 200 }, + ); + }), + }, + ); + if (named.fallback) throw new Error(named.reason); + expect(named.answers.damaged.noul).toBe(0.1); + }); + + test("rejects extraction questions", async () => { + await expect( + systemOne( + { + state: "x", + questions: { field: { type: "extract", instructions: "pull it" } }, + config: { apiKey: "k" }, + }, + { deps: deps(async () => new Response("{}", { status: 200 })) }, + ), + ).rejects.toBeInstanceOf(SystemOneError); + }); +}); diff --git a/src/decisions.ts b/src/decisions.ts new file mode 100644 index 0000000..7470e22 --- /dev/null +++ b/src/decisions.ts @@ -0,0 +1,304 @@ +import { type } from "arktype"; + +import { contentTypeOf, Image, serializeRequest } from "./assets.js"; +import type { EvaluateDeps } from "./client.js"; +import { evaluate, type EvaluateOptions } from "./evaluate.js"; +import { SystemOneError } from "./errors.js"; +import { + DecisionQuestionRecord, + QuestionList, + type Decision, + type Description, + type EvaluateConfig, + type FallbackResult, + type State, +} from "./schemas.js"; + +type ChoiceCriteria = Record; + +type NoulCriteria = { + true?: Description; + false?: Description; +}; + +export function choice( + instructions: unknown, + criteria: Criteria, +): { type: "choice"; instructions: unknown; criteria: Criteria } { + if (Array.isArray(criteria)) { + throw new SystemOneError( + "parse-error", + "Choice criteria must be a map of labels to descriptions, not a list.", + ); + } + return { type: "choice", instructions, criteria }; +} + +export function noul( + instructions: unknown, + criteria?: NoulCriteria, +): { type: "noul"; instructions: unknown; criteria?: NoulCriteria } { + const question: { + type: "noul"; + instructions: unknown; + criteria?: NoulCriteria; + } = { type: "noul", instructions }; + if (criteria !== undefined) question.criteria = criteria; + return question; +} + +export function score( + instructions: unknown, + criteria: Criteria, +): { type: "score"; instructions: unknown; criteria: Criteria } { + if (!Array.isArray(criteria)) { + throw new SystemOneError( + "parse-error", + "Score criteria must be a list of descriptions indexed by score from zero, not a map.", + ); + } + return { type: "score", instructions, criteria }; +} + +/** + * Locate what `instructions` describe. The answer's `box` edges are fractions + * of the image. Box questions need the `jev-preview` model; pass it as + * `model` (or on the endpoint). This client does not reject other models. + */ +export function box(instructions: unknown): { + type: "box"; + instructions: unknown; +} { + return { type: "box", instructions }; +} + +type ChoiceAnswer = { + type: "choice"; + choice: keyof Criteria & string; + probabilities: Record; + confidence: number; +}; + +type NoulAnswer = { type: "noul"; noul: number }; + +type ScoreAnswer = { + type: "score"; + score: number; + legend: Record; + probabilities: Record; + confidence: number; +}; + +type BoxAnswer = { + type: "box"; + box: { top: number; bottom: number; left: number; right: number }; + confidence: number | null; +}; + +type AnswerOf = QuestionSpec extends { + type: "choice"; + criteria: infer Criteria extends ChoiceCriteria; +} + ? ChoiceAnswer + : QuestionSpec extends { type: "noul" } + ? NoulAnswer + : QuestionSpec extends { type: "score" } + ? ScoreAnswer + : QuestionSpec extends { type: "box" } + ? BoxAnswer + : never; + +export type SystemOneAnswers> = { + [Key in keyof Questions]: AnswerOf; +}; + +export type SystemOneSuccess> = { + fallback: false; + answers: SystemOneAnswers; + modelId: string; + backend: "corbits-system-one" | "gateway" | "custom"; + latencyMs: number; + usage?: { inputTokens: number; outputTokens: number }; +}; + +export type SystemOneResult> = + | SystemOneSuccess + | FallbackResult; + +export type SystemOneInput> = { + state: unknown; + questions: Questions; + assets?: Record; + model?: string; + config?: EvaluateConfig; +}; + +function withModel( + config: EvaluateConfig | undefined, + model: string | undefined, +): EvaluateConfig { + const next: EvaluateConfig = {}; + if (config?.apiKey !== undefined) next.apiKey = config.apiKey; + if (config?.timeoutMs !== undefined) next.timeoutMs = config.timeoutMs; + const endpoint = config?.endpoint; + if (model !== undefined) { + next.endpoint = { ...(endpoint ?? { kind: "official" }), model }; + } else if (endpoint !== undefined) { + next.endpoint = endpoint; + } + return next; +} + +function answersFrom(decisions: Decision[]): Record { + const answers: Record = {}; + for (const decision of decisions) { + if (decision.type === "choice") { + answers[decision.id] = { + type: "choice", + choice: decision.choice, + probabilities: decision.probabilities, + confidence: decision.confidence, + }; + } else if (decision.type === "noul") { + answers[decision.id] = { type: "noul", noul: decision.noul }; + } else if (decision.type === "score") { + answers[decision.id] = { + type: "score", + score: decision.score, + legend: decision.legend, + probabilities: decision.probabilities, + confidence: decision.confidence, + }; + } else { + answers[decision.id] = { + type: "box", + box: decision.box, + confidence: decision.confidence, + }; + } + } + return answers; +} + +async function loadImages( + assets: Record, +): Promise> { + const parts: NonNullable["parts"] = []; + const wire: NonNullable["assets"] = {}; + let index = 0; + for (const [alias, asset] of Object.entries(assets)) { + let bytes: Uint8Array; + if (asset.data instanceof Blob) { + bytes = new Uint8Array(await asset.data.arrayBuffer()); + } else { + bytes = asset.data; + } + const contentType = contentTypeOf(bytes); + if (contentType !== "image/png" && contentType !== "image/jpeg") { + throw new SystemOneError( + "parse-error", + `Image "${alias}" must be PNG or JPEG.`, + ); + } + const name = `asset-${index}`; + parts.push({ + name, + bytes, + contentType, + }); + wire[alias] = { type: "image", part: name }; + index += 1; + } + return { assets: wire, parts }; +} + +/** + * Keyed decisions call. `choice`, `noul`, `score`, and `box` compile to the + * question list `evaluate` already sends. Transport failures return + * `fallback` and `reason` and do not throw. + */ +export async function systemOne< + const Questions extends Record, +>( + input: SystemOneInput, + options?: Pick, +): Promise> { + const parsed = DecisionQuestionRecord(input.questions); + if (parsed instanceof type.errors) { + throw new SystemOneError( + "parse-error", + `systemOne: invalid questions: ${parsed.summary}`, + ); + } + const serialized = serializeRequest( + { state: input.state, questions: input.questions }, + input.assets, + ); + let payload: { state?: unknown; questions?: unknown }; + try { + payload = JSON.parse(serialized.json) as { + state?: unknown; + questions?: unknown; + }; + } catch (cause) { + throw new SystemOneError( + "parse-error", + `systemOne: request is not JSON: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + } + const questions = QuestionList( + Object.entries( + payload.questions !== null && + typeof payload.questions === "object" && + !Array.isArray(payload.questions) + ? payload.questions + : {}, + ).map(([id, question]) => + typeof question === "object" && question !== null + ? { id, ...question } + : { id }, + ), + ); + if (questions instanceof type.errors) { + throw new SystemOneError( + "parse-error", + `systemOne: invalid questions: ${questions.summary}`, + ); + } + const state = payload.state; + if (state === undefined) { + throw new SystemOneError("parse-error", "systemOne: state is required"); + } + const images = + serialized.assets === undefined + ? undefined + : await loadImages(serialized.assets); + const config = withModel(input.config, input.model); + const result = await evaluate( + { + state: state as State, + questions, + ...(input.config !== undefined || input.model !== undefined + ? { config } + : {}), + }, + { + ...(options?.deps !== undefined ? { deps: options.deps } : {}), + ...(options?.onTelemetry !== undefined + ? { onTelemetry: options.onTelemetry } + : {}), + ...(images !== undefined ? { images } : {}), + }, + ); + if (result.fallback) return result; + return { + fallback: false, + answers: answersFrom(result.decisions) as SystemOneAnswers, + modelId: result.modelId, + backend: result.backend, + latencyMs: result.latencyMs, + ...(result.usage !== undefined ? { usage: result.usage } : {}), + }; +} + +export type { EvaluateDeps }; diff --git a/src/evaluate.ts b/src/evaluate.ts index a0c096f..fb8b991 100644 --- a/src/evaluate.ts +++ b/src/evaluate.ts @@ -120,7 +120,18 @@ function checkAnswer( if (answer.noul === undefined) return `answer "${id}" is missing noul`; return undefined; } - if (answer.confidence === undefined) { + if (question.type === "box" && answer.type === "box") { + if (answer.box === undefined) return `answer "${id}" is missing box`; + const edges = answer.box; + for (const edge of ["top", "bottom", "left", "right"] as const) { + const value = edges[edge]; + if (!Number.isFinite(value) || value < 0 || value > 1) { + return `answer "${id}" box.${edge} ${String(value)} outside [0, 1]`; + } + } + return undefined; + } + if (answer.confidence == null) { return `answer "${id}" is missing confidence`; } if (question.type === "choice" && answer.type === "choice") { @@ -225,6 +236,15 @@ function envKey(name: string): string | undefined { export type EvaluateOptions = { onTelemetry?: (event: SystemOneTelemetryEvent) => void; deps?: EvaluateDeps; + /** Image parts already checked as PNG or JPEG. Omitted for JSON requests. */ + images?: { + assets: Record; + parts: { + name: string; + bytes: Uint8Array; + contentType: "image/png" | "image/jpeg"; + }[]; + }; }; /** @@ -338,10 +358,14 @@ export async function evaluate( }); } + const images = options?.images; const wire = WireRequest({ model, state: parsed.state, questions: toWireQuestions(parsed.questions), + ...(images !== undefined && images.parts.length > 0 + ? { assets: images.assets } + : {}), }); if (wire instanceof type.errors) { throw new SystemOneError( @@ -356,7 +380,13 @@ export async function evaluate( const response = await postEvaluate( endpoint.url, wire, - { timeoutMs, apiKey }, + { + timeoutMs, + apiKey, + ...(images !== undefined && images.parts.length > 0 + ? { imageParts: images.parts } + : {}), + }, deps, ); data = response.data; diff --git a/src/index.ts b/src/index.ts index 7d7b4a0..9196de8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,9 @@ export { DEFAULT_TIMEOUT_MS } from "./config.js"; export type { EvaluateDeps } from "./client.js"; export { evaluate } from "./evaluate.js"; export type { EvaluateOptions } from "./evaluate.js"; +export { choice, noul, score, box, systemOne } from "./decisions.js"; +export type { SystemOneResult, SystemOneSuccess } from "./decisions.js"; +export { Image, image, withAssets } from "./assets.js"; export { createSystemOneAdapter, createSystemOneAdapterFactory, @@ -14,6 +17,8 @@ export { BooleanQuestion, ChoiceQuestion, Decision, + DecisionQuestion, + DecisionQuestionRecord, Description, EndpointConfig, EvaluateConfig, diff --git a/src/schemas.ts b/src/schemas.ts index a9ab389..5e65322 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -114,11 +114,60 @@ export const BooleanQuestion = type({ }).narrow((q) => nonEmpty(q.instructions)); export type BooleanQuestion = typeof BooleanQuestion.infer; +// Locates a region in an image. Edges come back as fractions of the image. +// Callers should set the model to `jev-preview`; the client does not require it. +export const BoxQuestion = type({ + id: "string", + type: "'box'", + instructions: Description, + "+": "reject", +}).narrow((q) => nonEmpty(q.instructions)); +export type BoxQuestion = typeof BoxQuestion.infer; + +// Keyed decisions questions: choice, noul, score, or box only. Extraction-style +// helpers are not part of this contract and fail the union. +export const DecisionQuestion = type({ + type: "'choice'", + instructions: "unknown", + criteria: "unknown", + "+": "reject", +}) + .or( + type({ + type: "'noul'", + "instructions?": "unknown", + "criteria?": "unknown", + "+": "reject", + }), + ) + .or( + type({ + type: "'score'", + instructions: "unknown", + criteria: "unknown", + "+": "reject", + }), + ) + .or( + type({ + type: "'box'", + instructions: "unknown", + "+": "reject", + }), + ); +export type DecisionQuestion = typeof DecisionQuestion.infer; + +export const DecisionQuestionRecord = type({ + "[string]": DecisionQuestion, +}).narrow((qs) => Object.keys(qs).length >= 1); +export type DecisionQuestionRecord = typeof DecisionQuestionRecord.infer; + // The question union a caller may ask in one evaluation. `.or` chains keep // each branch's literal `type`, so the union still discriminates on it. export const Question = ChoiceQuestion.or(ScoreQuestion) .or(NoulQuestion) - .or(BooleanQuestion); + .or(BooleanQuestion) + .or(BoxQuestion); export type Question = typeof Question.infer; // A non-empty question list with unique ids: a duplicate would make the @@ -186,18 +235,25 @@ export type EvaluateInput = typeof EvaluateInput.infer; /** One question in wire form: no id (the map key carries it). */ export const WireQuestion = type({ - type: "'noul' | 'choice' | 'score'", + type: "'noul' | 'choice' | 'score' | 'box'", instructions: Description, "criteria?": "unknown", "+": "reject", }); export type WireQuestion = typeof WireQuestion.infer; -/** The exact POST body: model + state + id-keyed questions. */ +const WireAsset = type({ + type: "'image'", + part: "string", + "+": "reject", +}); + +/** The exact POST body: model + state + id-keyed questions, plus image parts. */ export const WireRequest = type({ model: "string", state: State, questions: { "[string]": WireQuestion }, + "assets?": { "[string]": WireAsset }, "+": "reject", }).narrow((body) => Object.keys(body.questions).length >= 1); export type WireRequest = typeof WireRequest.infer; @@ -215,7 +271,9 @@ export function toWireQuestions( type: question.type === "boolean" ? "noul" : question.type, instructions: question.instructions, }; - if (question.criteria !== undefined) mapped.criteria = question.criteria; + if ("criteria" in question && question.criteria !== undefined) { + mapped.criteria = question.criteria; + } wire[question.id] = mapped; } return wire; @@ -247,13 +305,19 @@ export const APPROXIMATE_SUM_TOLERANCE = 0.02; // `type` must carry. No `"+": "reject"`: the backend may attach per-answer // metadata that `toDecision` strips. export const WireAnswer = type({ - type: "'noul' | 'choice' | 'score'", + type: "'noul' | 'choice' | 'score' | 'box'", "choice?": "string", "score?": "number", "noul?": Confidence, "probabilities?": ProbabilityMap, - "confidence?": Confidence, + "confidence?": Confidence.or("null"), "legend?": { "[string]": "string" }, + "box?": { + top: "number", + bottom: "number", + left: "number", + right: "number", + }, }); export type WireAnswer = typeof WireAnswer.infer; @@ -305,6 +369,21 @@ export const Decision = type({ confidence: Confidence, "+": "reject", }), + ) + .or( + type({ + id: "string", + type: "'box'", + box: { + top: "number", + bottom: "number", + left: "number", + right: "number", + "+": "reject", + }, + confidence: Confidence.or("null"), + "+": "reject", + }), ); export type Decision = typeof Decision.infer; @@ -326,6 +405,13 @@ export function toDecision(id: string, answer: WireAnswer): Decision { probabilities: answer.probabilities, confidence: answer.confidence, }; + } else if (answer.type === "box") { + candidate = { + id, + type: "box", + box: answer.box, + confidence: answer.confidence ?? null, + }; } else { candidate = { id, From 1dbc258bd022b1ed92120909713f0524ccd49387 Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 16:28:44 -0700 Subject: [PATCH 2/3] fix: match image asset names as whole tokens Asset names no longer match inside JSON keys, box edges are checked on the adapter path, and the keyed API is documented. --- README.md | 53 +++++++++++++--- src/adapter.test.ts | 17 +++++ src/adapter.ts | 8 +++ src/assets.ts | 39 +++++++++++- src/decisions.test.ts | 143 ++++++++++++++++++++++++++++++++++++------ src/evaluate.ts | 10 +-- src/schemas.ts | 14 +++++ 7 files changed, 250 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index e1090c0..e5410a9 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,10 @@ Boolean questions answer as `noul`, a probability from 0 to 1 that the answer is | Export | Description | | --------------------------------- | ----------------------------------------------------------------------------------- | | `evaluate(input, options?)` | Runs one evaluation. Returns `EvaluateResult`: decisions or a `FallbackResult`. | +| `systemOne(input, options?)` | Keyed decisions. Returns `answers` by question name, or the same fallback. | +| `choice`, `noul`, `score`, `box` | Build one keyed question. `box` locates a region and needs model `jev-preview`. | +| `image`, `Image.fromBase64` | PNG or JPEG bytes, a `Blob`, or base64. Other formats are rejected before send. | +| `withAssets` | Tag a template so interpolated images are sent. Also `withAssets.join`. | | `createSystemOneAdapter(config?)` | Returns a `ProviderAdapter` for `runInference`. `config` is an `EvaluateConfig`. | | `createSystemOneAdapterFactory` | Interchange `AdapterFactory` for `SIDECAR_ADAPTER_MANIFEST`; takes offering quirks. | | `SYSTEM_ONE_PROVIDER` | The `"corbits-system-one"` provider id. | @@ -82,14 +86,49 @@ Boolean questions answer as `noul`, a probability from 0 to 1 that the answer is ### Questions -| `type` | Fields | Decision fields | -| --------- | ------------------------------------------------------------------------------- | ------------------------------------------------ | -| `choice` | `instructions`, `criteria` (option name → description or `null`, 1–255 options) | `choice`, `probabilities`, `confidence` | -| `score` | `instructions`, `criteria` (2–10 descriptions) | `score`, `legend`, `probabilities`, `confidence` | -| `boolean` | `instructions`, optional `criteria: { true?, false? }` | `noul` | -| `noul` | Same as `boolean` | `noul` | +| `type` | Fields | Decision fields | +| --------- | ------------------------------------------------------------------------------- | ------------------------------------------------------------------- | +| `choice` | `instructions`, `criteria` (option name → description or `null`, 1–255 options) | `choice`, `probabilities`, `confidence` | +| `score` | `instructions`, `criteria` (2–10 descriptions) | `score`, `legend`, `probabilities`, `confidence` | +| `boolean` | `instructions`, optional `criteria: { true?, false? }` | `noul` | +| `noul` | Same as `boolean` | `noul` | +| `box` | `instructions` describing what to locate. Use model `jev-preview`. | `box` (`top`, `bottom`, `left`, `right` as fractions), `confidence` | -Every question has an `id`, unique within the list. +Every list question has an `id`, unique within the list. `box` edges outside `[0, 1]` are rejected. + +### Keyed questions + +`systemOne` takes the same state and a record of questions built with `choice`, `noul`, `score`, and `box`. The names are the answer keys. Those helpers compile to the list `evaluate` already sends. The same fallback reasons apply, and those failures do not throw. The helpers are also exported from `@corbits/system-one/decisions`. + +```ts +import { choice, noul, score, systemOne } from "@corbits/system-one"; + +const result = await systemOne({ + state: { document: "I was charged twice." }, + questions: { + category: choice("What is this?", { billing: null, technical: null }), + escalate: noul("Must this be escalated?"), + severity: score("How severe?", ["none", "cosmetic", "unusable"]), + }, +}); +``` + +On success, read `result.answers.category.choice` (plus `probabilities` and `confidence`), `result.answers.escalate.noul`, and `result.answers.severity.score`. + +`image()` accepts a `Blob` or `Uint8Array`. `Image.fromBase64()` accepts base64 file bytes. Only PNG and JPEG are sent. Place an image in `state`, instructions, or criteria, or interpolate it with `withAssets` (and `withAssets.join` for a list). An image in an ordinary template string is rejected before the request. `assets` maps a name to an image for plain strings; every name must appear as its own token in the request. A request with any image is `multipart/form-data` and sends `TypeSafe-Danger-Zone-Images: true`. Otherwise the body stays JSON. + +```ts +import { box, image, systemOne, withAssets } from "@corbits/system-one"; + +const photo = image(pngBytes); +const result = await systemOne({ + state: withAssets`Returned item: ${photo}`, + questions: { damage: box("The damaged area") }, + model: "jev-preview", +}); +``` + +`providerOptions.systemOne.questions` accepts this keyed record as well as a question list. ### Config diff --git a/src/adapter.test.ts b/src/adapter.test.ts index c04d8b4..4026091 100644 --- a/src/adapter.test.ts +++ b/src/adapter.test.ts @@ -205,4 +205,21 @@ describe("adapter factory", () => { }; expect(body.questions["category"]?.type).toBe("choice"); }); + + test("parseJSONResponse rejects a box outside the image", () => { + const adapter = createSystemOneAdapter(); + expect(() => + adapter.parseJSONResponse( + JSON.stringify({ + answers: { + region: { + type: "box", + box: { top: -0.1, bottom: 0.4, left: 0.2, right: 0.5 }, + confidence: null, + }, + }, + }), + ), + ).toThrow(/outside \[0, 1\]/); + }); }); diff --git a/src/adapter.ts b/src/adapter.ts index 94364b5..1f2fba3 100644 --- a/src/adapter.ts +++ b/src/adapter.ts @@ -23,6 +23,7 @@ import { toDecision, toWireQuestions, WireResponseBody, + boxEdgesOutOfRange, EndpointConfig, QuestionList, State, @@ -163,6 +164,13 @@ function parseDecisions( ); } try { + if (answer.type === "box") { + if (answer.box === undefined) { + throw new Error(`answer "${id}" is missing box`); + } + const violation = boxEdgesOutOfRange(id, answer.box); + if (violation !== undefined) throw new Error(violation); + } return toDecision(id, answer); } catch (cause) { throw new ProtocolMismatchError( diff --git a/src/assets.ts b/src/assets.ts index 2298da1..0b7bd65 100644 --- a/src/assets.ts +++ b/src/assets.ts @@ -196,7 +196,7 @@ export function serializeRequest( return value; }); const unreferenced = Object.keys(assets).filter( - (name) => !json.includes(JSON.stringify(name).slice(1, -1)), + (name) => !jsonValueHasToken(json, name), ); if (unreferenced.length > 0) { fail( @@ -221,3 +221,40 @@ export function serializeRequest( assets: sent.size === 0 ? undefined : Object.fromEntries(sent), }; } + +const IDENT = /[0-9A-Za-z_]/; + +/** True when `name` is its own token in a JSON string value, not a key or a substring. */ +function jsonValueHasToken(json: string, name: string): boolean { + let parsed: unknown; + try { + parsed = JSON.parse(json); + } catch { + return false; + } + return valueHasToken(parsed, name); +} + +function valueHasToken(value: unknown, name: string): boolean { + if (typeof value === "string") return hasToken(value, name); + if (Array.isArray(value)) + return value.some((item) => valueHasToken(item, name)); + if (typeof value === "object" && value !== null) { + return Object.values(value).some((item) => valueHasToken(item, name)); + } + return false; +} + +function hasToken(text: string, name: string): boolean { + if (name.length === 0) return false; + let from = 0; + while (from <= text.length) { + const at = text.indexOf(name, from); + if (at < 0) return false; + const before = at === 0 ? "" : (text[at - 1] ?? ""); + const after = text[at + name.length] ?? ""; + if (!IDENT.test(before) && !IDENT.test(after)) return true; + from = at + 1; + } + return false; +} diff --git a/src/decisions.test.ts b/src/decisions.test.ts index a8a537a..022b498 100644 --- a/src/decisions.test.ts +++ b/src/decisions.test.ts @@ -95,7 +95,7 @@ describe("systemOne", () => { }); test("sends images as multipart and rejects other formats first", async () => { - const bodies: { headers: Headers; text: string }[] = []; + const bodies: { headers: Headers; text: string; raw: Uint8Array }[] = []; const photo = image(png); const result = await systemOne( { @@ -110,12 +110,9 @@ describe("systemOne", () => { { deps: deps(async (_url, init) => { const headers = new Headers(init?.headers); - const body = init?.body; - const text = - typeof body === "string" - ? body - : await new Response(body as Blob).text(); - bodies.push({ headers, text }); + const raw = await bodyBytes(init?.body); + const text = new TextDecoder().decode(raw); + bodies.push({ headers, text, raw }); return new Response( JSON.stringify({ answers: { @@ -144,8 +141,12 @@ describe("systemOne", () => { if (sent === undefined) throw new Error("expected a request"); expect(sent.headers.get("TypeSafe-Danger-Zone-Images")).toBe("true"); expect(sent.headers.get("content-type")).toStartWith("multipart/form-data"); - expect(sent.text).toContain('name="request"'); - expect(sent.text).toContain('name="asset-0"'); + const requestPart = multipartRequest(sent.headers, sent.raw); + const assetKey = Object.keys(requestPart.assets)[0]; + if (assetKey === undefined) throw new Error("expected an asset"); + expect(requestPart.assets[assetKey]?.part).toBe("asset-0"); + expect(requestPart.parts["asset-0"]).toEqual(png); + expect(requestPart.jsonBytes.includes(png[0] ?? 0)).toBe(false); expect(sent.text).toContain("jev-preview"); let fetched = false; @@ -202,6 +203,19 @@ describe("systemOne", () => { ).rejects.toBeInstanceOf(SystemOneError); expect(fetched).toBe(false); + await expect( + systemOne( + { + state: { document: "state type" }, + questions: { damaged: noul("Is it damaged?") }, + assets: { t: image(png) }, + config: { apiKey: "test-key" }, + }, + { deps: transport }, + ), + ).rejects.toBeInstanceOf(SystemOneError); + + const taggedBodies: { headers: Headers; raw: Uint8Array }[] = []; const tagged = await systemOne( { state: withAssets`compare ${photo}`, @@ -211,19 +225,30 @@ describe("systemOne", () => { config: { apiKey: "test-key" }, }, { - deps: deps( - async () => - new Response( - JSON.stringify({ - answers: { same: { type: "noul", noul: 0.4 } }, - }), - { status: 200 }, - ), - ), + deps: deps(async (_url, init) => { + taggedBodies.push({ + headers: new Headers(init?.headers), + raw: await bodyBytes(init?.body), + }); + return new Response( + JSON.stringify({ + answers: { same: { type: "noul", noul: 0.4 } }, + }), + { status: 200 }, + ); + }), }, ); if (tagged.fallback) throw new Error(tagged.reason); expect(tagged.answers.same.noul).toBe(0.4); + const taggedSent = taggedBodies[0]; + if (taggedSent === undefined) throw new Error("expected a request"); + const taggedRequest = multipartRequest(taggedSent.headers, taggedSent.raw); + const taggedKey = Object.keys(taggedRequest.assets)[0]; + if (taggedKey === undefined) throw new Error("expected an asset"); + expect(taggedRequest.assets[taggedKey]?.part).toBe("asset-0"); + expect(taggedRequest.parts["asset-0"]).toEqual(jpeg); + expect(taggedRequest.jsonBytes.includes(jpeg[0] ?? 0)).toBe(false); const joined = withAssets.join([image(png), image(jpeg)], ", "); expect(String(joined)).toContain(" { ).rejects.toBeInstanceOf(SystemOneError); }); }); + +const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47]; + +async function bodyBytes(body: unknown): Promise { + if (body instanceof Uint8Array) return new Uint8Array(body); + if (typeof body === "string") return new TextEncoder().encode(body); + return new Uint8Array(await new Response(body as Blob).arrayBuffer()); +} + +function multipartRequest( + headers: Headers, + raw: Uint8Array, +): { + assets: Record; + parts: Record; + jsonBytes: Uint8Array; +} { + const contentType = headers.get("content-type") ?? ""; + const boundary = /boundary=([^;]+)/.exec(contentType)?.[1]; + if (boundary === undefined) throw new Error("missing multipart boundary"); + const marker = new TextEncoder().encode(`--${boundary}`); + const chunks: Uint8Array[] = []; + let from = 0; + while (from < raw.length) { + const at = indexOfBytes(raw, marker, from); + if (at < 0) break; + if (at > from) chunks.push(raw.subarray(from, at)); + from = at + marker.length; + } + const parts: Record = {}; + let requestJson = ""; + let jsonBytes = new Uint8Array(); + for (const chunk of chunks) { + const headerEnd = indexOfBytes( + chunk, + new TextEncoder().encode("\r\n\r\n"), + 0, + ); + if (headerEnd < 0) continue; + const header = new TextDecoder().decode(chunk.subarray(0, headerEnd)); + const name = /name="([^"]+)"/.exec(header)?.[1]; + if (name === undefined) continue; + let content = chunk.subarray(headerEnd + 4); + if ( + content.length >= 2 && + content[content.length - 2] === 13 && + content[content.length - 1] === 10 + ) { + content = content.subarray(0, content.length - 2); + } + if (name === "request") { + jsonBytes = new Uint8Array(content); + requestJson = new TextDecoder().decode(content); + } else { + parts[name] = new Uint8Array(content); + } + } + const payload = JSON.parse(requestJson) as { + assets?: Record; + }; + const pngAt = indexOfBytes(jsonBytes, Uint8Array.from(PNG_SIGNATURE), 0); + if (pngAt >= 0) throw new Error("PNG bytes leaked into the JSON part"); + return { assets: payload.assets ?? {}, parts, jsonBytes }; +} + +function indexOfBytes( + haystack: Uint8Array, + needle: Uint8Array, + from: number, +): number { + for (let index = from; index <= haystack.length - needle.length; index += 1) { + let match = true; + for (let offset = 0; offset < needle.length; offset += 1) { + if (haystack[index + offset] !== needle[offset]) { + match = false; + break; + } + } + if (match) return index; + } + return -1; +} diff --git a/src/evaluate.ts b/src/evaluate.ts index fb8b991..3169f5a 100644 --- a/src/evaluate.ts +++ b/src/evaluate.ts @@ -23,6 +23,7 @@ import { toWireQuestions, WireRequest, WireResponseBody, + boxEdgesOutOfRange, type Decision, type EvaluateResult, type FallbackReason, @@ -122,14 +123,7 @@ function checkAnswer( } if (question.type === "box" && answer.type === "box") { if (answer.box === undefined) return `answer "${id}" is missing box`; - const edges = answer.box; - for (const edge of ["top", "bottom", "left", "right"] as const) { - const value = edges[edge]; - if (!Number.isFinite(value) || value < 0 || value > 1) { - return `answer "${id}" box.${edge} ${String(value)} outside [0, 1]`; - } - } - return undefined; + return boxEdgesOutOfRange(id, answer.box); } if (answer.confidence == null) { return `answer "${id}" is missing confidence`; diff --git a/src/schemas.ts b/src/schemas.ts index 5e65322..6ed1d42 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -321,6 +321,20 @@ export const WireAnswer = type({ }); export type WireAnswer = typeof WireAnswer.infer; +/** Box edges are fractions of the image. Shared by `evaluate` and the adapter. */ +export function boxEdgesOutOfRange( + id: string, + box: { top: number; bottom: number; left: number; right: number }, +): string | undefined { + for (const edge of ["top", "bottom", "left", "right"] as const) { + const value = box[edge]; + if (!Number.isFinite(value) || value < 0 || value > 1) { + return `answer "${id}" box.${edge} ${String(value)} outside [0, 1]`; + } + } + return undefined; +} + // Token usage reported on the response envelope. export const WireUsage = type({ input_tokens: "number", From 946c2d700066d5bf267ce91b7d2ca187026bcefe Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 26 Oct 2026 20:48:18 -0700 Subject: [PATCH 3/3] fix: scope asset names, reject inverted boxes, and open the image gate --- src/adapter.test.ts | 29 +++++++ src/assets.ts | 35 ++++++-- src/client.ts | 17 +++- src/decisions.test.ts | 194 ++++++++++++++++++++++++++++++++++++++++++ src/evaluate.test.ts | 72 ++++++++++++++++ src/schemas.ts | 6 ++ 6 files changed, 346 insertions(+), 7 deletions(-) diff --git a/src/adapter.test.ts b/src/adapter.test.ts index 4026091..e4ab9d0 100644 --- a/src/adapter.test.ts +++ b/src/adapter.test.ts @@ -222,4 +222,33 @@ describe("adapter factory", () => { ), ).toThrow(/outside \[0, 1\]/); }); + + test("parseJSONResponse rejects an inverted box", () => { + const adapter = createSystemOneAdapter(); + const reject = (box: { + top: number; + bottom: number; + left: number; + right: number; + }) => + adapter.parseJSONResponse( + JSON.stringify({ + answers: { + region: { type: "box", box, confidence: null }, + }, + }), + ); + expect(() => + reject({ top: 0.8, bottom: 0.1, left: 0.2, right: 0.5 }), + ).toThrow(/not below/); + expect(() => + reject({ top: 0.2, bottom: 0.2, left: 0.1, right: 0.4 }), + ).toThrow(/not below/); + expect(() => + reject({ top: 0.1, bottom: 0.4, left: 0.9, right: 0.2 }), + ).toThrow(/not left of/); + expect(() => + reject({ top: 0.1, bottom: 0.4, left: 0.5, right: 0.5 }), + ).toThrow(/not left of/); + }); }); diff --git a/src/assets.ts b/src/assets.ts index 0b7bd65..0cf6a9d 100644 --- a/src/assets.ts +++ b/src/assets.ts @@ -224,7 +224,7 @@ export function serializeRequest( const IDENT = /[0-9A-Za-z_]/; -/** True when `name` is its own token in a JSON string value, not a key or a substring. */ +/** True when `name` is its own token in state, instructions, or criteria. */ function jsonValueHasToken(json: string, name: string): boolean { let parsed: unknown; try { @@ -232,7 +232,26 @@ function jsonValueHasToken(json: string, name: string): boolean { } catch { return false; } - return valueHasToken(parsed, name); + return requestReferencesName(parsed, name); +} + +function own(value: object, key: string): unknown { + if (!Object.hasOwn(value, key)) return undefined; + return Object.getOwnPropertyDescriptor(value, key)?.value; +} + +/** Question `type` and other fields are not references. Keys are not either. */ +function requestReferencesName(payload: unknown, name: string): boolean { + if (typeof payload !== "object" || payload === null) return false; + if (valueHasToken(own(payload, "state"), name)) return true; + const questions = own(payload, "questions"); + if (typeof questions !== "object" || questions === null) return false; + for (const question of Object.values(questions)) { + if (typeof question !== "object" || question === null) continue; + if (valueHasToken(own(question, "instructions"), name)) return true; + if (valueHasToken(own(question, "criteria"), name)) return true; + } + return false; } function valueHasToken(value: unknown, name: string): boolean { @@ -247,12 +266,16 @@ function valueHasToken(value: unknown, name: string): boolean { function hasToken(text: string, name: string): boolean { if (name.length === 0) return false; + // `` is one generated reference, not a use of an asset named `image`. + const visible = text.replace(//g, (reference) => + " ".repeat(reference.length), + ); let from = 0; - while (from <= text.length) { - const at = text.indexOf(name, from); + while (from <= visible.length) { + const at = visible.indexOf(name, from); if (at < 0) return false; - const before = at === 0 ? "" : (text[at - 1] ?? ""); - const after = text[at + name.length] ?? ""; + const before = at === 0 ? "" : (visible[at - 1] ?? ""); + const after = visible[at + name.length] ?? ""; if (!IDENT.test(before) && !IDENT.test(after)) return true; from = at + 1; } diff --git a/src/client.ts b/src/client.ts index 07b0c56..6b834bf 100644 --- a/src/client.ts +++ b/src/client.ts @@ -40,6 +40,18 @@ export type PostEvaluateResponse = { httpStatus: number; }; +function bodyHasBoxQuestion(body: unknown): boolean { + if (typeof body !== "object" || body === null) return false; + if (!Object.hasOwn(body, "questions")) return false; + const questions = Object.getOwnPropertyDescriptor(body, "questions")?.value; + if (typeof questions !== "object" || questions === null) return false; + return Object.values(questions).some((question) => { + if (typeof question !== "object" || question === null) return false; + if (!Object.hasOwn(question, "type")) return false; + return Object.getOwnPropertyDescriptor(question, "type")?.value === "box"; + }); +} + function encodeMultipart( payload: unknown, parts: NonNullable, @@ -152,7 +164,10 @@ export async function postEvaluate( "content-type": contentType, accept: "application/json", }; - if (imageParts !== undefined && imageParts.length > 0) { + if ( + (imageParts !== undefined && imageParts.length > 0) || + bodyHasBoxQuestion(body) + ) { headers["TypeSafe-Danger-Zone-Images"] = "true"; } if (options.apiKey !== undefined && options.apiKey !== "") { diff --git a/src/decisions.test.ts b/src/decisions.test.ts index 022b498..b7d7a4f 100644 --- a/src/decisions.test.ts +++ b/src/decisions.test.ts @@ -292,6 +292,200 @@ describe("systemOne", () => { ), ).rejects.toBeInstanceOf(SystemOneError); }); + + test("asset names match only state, instructions, and criteria", async () => { + let fetched = false; + const transport = deps(async () => { + fetched = true; + return new Response("{}", { status: 200 }); + }); + const unreferenced: { + state: unknown; + questions: Record; + assets: Record; + }[] = [ + { + state: "plain", + questions: { q: choice("look", { a: null }) }, + assets: { choice: image(png) }, + }, + { + state: "plain", + questions: { q: noul("look") }, + assets: { noul: image(png) }, + }, + { + state: "plain", + questions: { q: score("look", ["none", "some"]) }, + assets: { score: image(png) }, + }, + { + state: "plain", + questions: { q: box("the region") }, + assets: { box: image(png) }, + }, + { + state: { shot: image(png) }, + questions: { q: noul("look") }, + assets: { image: image(jpeg) }, + }, + { + state: { photo: "x" }, + questions: { q: noul("look") }, + assets: { photo: image(png) }, + }, + { + state: { "a3f1c2e8-found": "x" }, + questions: { q: noul("look") }, + assets: { found: image(png) }, + }, + { + state: "photograph", + questions: { q: noul("look") }, + assets: { photo: image(png) }, + }, + { + state: "plain", + questions: { q: choice("look", { photo: null }) }, + assets: { photo: image(png) }, + }, + ]; + for (const input of unreferenced) { + await expect( + systemOne( + { ...input, config: { apiKey: "test-key" } }, + { deps: transport }, + ), + ).rejects.toThrow(/not referenced/); + } + expect(fetched).toBe(false); + + const photo = image(png); + const named = await systemOne( + { + state: withAssets`see image ${photo}`, + questions: { q: noul("use choice") }, + assets: { image: image(jpeg), choice: image(png) }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async () => + Promise.resolve( + new Response( + JSON.stringify({ answers: { q: { type: "noul", noul: 0.2 } } }), + { status: 200 }, + ), + ), + ), + }, + ); + if (named.fallback) throw new Error(named.reason); + expect(named.answers.q.noul).toBe(0.2); + }); + + test("sends the images header for a box question with no upload", async () => { + const seen = { header: "", contentType: "" }; + const result = await systemOne( + { + state: "a form", + questions: { region: box("the signature") }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async (_url, init) => { + const headers = new Headers(init?.headers); + seen.header = headers.get("TypeSafe-Danger-Zone-Images") ?? ""; + seen.contentType = headers.get("content-type") ?? ""; + return new Response( + JSON.stringify({ + answers: { + region: { + type: "box", + box: { top: 0.1, bottom: 0.4, left: 0.2, right: 0.5 }, + confidence: 0.4, + }, + }, + }), + { status: 200 }, + ); + }), + }, + ); + expect(seen.header).toBe("true"); + expect(seen.contentType).toBe("application/json"); + if (result.fallback) throw new Error(result.detail ?? result.reason); + + const choiceSeen = { header: "unset" }; + await systemOne( + { + state: "plain", + questions: { category: choice("What?", { a: null }) }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async (_url, init) => { + choiceSeen.header = + new Headers(init?.headers).get("TypeSafe-Danger-Zone-Images") ?? ""; + return new Response( + JSON.stringify({ + answers: { + category: { + type: "choice", + choice: "a", + probabilities: { a: 1 }, + confidence: 0.5, + }, + }, + }), + { status: 200 }, + ); + }), + }, + ); + expect(choiceSeen.header).toBe(""); + }); + + test("rejects an inverted box instead of returning a decision", async () => { + const ask = (edges: { + top: number; + bottom: number; + left: number; + right: number; + }) => + systemOne( + { + state: "a form", + questions: { region: box("the signature") }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async () => + Promise.resolve( + new Response( + JSON.stringify({ + answers: { + region: { type: "box", box: edges, confidence: 0.5 }, + }, + }), + { status: 200 }, + ), + ), + ), + }, + ); + for (const edges of [ + { top: 0.8, bottom: 0.1, left: 0.2, right: 0.5 }, + { top: 0.4, bottom: 0.4, left: 0.2, right: 0.5 }, + { top: 0.1, bottom: 0.4, left: 0.9, right: 0.2 }, + { top: 0.1, bottom: 0.4, left: 0.5, right: 0.5 }, + { top: -0.1, bottom: 0.4, left: 0.2, right: 0.5 }, + ]) { + const result = await ask(edges); + expect(result.fallback).toBe(true); + if (!result.fallback) throw new Error("expected fallback"); + expect(result.reason).toBe("parse-error"); + } + }); }); const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47]; diff --git a/src/evaluate.test.ts b/src/evaluate.test.ts index 6db0e63..f815a94 100644 --- a/src/evaluate.test.ts +++ b/src/evaluate.test.ts @@ -305,3 +305,75 @@ describe("key material", () => { expect(JSON.stringify(events)).not.toContain(secret); }); }); + +describe("box answers", () => { + const boxQuestion: Question[] = [ + { id: "region", type: "box", instructions: "the signature" }, + ]; + + function boxDeps( + box: { top: number; bottom: number; left: number; right: number }, + seen?: { header: string | null; contentType: string | null }, + ): EvaluateDeps { + return { + fetch: (_url, init) => { + if (seen !== undefined) { + const headers = new Headers(init?.headers); + seen.header = headers.get("TypeSafe-Danger-Zone-Images"); + seen.contentType = headers.get("content-type"); + } + return Promise.resolve( + new Response( + JSON.stringify({ + answers: { region: { type: "box", box, confidence: 0.4 } }, + }), + { status: 200 }, + ), + ); + }, + scheduler: createDefaultScheduler(), + }; + } + + test("an inverted box is a parse fallback, not a decision", async () => { + for (const box of [ + { top: 0.8, bottom: 0.1, left: 0.2, right: 0.5 }, + { top: 0.4, bottom: 0.4, left: 0.1, right: 0.4 }, + { top: 0.1, bottom: 0.4, left: 0.9, right: 0.2 }, + { top: 0.1, bottom: 0.4, left: 0.5, right: 0.5 }, + ]) { + const result = await evaluate( + { state: "form", questions: boxQuestion, config: { apiKey: "key" } }, + { deps: boxDeps(box) }, + ); + expect(result.fallback).toBe(true); + if (!result.fallback) throw new Error("expected fallback"); + expect(result.reason).toBe("parse-error"); + } + const outside = await evaluate( + { state: "form", questions: boxQuestion, config: { apiKey: "key" } }, + { + deps: boxDeps({ top: -0.1, bottom: 0.4, left: 0.2, right: 0.5 }), + }, + ); + expect(outside).toMatchObject({ fallback: true, reason: "parse-error" }); + if (!outside.fallback) throw new Error("expected fallback"); + expect(outside.detail).toContain("outside [0, 1]"); + }); + + test("a box question with no image parts still opens the images gate", async () => { + const seen = { + header: null as string | null, + contentType: null as string | null, + }; + const result = await evaluate( + { state: "form", questions: boxQuestion, config: { apiKey: "key" } }, + { + deps: boxDeps({ top: 0.1, bottom: 0.4, left: 0.2, right: 0.5 }, seen), + }, + ); + expect(seen.header).toBe("true"); + expect(seen.contentType).toBe("application/json"); + expect(result.fallback).toBe(false); + }); +}); diff --git a/src/schemas.ts b/src/schemas.ts index 6ed1d42..0d05621 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -332,6 +332,12 @@ export function boxEdgesOutOfRange( return `answer "${id}" box.${edge} ${String(value)} outside [0, 1]`; } } + if (box.top >= box.bottom) { + return `answer "${id}" box.top ${String(box.top)} is not below box.bottom ${String(box.bottom)}`; + } + if (box.left >= box.right) { + return `answer "${id}" box.left ${String(box.left)} is not left of box.right ${String(box.right)}`; + } return undefined; }