From 6ca931a0b11571d12c691f06e494b988891424a8 Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 15:58:25 -0700 Subject: [PATCH 1/9] feat: let custom endpoints opt out of API key auth A custom endpoint with auth set to none skips the no-key fallback and sends no Authorization header. Official and gateway endpoints still require a key. --- README.md | 29 ++++++------ src/adapter.test.ts | 32 +++++++++++++ src/adapter.ts | 8 +++- src/config.ts | 6 +++ src/evaluate.test.ts | 107 +++++++++++++++++++++++++++++++++++++++++++ src/evaluate.ts | 13 ++++-- src/schemas.ts | 3 ++ 7 files changed, 179 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index e1090c0..2c54c2e 100644 --- a/README.md +++ b/README.md @@ -95,11 +95,11 @@ Every question has an `id`, unique within the list. `input.config` for `evaluate`, or the argument to `createSystemOneAdapter`: -| Field | Type | Default | Description | -| ----------- | ---------------- | ---------------------- | -------------------------------------------------------------------------------- | -| `endpoint` | `EndpointConfig` | `{ kind: "official" }` | `official`, `gateway`, or `custom` with a `url`. Each takes an optional `model`. | -| `timeoutMs` | `number` | `1500` | Bound for the whole round trip. | -| `apiKey` | `string` | from the environment | Overrides the environment key. | +| Field | Type | Default | Description | +| ----------- | ---------------- | ---------------------- | ----------------------------------------------------------------------------------------------------------------- | +| `endpoint` | `EndpointConfig` | `{ kind: "official" }` | `official`, `gateway`, or `custom` with a `url`. Each takes an optional `model`. `custom` may set `auth: "none"`. | +| `timeoutMs` | `number` | `1500` | Bound for the whole round trip. | +| `apiKey` | `string` | from the environment | Overrides the environment key. | ### Endpoints and keys @@ -107,19 +107,20 @@ Every question has an `id`, unique within the list. | ---------- | ---------------------------------------------------- | ----------------- | ---------------------------------------------- | | `official` | `https://api.typesafe.ai/v1/systemone` | `jev-latest` | `TYPESAFE_API_KEY` | | `gateway` | `https://ai-gateway.vercel.sh/typesafe/v1/systemone` | `typesafe-ai/jev` | `AI_GATEWAY_API_KEY`, then `VERCEL_OIDC_TOKEN` | -| `custom` | `url` | `typesafe-ai/jev` | `SYSTEM_ONE_API_KEY` | +| `custom` | `url` | `typesafe-ai/jev` | `SYSTEM_ONE_API_KEY`, unless `auth: "none"` | `SYSTEM_ONE_API_KEY` is also a supported alias on every endpoint, read after the endpoint's own key. +A custom endpoint with `auth: "none"` does not require a key and does not send `Authorization`. Official and gateway always require a key. Any other `auth` value is invalid input and throws. ### Fallback reasons -| `reason` | When | -| ------------- | ------------------------------------------------------------- | -| `no-key` | No key in `apiKey` or the environment. No request is sent. | -| `timeout` | The round trip, including the body read, exceeds `timeoutMs`. | -| `network` | The request fails before a response. | -| `http-error` | Non-2xx response. `httpStatus` holds the status. | -| `parse-error` | The body is not JSON or fails validation. | +| `reason` | When | +| ------------- | -------------------------------------------------------------------------------------------------------------- | +| `no-key` | No key in `apiKey` or the environment, and the endpoint is not custom with `auth: "none"`. No request is sent. | +| `timeout` | The round trip, including the body read, exceeds `timeoutMs`. | +| `network` | The request fails before a response. | +| `http-error` | Non-2xx response. `httpStatus` holds the status. | +| `parse-error` | The body is not JSON or fails validation. | `detail` carries the underlying message. @@ -208,7 +209,7 @@ The hub offering's `quirks` are validated and all optional: } ``` -Without `endpoint`, the adapter posts to the catalog provider's `baseURL` + `/systemone`, the host the offering's credential was issued for. A `model` quirk (top level or on `endpoint`) is the model sent on the wire, so one catalog model can be served by providers that name it differently. `questions` and `state` are per-call defaults; `providerOptions.systemOne` on a call wins field by field. Unknown keys throw. The adapter sends the bearer sentinel and the harness injects the offering's credential. +Without `endpoint`, the adapter posts to the catalog provider's `baseURL` + `/systemone`, the host the offering's credential was issued for. A `model` quirk (top level or on `endpoint`) is the model sent on the wire, so one catalog model can be served by providers that name it differently. `questions` and `state` are per-call defaults; `providerOptions.systemOne` on a call wins field by field. Unknown keys throw. The adapter sends the bearer sentinel and the harness injects the offering's credential, except a custom endpoint with `auth: "none"`, which omits `Authorization` so the host does not attach a key. | Provider | Provider `baseURL` | `model` quirk | | ----------------- | ------------------------------------------ | ----------------------------- | diff --git a/src/adapter.test.ts b/src/adapter.test.ts index 41fefdb..f4a12d3 100644 --- a/src/adapter.test.ts +++ b/src/adapter.test.ts @@ -122,6 +122,38 @@ describe("adapter factory", () => { { role: "user", timestamp: 0, content: [{ type: "text", text: "Hi" }] }, ]; + test("buildRequest omits Authorization for custom auth none and keeps the sentinel otherwise", () => { + const turns = (): ConversationTurn[] => [ + { role: "user", timestamp: 0, content: [{ type: "text", text: "Hi" }] }, + ]; + const keyless = createSystemOneAdapter({ + endpoint: { + kind: "custom", + url: "https://local.test/eval", + auth: "none", + }, + }).buildRequest(turns(), "jev-latest", {}); + expect(keyless.headers["authorization"]).toBeUndefined(); + expect(Object.keys(keyless.headers)).not.toContain("authorization"); + + const official = createSystemOneAdapter({ + endpoint: { kind: "official" }, + }).buildRequest(turns(), "jev-latest", {}); + expect(official.headers["authorization"]).toBe(BEARER_CREDENTIAL_SENTINEL); + + const gateway = createSystemOneAdapter({ + endpoint: { kind: "gateway" }, + }).buildRequest(turns(), "jev-latest", {}); + expect(gateway.headers["authorization"]).toBe(BEARER_CREDENTIAL_SENTINEL); + + const keyedCustom = createSystemOneAdapter({ + endpoint: { kind: "custom", url: "https://h.test/e" }, + }).buildRequest(turns(), "jev-latest", {}); + expect(keyedCustom.headers["authorization"]).toBe( + BEARER_CREDENTIAL_SENTINEL, + ); + }); + test("no quirks routes to the provider baseURL with sentinel auth", () => { const request = createSystemOneAdapterFactory(source).buildRequest( turns(), diff --git a/src/adapter.ts b/src/adapter.ts index ea67599..2752bf3 100644 --- a/src/adapter.ts +++ b/src/adapter.ts @@ -158,7 +158,9 @@ function parseDecisions( * host's single copy of `@intx/inference` (peer dependency, never vendored). * The optional `config` pins the endpoint (official by default, gateway, or * a custom URL) and the per-call model default; the inference call's own - * `model` argument wins for the request body. + * `model` argument wins for the request body. Official and gateway requests + * carry `authorization: BEARER_CREDENTIAL_SENTINEL` so the host injects a + * key. A custom endpoint with `auth: "none"` omits that header. */ function createSystemOneProviderAdapter( config?: EvaluateConfig, @@ -193,8 +195,10 @@ function createSystemOneProviderAdapter( const headers: Record = { "content-type": "application/json", accept: "application/json", - authorization: BEARER_CREDENTIAL_SENTINEL, }; + if (endpoint.auth !== "none") { + headers["authorization"] = BEARER_CREDENTIAL_SENTINEL; + } return { url: endpoint.url, headers, diff --git a/src/config.ts b/src/config.ts index c496852..262b995 100644 --- a/src/config.ts +++ b/src/config.ts @@ -69,6 +69,8 @@ export type ResolvedEndpoint = { url: string; backend: "corbits-system-one" | "gateway" | "custom"; model: string; + /** `none` only when a custom endpoint explicitly opts out of a key. */ + auth: "key" | "none"; }; /** @@ -79,6 +81,8 @@ export type ResolvedEndpoint = { * distinguishes caller-routed traffic from the official endpoint — * `'gateway'` is reserved for the proxy quirks path. The model always * resolves here: a `custom` target without one sends `typesafe-ai/jev`. + * `auth` is `"none"` only when the caller set `auth: "none"` on a custom + * endpoint; every other target still requires a key. */ export function resolveEndpoint(endpoint?: EndpointConfig): ResolvedEndpoint { if (endpoint?.kind === "custom") { @@ -92,6 +96,7 @@ export function resolveEndpoint(endpoint?: EndpointConfig): ResolvedEndpoint { url: endpoint.url, backend: "custom", model: endpoint.model ?? GATEWAY_QUIRKS.model, + auth: endpoint.auth === "none" ? "none" : "key", }; } const kind = endpoint?.kind ?? "official"; @@ -108,5 +113,6 @@ export function resolveEndpoint(endpoint?: EndpointConfig): ResolvedEndpoint { url, backend: kind === "gateway" ? "gateway" : "corbits-system-one", model: endpoint?.model ?? quirks.model, + auth: "key", }; } diff --git a/src/evaluate.test.ts b/src/evaluate.test.ts index 6db0e63..86b4d14 100644 --- a/src/evaluate.test.ts +++ b/src/evaluate.test.ts @@ -230,6 +230,113 @@ async function bearerFor( return seen.authorization; } +describe("custom auth none", () => { + const keyless: EndpointConfig = { + kind: "custom", + url: "https://local.test/eval", + auth: "none", + }; + + test("skips no-key and sends no Authorization, even with a key present", async () => { + process.env["SYSTEM_ONE_API_KEY"] = "legacy"; + let authorization: string | null = "unset"; + let called = false; + const result = await evaluate( + { + state: {}, + questions, + config: { endpoint: keyless, apiKey: "should-not-send" }, + }, + { + deps: { + fetch: (_input, init) => { + called = true; + authorization = new Headers(init?.headers).get("authorization"); + return Promise.resolve( + new Response(JSON.stringify({ answers: answers() }), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + }, + scheduler: createDefaultScheduler(), + }, + }, + ); + expect(called).toBe(true); + expect(authorization).toBeNull(); + if (result.fallback !== false) { + throw new Error(`expected decisions, got ${JSON.stringify(result)}`); + } + expect(result.decisions).toHaveLength(3); + }); + + test("official and gateway still fail closed with no-key and do not fetch", async () => { + const deps = { + fetch: () => Promise.reject(new Error("fetch must not be called")), + scheduler: createDefaultScheduler(), + }; + const official = await evaluate( + { state: {}, questions, config: { endpoint: { kind: "official" } } }, + { deps }, + ); + const gateway = await evaluate( + { state: {}, questions, config: { endpoint: { kind: "gateway" } } }, + { deps }, + ); + const custom = await evaluate( + { + state: {}, + questions, + config: { endpoint: { kind: "custom", url: "https://h.test/e" } }, + }, + { deps }, + ); + expect(official).toMatchObject({ fallback: true, reason: "no-key" }); + expect(gateway).toMatchObject({ fallback: true, reason: "no-key" }); + expect(custom).toMatchObject({ fallback: true, reason: "no-key" }); + }); + + test("auth other than none is invalid input and throws", async () => { + let thrown: unknown; + try { + await evaluate({ + state: {}, + questions, + config: { + endpoint: { + kind: "custom", + url: "https://h.test/e", + auth: "bearer", + } as unknown as EndpointConfig, + }, + }); + } catch (cause) { + thrown = cause; + } + expect(thrown).toBeInstanceOf(SystemOneError); + }); + + test("official rejects an auth field", async () => { + let thrown: unknown; + try { + await evaluate({ + state: {}, + questions, + config: { + endpoint: { + kind: "official", + auth: "none", + } as unknown as EndpointConfig, + }, + }); + } catch (cause) { + thrown = cause; + } + expect(thrown).toBeInstanceOf(SystemOneError); + }); +}); + describe("credential precedence", () => { test("official prefers TYPESAFE_API_KEY, then SYSTEM_ONE_API_KEY", async () => { process.env["SYSTEM_ONE_API_KEY"] = "legacy"; diff --git a/src/evaluate.ts b/src/evaluate.ts index a0c096f..967e07f 100644 --- a/src/evaluate.ts +++ b/src/evaluate.ts @@ -236,6 +236,9 @@ export type EvaluateOptions = { * `TYPESAFE_API_KEY` for official, `AI_GATEWAY_API_KEY` then * `VERCEL_OIDC_TOKEN` for gateway — then the `SYSTEM_ONE_API_KEY` alias, * then a fail-closed `'no-key'` fallback with zero fetch calls. + * A custom endpoint with `auth: "none"` skips that check and sends no + * `Authorization` header, even if a key is present. Official and gateway + * always require a key; a custom endpoint without `auth: "none"` does too. * A `timeoutMs` that is not a finite number >= 0 falls back to * `DEFAULT_TIMEOUT_MS`. Transport failures map to matching fallback reasons * (`'timeout'`, `'network'`, `'http-error'`, or `'parse-error'` for a 2xx @@ -282,7 +285,11 @@ export async function evaluate( const rawApiKey = parsed.config?.apiKey; let apiKey: string | undefined; let keySources: string; - if (rawApiKey !== undefined && rawApiKey !== "") { + const keyless = endpoint.auth === "none"; + if (keyless) { + apiKey = undefined; + keySources = ""; + } else if (rawApiKey !== undefined && rawApiKey !== "") { apiKey = rawApiKey; keySources = "config.apiKey"; } else if (endpoint.backend === "corbits-system-one") { @@ -332,7 +339,7 @@ export async function evaluate( }; emit(startEvent); - if (apiKey === undefined || apiKey === "") { + if (!keyless && (apiKey === undefined || apiKey === "")) { return fallback("no-key", { detail: `no API key supplied; set config.apiKey or ${keySources}`, }); @@ -356,7 +363,7 @@ export async function evaluate( const response = await postEvaluate( endpoint.url, wire, - { timeoutMs, apiKey }, + apiKey === undefined ? { timeoutMs } : { timeoutMs, apiKey }, deps, ); data = response.data; diff --git a/src/schemas.ts b/src/schemas.ts index a9ab389..aabebd7 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -138,10 +138,13 @@ export type QuestionList = typeof QuestionList.infer; // `gateway` the proxy endpoint, `custom` a caller-supplied URL. `url` only // exists on `custom` — a `url` on another kind would be silently ignored, // so the schema rejects it instead of accepting misleading config. +// `auth: "none"` exists only on `custom`: it is the explicit opt-out from +// the fail-closed key requirement. Official and gateway cannot carry it. export const EndpointConfig = type({ kind: "'custom'", url: "string", "model?": "string", + "auth?": "'none'", "+": "reject", }).or( type({ From 235de80ed9da55f66f4cfdcc2f0a0c56794c7a9f Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 16:25:53 -0700 Subject: [PATCH 2/9] fix: reject invalid adapter endpoint auth before a request createSystemOneAdapter now parses its config the same way evaluate does, so an auth value other than none throws instead of sending the bearer sentinel. --- README.md | 2 +- src/adapter.test.ts | 17 +++++++++++++++++ src/adapter.ts | 13 +++++++++++-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2c54c2e..19ac324 100644 --- a/README.md +++ b/README.md @@ -222,7 +222,7 @@ Without `endpoint`, the adapter posts to the catalog provider's `baseURL` + `/sy - `createSystemOneAdapterFactory` without an `endpoint` quirk now posts to the catalog provider's `baseURL` + `/systemone` instead of the official TypeSafe URL. Set the provider `baseURL` to the API root (for example `https://api.typesafe.ai/v1`), or keep the old behavior with `{"endpoint":{"kind":"official"}}`. The harness sends the offering's credential to whatever URL the adapter returns, so only set an explicit `endpoint` on a host that credential belongs to. - A `model` quirk now overrides the model the harness passes (the catalog canonical name). -- `createSystemOneAdapter` and `evaluate` are unchanged. +- `createSystemOneAdapter` and `evaluate` accept `auth: "none"` on a custom endpoint and omit `Authorization`. Any other `auth` value throws `SystemOneError` before a request is built. ## Upgrading from 0.1 diff --git a/src/adapter.test.ts b/src/adapter.test.ts index f4a12d3..198fb63 100644 --- a/src/adapter.test.ts +++ b/src/adapter.test.ts @@ -9,6 +9,7 @@ import { createSystemOneAdapter, createSystemOneAdapterFactory, } from "./adapter"; +import { SystemOneError } from "./errors"; describe("adapter", () => { function turns(): ConversationTurn[] { @@ -154,6 +155,22 @@ describe("adapter factory", () => { ); }); + test("invalid auth throws before a request is built", () => { + let thrown: unknown; + try { + createSystemOneAdapter({ + endpoint: { + kind: "custom", + url: "https://h.test/e", + auth: "bearer", + }, + } as unknown as Parameters[0]); + } catch (cause) { + thrown = cause; + } + expect(thrown).toBeInstanceOf(SystemOneError); + }); + test("no quirks routes to the provider baseURL with sentinel auth", () => { const request = createSystemOneAdapterFactory(source).buildRequest( turns(), diff --git a/src/adapter.ts b/src/adapter.ts index 2752bf3..a67fde8 100644 --- a/src/adapter.ts +++ b/src/adapter.ts @@ -17,15 +17,16 @@ import type { } from "@intx/types/runtime"; import { resolveEndpoint } from "./config.js"; +import { SystemOneError } from "./errors.js"; import { Decision, toDecision, toWireQuestions, WireResponseBody, EndpointConfig, + EvaluateConfig, QuestionList, State, - type EvaluateConfig, type WireUsage, } from "./schemas.js"; @@ -268,7 +269,15 @@ function createSystemOneProviderAdapter( export function createSystemOneAdapter( config?: EvaluateConfig, ): ProviderAdapter { - return createSystemOneProviderAdapter(config); + if (config === undefined) return createSystemOneProviderAdapter(); + const parsed = EvaluateConfig(config); + if (parsed instanceof type.errors) { + throw new SystemOneError( + "parse-error", + `createSystemOneAdapter: invalid config: ${parsed.summary}`, + ); + } + return createSystemOneProviderAdapter(parsed); } // Deployment quirks a hub offering carries for this provider. `questions` From 06451c2e6d7e9999c44775e77522a4d94a0a249c Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 16:03:13 -0700 Subject: [PATCH 3/9] feat: add a keyed decisions API with image assets Callers can ask named choice, noul, score, and box questions and attach PNG or JPEG images. The existing question list and adapter stay in place. --- package.json | 5 + src/adapter.test.ts | 22 +++ src/adapter.ts | 27 +++- src/assets.ts | 223 +++++++++++++++++++++++++++++++ src/client.ts | 77 ++++++++++- src/decisions.test.ts | 270 +++++++++++++++++++++++++++++++++++++ src/decisions.ts | 304 ++++++++++++++++++++++++++++++++++++++++++ src/evaluate.ts | 34 ++++- src/index.ts | 5 + src/schemas.ts | 98 +++++++++++++- 10 files changed, 1052 insertions(+), 13 deletions(-) create mode 100644 src/assets.ts create mode 100644 src/decisions.test.ts create mode 100644 src/decisions.ts diff --git a/package.json b/package.json index d181084..22b619a 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,11 @@ "corbits-system-one-src": "./src/index.ts", "types": "./dist/index.d.ts", "default": "./dist/index.js" + }, + "./decisions": { + "corbits-system-one-src": "./src/decisions.ts", + "types": "./dist/decisions.d.ts", + "default": "./dist/decisions.js" } }, "publishConfig": { diff --git a/src/adapter.test.ts b/src/adapter.test.ts index 198fb63..13ee92e 100644 --- a/src/adapter.test.ts +++ b/src/adapter.test.ts @@ -232,4 +232,26 @@ describe("adapter factory", () => { /model must not be empty/, ); }); + + test("providerOptions.systemOne accepts keyed questions", () => { + const adapter = createSystemOneAdapter(); + const request = adapter.buildRequest([], "jev-latest", { + providerOptions: { + systemOne: { + state: "ticket", + questions: { + category: { + type: "choice", + instructions: "What?", + criteria: { a: null }, + }, + }, + }, + }, + }); + const body = JSON.parse(request.body) as { + questions: Record; + }; + expect(body.questions["category"]?.type).toBe("choice"); + }); }); diff --git a/src/adapter.ts b/src/adapter.ts index a67fde8..067158f 100644 --- a/src/adapter.ts +++ b/src/adapter.ts @@ -20,6 +20,7 @@ import { resolveEndpoint } from "./config.js"; import { SystemOneError } from "./errors.js"; import { Decision, + DecisionQuestionRecord, toDecision, toWireQuestions, WireResponseBody, @@ -52,9 +53,11 @@ export const SYSTEM_ONE_PROVIDER = "corbits-system-one"; // `state` replaces the transcript-derived default; `questions` replaces the // single-question default. Both are validated, never cast — `QuestionList` // carries the same non-empty/unique-id contract as `EvaluateInput`. +const QuestionsInput = QuestionList.or(DecisionQuestionRecord); + const SystemOneProviderOptions = type({ "state?": State, - "questions?": QuestionList, + "questions?": QuestionsInput, "+": "reject", }); @@ -86,6 +89,24 @@ const DEFAULT_QUESTIONS: QuestionList = [ }, ]; +function asQuestionList( + questions: QuestionList | DecisionQuestionRecord, +): QuestionList { + if (Array.isArray(questions)) return questions; + const compiled = Object.entries(questions).map(([id, question]) => ({ + id, + ...question, + })); + const parsed = QuestionList(compiled); + if (parsed instanceof type.errors) { + throw new ProtocolMismatchError( + `${SYSTEM_ONE_PROVIDER} buildRequest: invalid providerOptions.systemOne questions: ${parsed.summary}`, + questions, + ); + } + return parsed; +} + function transcriptText(turns: ConversationTurn[]): string { const texts: string[] = []; for (const turn of turns) { @@ -190,7 +211,9 @@ function createSystemOneProviderAdapter( ); } if (overrides.state !== undefined) state = overrides.state; - if (overrides.questions !== undefined) questions = overrides.questions; + if (overrides.questions !== undefined) { + questions = asQuestionList(overrides.questions); + } } const requestModel = model || endpoint.model; const headers: Record = { diff --git a/src/assets.ts b/src/assets.ts new file mode 100644 index 0000000..2298da1 --- /dev/null +++ b/src/assets.ts @@ -0,0 +1,223 @@ +import { SystemOneError } from "./errors.js"; + +const CONTENT_TYPES = [ + ["image/png", [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]], + ["image/jpeg", [0xff, 0xd8, 0xff]], +] as const; + +const REFERENCE = //g; + +const IMAGE = Symbol.for("@typesafe-ai/sdk/image"); +const TEXT_WITH_ASSETS = Symbol.for("@typesafe-ai/sdk/text-with-assets"); + +const randomHex = (): string => + Array.from(crypto.getRandomValues(new Uint8Array(16)), (byte) => + byte.toString(16).padStart(2, "0"), + ).join(""); + +const isImage = (value: object): value is Image => IMAGE in value; + +const isTextWithAssets = (value: object): value is TextWithAssets => + TEXT_WITH_ASSETS in value; + +function fail(message: string): never { + throw new SystemOneError("parse-error", message); +} + +const toBytes = (data: ArrayBuffer | ArrayBufferView): Uint8Array => { + if ( + !ArrayBuffer.isView(data) && + Object.prototype.toString.call(data) !== "[object ArrayBuffer]" + ) { + fail("Image data must be file bytes or a Blob."); + } + return ArrayBuffer.isView(data) + ? new Uint8Array(data.buffer, data.byteOffset, data.byteLength) + : new Uint8Array(data); +}; + +export const contentTypeOf = (bytes: Uint8Array): string => { + for (const [contentType, signature] of CONTENT_TYPES) { + if (signature.every((byte, index) => bytes[index] === byte)) { + return contentType; + } + } + return "application/octet-stream"; +}; + +/** A PNG or JPEG image. Create one with `image` or `Image.fromBase64`. */ +export class Image { + readonly type = "image"; + readonly data: Uint8Array | Blob; + readonly #reference = ``; + + constructor(data: Blob | ArrayBuffer | ArrayBufferView) { + this.data = data instanceof Blob ? data : toBytes(data); + const size = + this.data instanceof Blob ? this.data.size : this.data.byteLength; + if (size === 0) fail("Image data is empty."); + } + + /** Decode base64 file bytes. Do not include a `data:` URL prefix. */ + static fromBase64(text: string): Image { + let binary: string; + try { + binary = atob(text); + } catch (cause) { + const error = new SystemOneError( + "parse-error", + "Image data must contain valid base64 text.", + ); + error.cause = cause; + throw error; + } + return new Image(Uint8Array.from(binary, (char) => char.charCodeAt(0))); + } + + toString(): string { + return this.#reference; + } + + toJSON(): string { + return this.#reference; + } +} + +Object.defineProperty(Image.prototype, IMAGE, { value: true }); + +class TextWithAssets { + readonly text: string; + readonly assets: Image[]; + + constructor(text: string, assets: Image[]) { + this.text = text; + this.assets = assets; + } + + toString(): string { + return this.text; + } + + toJSON(): string { + return this.text; + } +} + +Object.defineProperty(TextWithAssets.prototype, TEXT_WITH_ASSETS, { + value: true, +}); + +export type TextWithAssetsValue = TextWithAssets; + +/** Create an image from file bytes or a Blob. PNG and JPEG only, checked before send. */ +export const image = (data: Blob | ArrayBuffer | ArrayBufferView): Image => + new Image(data); + +const collectAssets = ( + value: unknown, + found: Map, + visited: Set, +): void => { + if (typeof value !== "object" || value === null || visited.has(value)) return; + visited.add(value); + if (isImage(value)) found.set(String(value), value); + else if (isTextWithAssets(value)) { + for (const asset of value.assets) found.set(String(asset), asset); + } else { + for (const item of Object.values(value)) + collectAssets(item, found, visited); + } +}; + +const textWithAssets = (text: string, values: unknown): TextWithAssets => { + const interpolated = new Map(); + collectAssets(values, interpolated, new Set()); + const assets = new Map(); + for (const [reference] of text.matchAll(REFERENCE)) { + const asset = interpolated.get(reference); + if (asset !== undefined) assets.set(reference, asset); + } + return new TextWithAssets(text, [...assets.values()]); +}; + +/** + * Tag a template so interpolated images are sent with the text. + * An image in an untagged template is rejected before the request is sent. + */ +export const withAssets = Object.assign( + (strings: TemplateStringsArray, ...values: unknown[]): TextWithAssets => + textWithAssets( + strings.reduce( + (joined, piece, index) => joined + String(values[index - 1]) + piece, + ), + values, + ), + { + join: (values: readonly unknown[], separator: string): TextWithAssets => + textWithAssets(values.join(separator), values), + }, +); + +export type SerializedRequest = { + json: string; + assets: Record | undefined; +}; + +/** Replace images with references and reject dangling or unused asset names. */ +export function serializeRequest( + payload: unknown, + assets: Record = {}, +): SerializedRequest { + const names = new Map( + Object.entries(assets).map(([name, asset]) => [asset, name]), + ); + const known = new Map( + Object.values(assets).map((asset) => [String(asset), asset]), + ); + const json = JSON.stringify(payload, function (key, value: unknown) { + if (typeof value === "number" && !Number.isFinite(value)) + return String(value); + const holder = this as Record; + const original = holder[key]; + if (typeof original !== "object" || original === null) return value; + if (isImage(original)) { + const name = names.get(original); + if (name !== undefined) return name; + known.set(String(original), original); + } else if (isTextWithAssets(original)) { + let text = original.text; + for (const asset of original.assets) { + const name = names.get(asset); + if (name === undefined) known.set(String(asset), asset); + else text = text.replaceAll(String(asset), name); + } + return text; + } + return value; + }); + const unreferenced = Object.keys(assets).filter( + (name) => !json.includes(JSON.stringify(name).slice(1, -1)), + ); + if (unreferenced.length > 0) { + fail( + `Assets not referenced in the request: ${unreferenced.join(", ")}. Write each name in \`assets\` into state, question instructions, or criteria.`, + ); + } + const sent = new Map(Object.entries(assets)); + const missing = new Set(); + for (const [reference] of json.matchAll(REFERENCE)) { + if (sent.has(reference)) continue; + const asset = known.get(reference); + if (asset === undefined) missing.add(reference); + else sent.set(reference, asset); + } + if (missing.size > 0) { + fail( + `Image references without an attached image: ${[...missing].join(", ")}. Tag template literals that interpolate images with \`withAssets\`.`, + ); + } + return { + json, + assets: sent.size === 0 ? undefined : Object.fromEntries(sent), + }; +} diff --git a/src/client.ts b/src/client.ts index 0d0e1eb..07b0c56 100644 --- a/src/client.ts +++ b/src/client.ts @@ -23,6 +23,11 @@ import { SystemOneError, TransportError } from "./errors.js"; export type PostEvaluateOptions = { timeoutMs?: number; apiKey?: string; + imageParts?: { + name: string; + bytes: Uint8Array; + contentType: "image/png" | "image/jpeg"; + }[]; }; /** Transport dependencies: the `fetch` to POST with and the timeout scheduler. */ @@ -35,6 +40,61 @@ export type PostEvaluateResponse = { httpStatus: number; }; +function encodeMultipart( + payload: unknown, + parts: NonNullable, +): { body: Uint8Array; contentType: string } { + const boundary = Array.from( + crypto.getRandomValues(new Uint8Array(16)), + (byte) => byte.toString(16).padStart(2, "0"), + ).join(""); + const encoder = new TextEncoder(); + const chunks: Uint8Array[] = []; + const fields: { + name: string; + contentType: string; + content: Uint8Array; + filename?: string; + }[] = [ + { + name: "request", + contentType: "application/json", + content: encoder.encode(JSON.stringify(payload)), + }, + ...parts.map((part) => ({ + name: part.name, + contentType: part.contentType, + content: part.bytes, + filename: part.name, + })), + ]; + for (const field of fields) { + const disposition = + field.filename === undefined + ? `name="${field.name}"` + : `name="${field.name}"; filename="${field.filename}"`; + chunks.push( + encoder.encode( + `--${boundary}\r\nContent-Disposition: form-data; ${disposition}\r\nContent-Type: ${field.contentType}\r\n\r\n`, + ), + field.content, + encoder.encode("\r\n"), + ); + } + chunks.push(encoder.encode(`--${boundary}--\r\n`)); + const size = chunks.reduce((total, chunk) => total + chunk.byteLength, 0); + const body = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return { + body, + contentType: `multipart/form-data; boundary=${boundary}`, + }; +} + /** * POSTs an evaluation payload and returns the decoded JSON body. Sends * `Authorization: Bearer ` only when a key is present; aborts the @@ -52,9 +112,17 @@ export async function postEvaluate( options: PostEvaluateOptions, deps: EvaluateDeps, ): Promise { - let serialized: string; + let serialized: string | Uint8Array; + const imageParts = options.imageParts; + let contentType = "application/json"; try { - serialized = JSON.stringify(body); + if (imageParts !== undefined && imageParts.length > 0) { + const encoded = encodeMultipart(body, imageParts); + serialized = encoded.body; + contentType = encoded.contentType; + } else { + serialized = JSON.stringify(body); + } } catch { throw new SystemOneError( "parse-error", @@ -81,9 +149,12 @@ export async function postEvaluate( }; try { const headers: Record = { - "content-type": "application/json", + "content-type": contentType, accept: "application/json", }; + if (imageParts !== undefined && imageParts.length > 0) { + headers["TypeSafe-Danger-Zone-Images"] = "true"; + } if (options.apiKey !== undefined && options.apiKey !== "") { headers["authorization"] = `Bearer ${options.apiKey}`; } diff --git a/src/decisions.test.ts b/src/decisions.test.ts new file mode 100644 index 0000000..a8a537a --- /dev/null +++ b/src/decisions.test.ts @@ -0,0 +1,270 @@ +import type { EvaluateDeps } from "./client"; +import { describe, expect, test } from "bun:test"; +import { createDefaultScheduler } from "@intx/inference"; + +import { Image, image, withAssets } from "./assets"; +import { box, choice, noul, score, systemOne } from "./decisions"; +import { SystemOneError } from "./errors"; + +const png = Uint8Array.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, +]); +const jpeg = Uint8Array.from([0xff, 0xd8, 0xff, 0x00]); +const gif = Uint8Array.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61]); + +function deps(fetchImpl: EvaluateDeps["fetch"]): EvaluateDeps { + return { fetch: fetchImpl, scheduler: createDefaultScheduler() }; +} + +function okBody() { + return { + model: "jev-latest", + answers: { + category: { + type: "choice", + choice: "billing", + probabilities: { billing: 0.8, technical: 0.2 }, + confidence: 0.7, + }, + escalate: { type: "noul", noul: 0.2 }, + severity: { + type: "score", + score: 1, + legend: { "0": "none", "1": "cosmetic", "2": "unusable" }, + probabilities: { "0": 0.2, "1": 0.6, "2": 0.2 }, + confidence: 0.5, + }, + }, + }; +} + +describe("systemOne", () => { + test("returns named answers and falls back without throwing", async () => { + let calls = 0; + const result = await systemOne( + { + state: { document: "charged twice" }, + questions: { + category: choice("What is this?", { billing: null, technical: null }), + escalate: noul("Must this be escalated?"), + severity: score("How severe?", ["none", "cosmetic", "unusable"]), + }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async () => { + calls += 1; + return new Response(JSON.stringify(okBody()), { status: 200 }); + }), + }, + ); + expect(calls).toBe(1); + if (result.fallback) throw new Error("expected answers"); + expect(result.answers.category.choice).toBe("billing"); + expect(result.answers.category.probabilities.billing).toBe(0.8); + expect(result.answers.category.confidence).toBe(0.7); + expect(result.answers.escalate.noul).toBe(0.2); + expect(result.answers.severity.score).toBe(1); + + const fallback = await systemOne( + { + state: { document: "x" }, + questions: { escalate: noul("Must this be escalated?") }, + }, + { + deps: deps(() => Promise.reject(new Error("fetch must not be called"))), + }, + ); + expect(fallback.fallback).toBe(true); + if (!fallback.fallback) throw new Error("expected fallback"); + expect(fallback.reason).toBe("no-key"); + + const http = await systemOne( + { + state: { document: "x" }, + questions: { escalate: noul("Must this be escalated?") }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async () => new Response("no", { status: 500 })), + }, + ); + expect(http.fallback).toBe(true); + if (!http.fallback) throw new Error("expected fallback"); + expect(http.reason).toBe("http-error"); + }); + + test("sends images as multipart and rejects other formats first", async () => { + const bodies: { headers: Headers; text: string }[] = []; + const photo = image(png); + const result = await systemOne( + { + state: { photo }, + questions: { + region: box("The damaged area"), + damaged: noul("Is it damaged?"), + }, + model: "jev-preview", + config: { apiKey: "test-key" }, + }, + { + deps: deps(async (_url, init) => { + const headers = new Headers(init?.headers); + const body = init?.body; + const text = + typeof body === "string" + ? body + : await new Response(body as Blob).text(); + bodies.push({ headers, text }); + return new Response( + JSON.stringify({ + answers: { + region: { + type: "box", + box: { top: 0.1, bottom: 0.4, left: 0.2, right: 0.5 }, + confidence: null, + }, + damaged: { type: "noul", noul: 0.9 }, + }, + }), + { status: 200 }, + ); + }), + }, + ); + if (result.fallback) throw new Error(result.reason); + expect(result.answers.region.box).toEqual({ + top: 0.1, + bottom: 0.4, + left: 0.2, + right: 0.5, + }); + expect(result.answers.region.confidence).toBeNull(); + const sent = bodies[0]; + if (sent === undefined) throw new Error("expected a request"); + expect(sent.headers.get("TypeSafe-Danger-Zone-Images")).toBe("true"); + expect(sent.headers.get("content-type")).toStartWith("multipart/form-data"); + expect(sent.text).toContain('name="request"'); + expect(sent.text).toContain('name="asset-0"'); + expect(sent.text).toContain("jev-preview"); + + let fetched = false; + let thrown: unknown; + try { + await systemOne( + { + state: image(gif), + questions: { damaged: noul("Is it damaged?") }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async () => { + fetched = true; + return new Response("{}", { status: 200 }); + }), + }, + ); + } catch (cause) { + thrown = cause; + } + expect(fetched).toBe(false); + expect(thrown).toBeInstanceOf(SystemOneError); + }); + + test("rejects a plain template image and requires every asset name", async () => { + const photo = image(jpeg); + let fetched = false; + const transport = deps(async () => { + fetched = true; + return new Response("{}", { status: 200 }); + }); + await expect( + systemOne( + { + state: `see ${photo}`, + questions: { damaged: noul("Is it damaged?") }, + config: { apiKey: "test-key" }, + }, + { deps: transport }, + ), + ).rejects.toBeInstanceOf(SystemOneError); + + await expect( + systemOne( + { + state: "no photo here", + questions: { damaged: noul("Is it damaged?") }, + assets: { "": image(png) }, + config: { apiKey: "test-key" }, + }, + { deps: transport }, + ), + ).rejects.toBeInstanceOf(SystemOneError); + expect(fetched).toBe(false); + + const tagged = await systemOne( + { + state: withAssets`compare ${photo}`, + questions: { + same: noul(withAssets`Does ${photo} match?`), + }, + config: { apiKey: "test-key" }, + }, + { + deps: deps( + async () => + new Response( + JSON.stringify({ + answers: { same: { type: "noul", noul: 0.4 } }, + }), + { status: 200 }, + ), + ), + }, + ); + if (tagged.fallback) throw new Error(tagged.reason); + expect(tagged.answers.same.noul).toBe(0.4); + + const joined = withAssets.join([image(png), image(jpeg)], ", "); + expect(String(joined)).toContain(" please", + questions: { damaged: noul("Is damaged?") }, + assets: { + "": Image.fromBase64(Buffer.from(png).toString("base64")), + }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async (_url, init) => { + const text = await new Response(init?.body as Blob).text(); + expect(text).toContain(""); + expect(text).toContain("image/png"); + return new Response( + JSON.stringify({ + answers: { damaged: { type: "noul", noul: 0.1 } }, + }), + { status: 200 }, + ); + }), + }, + ); + if (named.fallback) throw new Error(named.reason); + expect(named.answers.damaged.noul).toBe(0.1); + }); + + test("rejects extraction questions", async () => { + await expect( + systemOne( + { + state: "x", + questions: { field: { type: "extract", instructions: "pull it" } }, + config: { apiKey: "k" }, + }, + { deps: deps(async () => new Response("{}", { status: 200 })) }, + ), + ).rejects.toBeInstanceOf(SystemOneError); + }); +}); diff --git a/src/decisions.ts b/src/decisions.ts new file mode 100644 index 0000000..7470e22 --- /dev/null +++ b/src/decisions.ts @@ -0,0 +1,304 @@ +import { type } from "arktype"; + +import { contentTypeOf, Image, serializeRequest } from "./assets.js"; +import type { EvaluateDeps } from "./client.js"; +import { evaluate, type EvaluateOptions } from "./evaluate.js"; +import { SystemOneError } from "./errors.js"; +import { + DecisionQuestionRecord, + QuestionList, + type Decision, + type Description, + type EvaluateConfig, + type FallbackResult, + type State, +} from "./schemas.js"; + +type ChoiceCriteria = Record; + +type NoulCriteria = { + true?: Description; + false?: Description; +}; + +export function choice( + instructions: unknown, + criteria: Criteria, +): { type: "choice"; instructions: unknown; criteria: Criteria } { + if (Array.isArray(criteria)) { + throw new SystemOneError( + "parse-error", + "Choice criteria must be a map of labels to descriptions, not a list.", + ); + } + return { type: "choice", instructions, criteria }; +} + +export function noul( + instructions: unknown, + criteria?: NoulCriteria, +): { type: "noul"; instructions: unknown; criteria?: NoulCriteria } { + const question: { + type: "noul"; + instructions: unknown; + criteria?: NoulCriteria; + } = { type: "noul", instructions }; + if (criteria !== undefined) question.criteria = criteria; + return question; +} + +export function score( + instructions: unknown, + criteria: Criteria, +): { type: "score"; instructions: unknown; criteria: Criteria } { + if (!Array.isArray(criteria)) { + throw new SystemOneError( + "parse-error", + "Score criteria must be a list of descriptions indexed by score from zero, not a map.", + ); + } + return { type: "score", instructions, criteria }; +} + +/** + * Locate what `instructions` describe. The answer's `box` edges are fractions + * of the image. Box questions need the `jev-preview` model; pass it as + * `model` (or on the endpoint). This client does not reject other models. + */ +export function box(instructions: unknown): { + type: "box"; + instructions: unknown; +} { + return { type: "box", instructions }; +} + +type ChoiceAnswer = { + type: "choice"; + choice: keyof Criteria & string; + probabilities: Record; + confidence: number; +}; + +type NoulAnswer = { type: "noul"; noul: number }; + +type ScoreAnswer = { + type: "score"; + score: number; + legend: Record; + probabilities: Record; + confidence: number; +}; + +type BoxAnswer = { + type: "box"; + box: { top: number; bottom: number; left: number; right: number }; + confidence: number | null; +}; + +type AnswerOf = QuestionSpec extends { + type: "choice"; + criteria: infer Criteria extends ChoiceCriteria; +} + ? ChoiceAnswer + : QuestionSpec extends { type: "noul" } + ? NoulAnswer + : QuestionSpec extends { type: "score" } + ? ScoreAnswer + : QuestionSpec extends { type: "box" } + ? BoxAnswer + : never; + +export type SystemOneAnswers> = { + [Key in keyof Questions]: AnswerOf; +}; + +export type SystemOneSuccess> = { + fallback: false; + answers: SystemOneAnswers; + modelId: string; + backend: "corbits-system-one" | "gateway" | "custom"; + latencyMs: number; + usage?: { inputTokens: number; outputTokens: number }; +}; + +export type SystemOneResult> = + | SystemOneSuccess + | FallbackResult; + +export type SystemOneInput> = { + state: unknown; + questions: Questions; + assets?: Record; + model?: string; + config?: EvaluateConfig; +}; + +function withModel( + config: EvaluateConfig | undefined, + model: string | undefined, +): EvaluateConfig { + const next: EvaluateConfig = {}; + if (config?.apiKey !== undefined) next.apiKey = config.apiKey; + if (config?.timeoutMs !== undefined) next.timeoutMs = config.timeoutMs; + const endpoint = config?.endpoint; + if (model !== undefined) { + next.endpoint = { ...(endpoint ?? { kind: "official" }), model }; + } else if (endpoint !== undefined) { + next.endpoint = endpoint; + } + return next; +} + +function answersFrom(decisions: Decision[]): Record { + const answers: Record = {}; + for (const decision of decisions) { + if (decision.type === "choice") { + answers[decision.id] = { + type: "choice", + choice: decision.choice, + probabilities: decision.probabilities, + confidence: decision.confidence, + }; + } else if (decision.type === "noul") { + answers[decision.id] = { type: "noul", noul: decision.noul }; + } else if (decision.type === "score") { + answers[decision.id] = { + type: "score", + score: decision.score, + legend: decision.legend, + probabilities: decision.probabilities, + confidence: decision.confidence, + }; + } else { + answers[decision.id] = { + type: "box", + box: decision.box, + confidence: decision.confidence, + }; + } + } + return answers; +} + +async function loadImages( + assets: Record, +): Promise> { + const parts: NonNullable["parts"] = []; + const wire: NonNullable["assets"] = {}; + let index = 0; + for (const [alias, asset] of Object.entries(assets)) { + let bytes: Uint8Array; + if (asset.data instanceof Blob) { + bytes = new Uint8Array(await asset.data.arrayBuffer()); + } else { + bytes = asset.data; + } + const contentType = contentTypeOf(bytes); + if (contentType !== "image/png" && contentType !== "image/jpeg") { + throw new SystemOneError( + "parse-error", + `Image "${alias}" must be PNG or JPEG.`, + ); + } + const name = `asset-${index}`; + parts.push({ + name, + bytes, + contentType, + }); + wire[alias] = { type: "image", part: name }; + index += 1; + } + return { assets: wire, parts }; +} + +/** + * Keyed decisions call. `choice`, `noul`, `score`, and `box` compile to the + * question list `evaluate` already sends. Transport failures return + * `fallback` and `reason` and do not throw. + */ +export async function systemOne< + const Questions extends Record, +>( + input: SystemOneInput, + options?: Pick, +): Promise> { + const parsed = DecisionQuestionRecord(input.questions); + if (parsed instanceof type.errors) { + throw new SystemOneError( + "parse-error", + `systemOne: invalid questions: ${parsed.summary}`, + ); + } + const serialized = serializeRequest( + { state: input.state, questions: input.questions }, + input.assets, + ); + let payload: { state?: unknown; questions?: unknown }; + try { + payload = JSON.parse(serialized.json) as { + state?: unknown; + questions?: unknown; + }; + } catch (cause) { + throw new SystemOneError( + "parse-error", + `systemOne: request is not JSON: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + } + const questions = QuestionList( + Object.entries( + payload.questions !== null && + typeof payload.questions === "object" && + !Array.isArray(payload.questions) + ? payload.questions + : {}, + ).map(([id, question]) => + typeof question === "object" && question !== null + ? { id, ...question } + : { id }, + ), + ); + if (questions instanceof type.errors) { + throw new SystemOneError( + "parse-error", + `systemOne: invalid questions: ${questions.summary}`, + ); + } + const state = payload.state; + if (state === undefined) { + throw new SystemOneError("parse-error", "systemOne: state is required"); + } + const images = + serialized.assets === undefined + ? undefined + : await loadImages(serialized.assets); + const config = withModel(input.config, input.model); + const result = await evaluate( + { + state: state as State, + questions, + ...(input.config !== undefined || input.model !== undefined + ? { config } + : {}), + }, + { + ...(options?.deps !== undefined ? { deps: options.deps } : {}), + ...(options?.onTelemetry !== undefined + ? { onTelemetry: options.onTelemetry } + : {}), + ...(images !== undefined ? { images } : {}), + }, + ); + if (result.fallback) return result; + return { + fallback: false, + answers: answersFrom(result.decisions) as SystemOneAnswers, + modelId: result.modelId, + backend: result.backend, + latencyMs: result.latencyMs, + ...(result.usage !== undefined ? { usage: result.usage } : {}), + }; +} + +export type { EvaluateDeps }; diff --git a/src/evaluate.ts b/src/evaluate.ts index 967e07f..adf5608 100644 --- a/src/evaluate.ts +++ b/src/evaluate.ts @@ -120,7 +120,18 @@ function checkAnswer( if (answer.noul === undefined) return `answer "${id}" is missing noul`; return undefined; } - if (answer.confidence === undefined) { + if (question.type === "box" && answer.type === "box") { + if (answer.box === undefined) return `answer "${id}" is missing box`; + const edges = answer.box; + for (const edge of ["top", "bottom", "left", "right"] as const) { + const value = edges[edge]; + if (!Number.isFinite(value) || value < 0 || value > 1) { + return `answer "${id}" box.${edge} ${String(value)} outside [0, 1]`; + } + } + return undefined; + } + if (answer.confidence == null) { return `answer "${id}" is missing confidence`; } if (question.type === "choice" && answer.type === "choice") { @@ -225,6 +236,15 @@ function envKey(name: string): string | undefined { export type EvaluateOptions = { onTelemetry?: (event: SystemOneTelemetryEvent) => void; deps?: EvaluateDeps; + /** Image parts already checked as PNG or JPEG. Omitted for JSON requests. */ + images?: { + assets: Record; + parts: { + name: string; + bytes: Uint8Array; + contentType: "image/png" | "image/jpeg"; + }[]; + }; }; /** @@ -345,10 +365,14 @@ export async function evaluate( }); } + const images = options?.images; const wire = WireRequest({ model, state: parsed.state, questions: toWireQuestions(parsed.questions), + ...(images !== undefined && images.parts.length > 0 + ? { assets: images.assets } + : {}), }); if (wire instanceof type.errors) { throw new SystemOneError( @@ -363,7 +387,13 @@ export async function evaluate( const response = await postEvaluate( endpoint.url, wire, - apiKey === undefined ? { timeoutMs } : { timeoutMs, apiKey }, + { + timeoutMs, + ...(apiKey === undefined ? {} : { apiKey }), + ...(images !== undefined && images.parts.length > 0 + ? { imageParts: images.parts } + : {}), + }, deps, ); data = response.data; diff --git a/src/index.ts b/src/index.ts index 7d7b4a0..9196de8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,9 @@ export { DEFAULT_TIMEOUT_MS } from "./config.js"; export type { EvaluateDeps } from "./client.js"; export { evaluate } from "./evaluate.js"; export type { EvaluateOptions } from "./evaluate.js"; +export { choice, noul, score, box, systemOne } from "./decisions.js"; +export type { SystemOneResult, SystemOneSuccess } from "./decisions.js"; +export { Image, image, withAssets } from "./assets.js"; export { createSystemOneAdapter, createSystemOneAdapterFactory, @@ -14,6 +17,8 @@ export { BooleanQuestion, ChoiceQuestion, Decision, + DecisionQuestion, + DecisionQuestionRecord, Description, EndpointConfig, EvaluateConfig, diff --git a/src/schemas.ts b/src/schemas.ts index aabebd7..5385554 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -114,11 +114,60 @@ export const BooleanQuestion = type({ }).narrow((q) => nonEmpty(q.instructions)); export type BooleanQuestion = typeof BooleanQuestion.infer; +// Locates a region in an image. Edges come back as fractions of the image. +// Callers should set the model to `jev-preview`; the client does not require it. +export const BoxQuestion = type({ + id: "string", + type: "'box'", + instructions: Description, + "+": "reject", +}).narrow((q) => nonEmpty(q.instructions)); +export type BoxQuestion = typeof BoxQuestion.infer; + +// Keyed decisions questions: choice, noul, score, or box only. Extraction-style +// helpers are not part of this contract and fail the union. +export const DecisionQuestion = type({ + type: "'choice'", + instructions: "unknown", + criteria: "unknown", + "+": "reject", +}) + .or( + type({ + type: "'noul'", + "instructions?": "unknown", + "criteria?": "unknown", + "+": "reject", + }), + ) + .or( + type({ + type: "'score'", + instructions: "unknown", + criteria: "unknown", + "+": "reject", + }), + ) + .or( + type({ + type: "'box'", + instructions: "unknown", + "+": "reject", + }), + ); +export type DecisionQuestion = typeof DecisionQuestion.infer; + +export const DecisionQuestionRecord = type({ + "[string]": DecisionQuestion, +}).narrow((qs) => Object.keys(qs).length >= 1); +export type DecisionQuestionRecord = typeof DecisionQuestionRecord.infer; + // The question union a caller may ask in one evaluation. `.or` chains keep // each branch's literal `type`, so the union still discriminates on it. export const Question = ChoiceQuestion.or(ScoreQuestion) .or(NoulQuestion) - .or(BooleanQuestion); + .or(BooleanQuestion) + .or(BoxQuestion); export type Question = typeof Question.infer; // A non-empty question list with unique ids: a duplicate would make the @@ -189,18 +238,25 @@ export type EvaluateInput = typeof EvaluateInput.infer; /** One question in wire form: no id (the map key carries it). */ export const WireQuestion = type({ - type: "'noul' | 'choice' | 'score'", + type: "'noul' | 'choice' | 'score' | 'box'", instructions: Description, "criteria?": "unknown", "+": "reject", }); export type WireQuestion = typeof WireQuestion.infer; -/** The exact POST body: model + state + id-keyed questions. */ +const WireAsset = type({ + type: "'image'", + part: "string", + "+": "reject", +}); + +/** The exact POST body: model + state + id-keyed questions, plus image parts. */ export const WireRequest = type({ model: "string", state: State, questions: { "[string]": WireQuestion }, + "assets?": { "[string]": WireAsset }, "+": "reject", }).narrow((body) => Object.keys(body.questions).length >= 1); export type WireRequest = typeof WireRequest.infer; @@ -218,7 +274,9 @@ export function toWireQuestions( type: question.type === "boolean" ? "noul" : question.type, instructions: question.instructions, }; - if (question.criteria !== undefined) mapped.criteria = question.criteria; + if ("criteria" in question && question.criteria !== undefined) { + mapped.criteria = question.criteria; + } wire[question.id] = mapped; } return wire; @@ -250,13 +308,19 @@ export const APPROXIMATE_SUM_TOLERANCE = 0.02; // `type` must carry. No `"+": "reject"`: the backend may attach per-answer // metadata that `toDecision` strips. export const WireAnswer = type({ - type: "'noul' | 'choice' | 'score'", + type: "'noul' | 'choice' | 'score' | 'box'", "choice?": "string", "score?": "number", "noul?": Confidence, "probabilities?": ProbabilityMap, - "confidence?": Confidence, + "confidence?": Confidence.or("null"), "legend?": { "[string]": "string" }, + "box?": { + top: "number", + bottom: "number", + left: "number", + right: "number", + }, }); export type WireAnswer = typeof WireAnswer.infer; @@ -308,6 +372,21 @@ export const Decision = type({ confidence: Confidence, "+": "reject", }), + ) + .or( + type({ + id: "string", + type: "'box'", + box: { + top: "number", + bottom: "number", + left: "number", + right: "number", + "+": "reject", + }, + confidence: Confidence.or("null"), + "+": "reject", + }), ); export type Decision = typeof Decision.infer; @@ -329,6 +408,13 @@ export function toDecision(id: string, answer: WireAnswer): Decision { probabilities: answer.probabilities, confidence: answer.confidence, }; + } else if (answer.type === "box") { + candidate = { + id, + type: "box", + box: answer.box, + confidence: answer.confidence ?? null, + }; } else { candidate = { id, From 8942214e6cb348cc2ee58a683227f31e7cfb318a Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 16:28:44 -0700 Subject: [PATCH 4/9] fix: match image asset names as whole tokens Asset names no longer match inside JSON keys, box edges are checked on the adapter path, and the keyed API is documented. --- README.md | 53 +++++++++++++--- src/adapter.test.ts | 17 +++++ src/adapter.ts | 8 +++ src/assets.ts | 39 +++++++++++- src/decisions.test.ts | 143 ++++++++++++++++++++++++++++++++++++------ src/evaluate.ts | 10 +-- src/schemas.ts | 14 +++++ 7 files changed, 250 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 19ac324..cd2186c 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,10 @@ Boolean questions answer as `noul`, a probability from 0 to 1 that the answer is | Export | Description | | --------------------------------- | ----------------------------------------------------------------------------------- | | `evaluate(input, options?)` | Runs one evaluation. Returns `EvaluateResult`: decisions or a `FallbackResult`. | +| `systemOne(input, options?)` | Keyed decisions. Returns `answers` by question name, or the same fallback. | +| `choice`, `noul`, `score`, `box` | Build one keyed question. `box` locates a region and needs model `jev-preview`. | +| `image`, `Image.fromBase64` | PNG or JPEG bytes, a `Blob`, or base64. Other formats are rejected before send. | +| `withAssets` | Tag a template so interpolated images are sent. Also `withAssets.join`. | | `createSystemOneAdapter(config?)` | Returns a `ProviderAdapter` for `runInference`. `config` is an `EvaluateConfig`. | | `createSystemOneAdapterFactory` | Interchange `AdapterFactory` for `SIDECAR_ADAPTER_MANIFEST`; takes offering quirks. | | `SYSTEM_ONE_PROVIDER` | The `"corbits-system-one"` provider id. | @@ -82,14 +86,49 @@ Boolean questions answer as `noul`, a probability from 0 to 1 that the answer is ### Questions -| `type` | Fields | Decision fields | -| --------- | ------------------------------------------------------------------------------- | ------------------------------------------------ | -| `choice` | `instructions`, `criteria` (option name → description or `null`, 1–255 options) | `choice`, `probabilities`, `confidence` | -| `score` | `instructions`, `criteria` (2–10 descriptions) | `score`, `legend`, `probabilities`, `confidence` | -| `boolean` | `instructions`, optional `criteria: { true?, false? }` | `noul` | -| `noul` | Same as `boolean` | `noul` | +| `type` | Fields | Decision fields | +| --------- | ------------------------------------------------------------------------------- | ------------------------------------------------------------------- | +| `choice` | `instructions`, `criteria` (option name → description or `null`, 1–255 options) | `choice`, `probabilities`, `confidence` | +| `score` | `instructions`, `criteria` (2–10 descriptions) | `score`, `legend`, `probabilities`, `confidence` | +| `boolean` | `instructions`, optional `criteria: { true?, false? }` | `noul` | +| `noul` | Same as `boolean` | `noul` | +| `box` | `instructions` describing what to locate. Use model `jev-preview`. | `box` (`top`, `bottom`, `left`, `right` as fractions), `confidence` | -Every question has an `id`, unique within the list. +Every list question has an `id`, unique within the list. `box` edges outside `[0, 1]` are rejected. + +### Keyed questions + +`systemOne` takes the same state and a record of questions built with `choice`, `noul`, `score`, and `box`. The names are the answer keys. Those helpers compile to the list `evaluate` already sends. The same fallback reasons apply, and those failures do not throw. The helpers are also exported from `@corbits/system-one/decisions`. + +```ts +import { choice, noul, score, systemOne } from "@corbits/system-one"; + +const result = await systemOne({ + state: { document: "I was charged twice." }, + questions: { + category: choice("What is this?", { billing: null, technical: null }), + escalate: noul("Must this be escalated?"), + severity: score("How severe?", ["none", "cosmetic", "unusable"]), + }, +}); +``` + +On success, read `result.answers.category.choice` (plus `probabilities` and `confidence`), `result.answers.escalate.noul`, and `result.answers.severity.score`. + +`image()` accepts a `Blob` or `Uint8Array`. `Image.fromBase64()` accepts base64 file bytes. Only PNG and JPEG are sent. Place an image in `state`, instructions, or criteria, or interpolate it with `withAssets` (and `withAssets.join` for a list). An image in an ordinary template string is rejected before the request. `assets` maps a name to an image for plain strings; every name must appear as its own token in the request. A request with any image is `multipart/form-data` and sends `TypeSafe-Danger-Zone-Images: true`. Otherwise the body stays JSON. + +```ts +import { box, image, systemOne, withAssets } from "@corbits/system-one"; + +const photo = image(pngBytes); +const result = await systemOne({ + state: withAssets`Returned item: ${photo}`, + questions: { damage: box("The damaged area") }, + model: "jev-preview", +}); +``` + +`providerOptions.systemOne.questions` accepts this keyed record as well as a question list. ### Config diff --git a/src/adapter.test.ts b/src/adapter.test.ts index 13ee92e..595a72c 100644 --- a/src/adapter.test.ts +++ b/src/adapter.test.ts @@ -254,4 +254,21 @@ describe("adapter factory", () => { }; expect(body.questions["category"]?.type).toBe("choice"); }); + + test("parseJSONResponse rejects a box outside the image", () => { + const adapter = createSystemOneAdapter(); + expect(() => + adapter.parseJSONResponse( + JSON.stringify({ + answers: { + region: { + type: "box", + box: { top: -0.1, bottom: 0.4, left: 0.2, right: 0.5 }, + confidence: null, + }, + }, + }), + ), + ).toThrow(/outside \[0, 1\]/); + }); }); diff --git a/src/adapter.ts b/src/adapter.ts index 067158f..1ff8e1c 100644 --- a/src/adapter.ts +++ b/src/adapter.ts @@ -24,6 +24,7 @@ import { toDecision, toWireQuestions, WireResponseBody, + boxEdgesOutOfRange, EndpointConfig, EvaluateConfig, QuestionList, @@ -164,6 +165,13 @@ function parseDecisions( ); } try { + if (answer.type === "box") { + if (answer.box === undefined) { + throw new Error(`answer "${id}" is missing box`); + } + const violation = boxEdgesOutOfRange(id, answer.box); + if (violation !== undefined) throw new Error(violation); + } return toDecision(id, answer); } catch (cause) { throw new ProtocolMismatchError( diff --git a/src/assets.ts b/src/assets.ts index 2298da1..0b7bd65 100644 --- a/src/assets.ts +++ b/src/assets.ts @@ -196,7 +196,7 @@ export function serializeRequest( return value; }); const unreferenced = Object.keys(assets).filter( - (name) => !json.includes(JSON.stringify(name).slice(1, -1)), + (name) => !jsonValueHasToken(json, name), ); if (unreferenced.length > 0) { fail( @@ -221,3 +221,40 @@ export function serializeRequest( assets: sent.size === 0 ? undefined : Object.fromEntries(sent), }; } + +const IDENT = /[0-9A-Za-z_]/; + +/** True when `name` is its own token in a JSON string value, not a key or a substring. */ +function jsonValueHasToken(json: string, name: string): boolean { + let parsed: unknown; + try { + parsed = JSON.parse(json); + } catch { + return false; + } + return valueHasToken(parsed, name); +} + +function valueHasToken(value: unknown, name: string): boolean { + if (typeof value === "string") return hasToken(value, name); + if (Array.isArray(value)) + return value.some((item) => valueHasToken(item, name)); + if (typeof value === "object" && value !== null) { + return Object.values(value).some((item) => valueHasToken(item, name)); + } + return false; +} + +function hasToken(text: string, name: string): boolean { + if (name.length === 0) return false; + let from = 0; + while (from <= text.length) { + const at = text.indexOf(name, from); + if (at < 0) return false; + const before = at === 0 ? "" : (text[at - 1] ?? ""); + const after = text[at + name.length] ?? ""; + if (!IDENT.test(before) && !IDENT.test(after)) return true; + from = at + 1; + } + return false; +} diff --git a/src/decisions.test.ts b/src/decisions.test.ts index a8a537a..022b498 100644 --- a/src/decisions.test.ts +++ b/src/decisions.test.ts @@ -95,7 +95,7 @@ describe("systemOne", () => { }); test("sends images as multipart and rejects other formats first", async () => { - const bodies: { headers: Headers; text: string }[] = []; + const bodies: { headers: Headers; text: string; raw: Uint8Array }[] = []; const photo = image(png); const result = await systemOne( { @@ -110,12 +110,9 @@ describe("systemOne", () => { { deps: deps(async (_url, init) => { const headers = new Headers(init?.headers); - const body = init?.body; - const text = - typeof body === "string" - ? body - : await new Response(body as Blob).text(); - bodies.push({ headers, text }); + const raw = await bodyBytes(init?.body); + const text = new TextDecoder().decode(raw); + bodies.push({ headers, text, raw }); return new Response( JSON.stringify({ answers: { @@ -144,8 +141,12 @@ describe("systemOne", () => { if (sent === undefined) throw new Error("expected a request"); expect(sent.headers.get("TypeSafe-Danger-Zone-Images")).toBe("true"); expect(sent.headers.get("content-type")).toStartWith("multipart/form-data"); - expect(sent.text).toContain('name="request"'); - expect(sent.text).toContain('name="asset-0"'); + const requestPart = multipartRequest(sent.headers, sent.raw); + const assetKey = Object.keys(requestPart.assets)[0]; + if (assetKey === undefined) throw new Error("expected an asset"); + expect(requestPart.assets[assetKey]?.part).toBe("asset-0"); + expect(requestPart.parts["asset-0"]).toEqual(png); + expect(requestPart.jsonBytes.includes(png[0] ?? 0)).toBe(false); expect(sent.text).toContain("jev-preview"); let fetched = false; @@ -202,6 +203,19 @@ describe("systemOne", () => { ).rejects.toBeInstanceOf(SystemOneError); expect(fetched).toBe(false); + await expect( + systemOne( + { + state: { document: "state type" }, + questions: { damaged: noul("Is it damaged?") }, + assets: { t: image(png) }, + config: { apiKey: "test-key" }, + }, + { deps: transport }, + ), + ).rejects.toBeInstanceOf(SystemOneError); + + const taggedBodies: { headers: Headers; raw: Uint8Array }[] = []; const tagged = await systemOne( { state: withAssets`compare ${photo}`, @@ -211,19 +225,30 @@ describe("systemOne", () => { config: { apiKey: "test-key" }, }, { - deps: deps( - async () => - new Response( - JSON.stringify({ - answers: { same: { type: "noul", noul: 0.4 } }, - }), - { status: 200 }, - ), - ), + deps: deps(async (_url, init) => { + taggedBodies.push({ + headers: new Headers(init?.headers), + raw: await bodyBytes(init?.body), + }); + return new Response( + JSON.stringify({ + answers: { same: { type: "noul", noul: 0.4 } }, + }), + { status: 200 }, + ); + }), }, ); if (tagged.fallback) throw new Error(tagged.reason); expect(tagged.answers.same.noul).toBe(0.4); + const taggedSent = taggedBodies[0]; + if (taggedSent === undefined) throw new Error("expected a request"); + const taggedRequest = multipartRequest(taggedSent.headers, taggedSent.raw); + const taggedKey = Object.keys(taggedRequest.assets)[0]; + if (taggedKey === undefined) throw new Error("expected an asset"); + expect(taggedRequest.assets[taggedKey]?.part).toBe("asset-0"); + expect(taggedRequest.parts["asset-0"]).toEqual(jpeg); + expect(taggedRequest.jsonBytes.includes(jpeg[0] ?? 0)).toBe(false); const joined = withAssets.join([image(png), image(jpeg)], ", "); expect(String(joined)).toContain(" { ).rejects.toBeInstanceOf(SystemOneError); }); }); + +const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47]; + +async function bodyBytes(body: unknown): Promise { + if (body instanceof Uint8Array) return new Uint8Array(body); + if (typeof body === "string") return new TextEncoder().encode(body); + return new Uint8Array(await new Response(body as Blob).arrayBuffer()); +} + +function multipartRequest( + headers: Headers, + raw: Uint8Array, +): { + assets: Record; + parts: Record; + jsonBytes: Uint8Array; +} { + const contentType = headers.get("content-type") ?? ""; + const boundary = /boundary=([^;]+)/.exec(contentType)?.[1]; + if (boundary === undefined) throw new Error("missing multipart boundary"); + const marker = new TextEncoder().encode(`--${boundary}`); + const chunks: Uint8Array[] = []; + let from = 0; + while (from < raw.length) { + const at = indexOfBytes(raw, marker, from); + if (at < 0) break; + if (at > from) chunks.push(raw.subarray(from, at)); + from = at + marker.length; + } + const parts: Record = {}; + let requestJson = ""; + let jsonBytes = new Uint8Array(); + for (const chunk of chunks) { + const headerEnd = indexOfBytes( + chunk, + new TextEncoder().encode("\r\n\r\n"), + 0, + ); + if (headerEnd < 0) continue; + const header = new TextDecoder().decode(chunk.subarray(0, headerEnd)); + const name = /name="([^"]+)"/.exec(header)?.[1]; + if (name === undefined) continue; + let content = chunk.subarray(headerEnd + 4); + if ( + content.length >= 2 && + content[content.length - 2] === 13 && + content[content.length - 1] === 10 + ) { + content = content.subarray(0, content.length - 2); + } + if (name === "request") { + jsonBytes = new Uint8Array(content); + requestJson = new TextDecoder().decode(content); + } else { + parts[name] = new Uint8Array(content); + } + } + const payload = JSON.parse(requestJson) as { + assets?: Record; + }; + const pngAt = indexOfBytes(jsonBytes, Uint8Array.from(PNG_SIGNATURE), 0); + if (pngAt >= 0) throw new Error("PNG bytes leaked into the JSON part"); + return { assets: payload.assets ?? {}, parts, jsonBytes }; +} + +function indexOfBytes( + haystack: Uint8Array, + needle: Uint8Array, + from: number, +): number { + for (let index = from; index <= haystack.length - needle.length; index += 1) { + let match = true; + for (let offset = 0; offset < needle.length; offset += 1) { + if (haystack[index + offset] !== needle[offset]) { + match = false; + break; + } + } + if (match) return index; + } + return -1; +} diff --git a/src/evaluate.ts b/src/evaluate.ts index adf5608..55e6d1e 100644 --- a/src/evaluate.ts +++ b/src/evaluate.ts @@ -23,6 +23,7 @@ import { toWireQuestions, WireRequest, WireResponseBody, + boxEdgesOutOfRange, type Decision, type EvaluateResult, type FallbackReason, @@ -122,14 +123,7 @@ function checkAnswer( } if (question.type === "box" && answer.type === "box") { if (answer.box === undefined) return `answer "${id}" is missing box`; - const edges = answer.box; - for (const edge of ["top", "bottom", "left", "right"] as const) { - const value = edges[edge]; - if (!Number.isFinite(value) || value < 0 || value > 1) { - return `answer "${id}" box.${edge} ${String(value)} outside [0, 1]`; - } - } - return undefined; + return boxEdgesOutOfRange(id, answer.box); } if (answer.confidence == null) { return `answer "${id}" is missing confidence`; diff --git a/src/schemas.ts b/src/schemas.ts index 5385554..9e851f3 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -324,6 +324,20 @@ export const WireAnswer = type({ }); export type WireAnswer = typeof WireAnswer.infer; +/** Box edges are fractions of the image. Shared by `evaluate` and the adapter. */ +export function boxEdgesOutOfRange( + id: string, + box: { top: number; bottom: number; left: number; right: number }, +): string | undefined { + for (const edge of ["top", "bottom", "left", "right"] as const) { + const value = box[edge]; + if (!Number.isFinite(value) || value < 0 || value > 1) { + return `answer "${id}" box.${edge} ${String(value)} outside [0, 1]`; + } + } + return undefined; +} + // Token usage reported on the response envelope. export const WireUsage = type({ input_tokens: "number", From 38b00dc3ee441e619a1cafaf85d8fadf15f72ce5 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 26 Oct 2026 20:48:18 -0700 Subject: [PATCH 5/9] fix: scope asset names, reject inverted boxes, and open the image gate --- src/adapter.test.ts | 29 +++++++ src/assets.ts | 35 ++++++-- src/client.ts | 17 +++- src/decisions.test.ts | 194 ++++++++++++++++++++++++++++++++++++++++++ src/evaluate.test.ts | 72 ++++++++++++++++ src/schemas.ts | 6 ++ 6 files changed, 346 insertions(+), 7 deletions(-) diff --git a/src/adapter.test.ts b/src/adapter.test.ts index 595a72c..ac9238d 100644 --- a/src/adapter.test.ts +++ b/src/adapter.test.ts @@ -271,4 +271,33 @@ describe("adapter factory", () => { ), ).toThrow(/outside \[0, 1\]/); }); + + test("parseJSONResponse rejects an inverted box", () => { + const adapter = createSystemOneAdapter(); + const reject = (box: { + top: number; + bottom: number; + left: number; + right: number; + }) => + adapter.parseJSONResponse( + JSON.stringify({ + answers: { + region: { type: "box", box, confidence: null }, + }, + }), + ); + expect(() => + reject({ top: 0.8, bottom: 0.1, left: 0.2, right: 0.5 }), + ).toThrow(/not below/); + expect(() => + reject({ top: 0.2, bottom: 0.2, left: 0.1, right: 0.4 }), + ).toThrow(/not below/); + expect(() => + reject({ top: 0.1, bottom: 0.4, left: 0.9, right: 0.2 }), + ).toThrow(/not left of/); + expect(() => + reject({ top: 0.1, bottom: 0.4, left: 0.5, right: 0.5 }), + ).toThrow(/not left of/); + }); }); diff --git a/src/assets.ts b/src/assets.ts index 0b7bd65..0cf6a9d 100644 --- a/src/assets.ts +++ b/src/assets.ts @@ -224,7 +224,7 @@ export function serializeRequest( const IDENT = /[0-9A-Za-z_]/; -/** True when `name` is its own token in a JSON string value, not a key or a substring. */ +/** True when `name` is its own token in state, instructions, or criteria. */ function jsonValueHasToken(json: string, name: string): boolean { let parsed: unknown; try { @@ -232,7 +232,26 @@ function jsonValueHasToken(json: string, name: string): boolean { } catch { return false; } - return valueHasToken(parsed, name); + return requestReferencesName(parsed, name); +} + +function own(value: object, key: string): unknown { + if (!Object.hasOwn(value, key)) return undefined; + return Object.getOwnPropertyDescriptor(value, key)?.value; +} + +/** Question `type` and other fields are not references. Keys are not either. */ +function requestReferencesName(payload: unknown, name: string): boolean { + if (typeof payload !== "object" || payload === null) return false; + if (valueHasToken(own(payload, "state"), name)) return true; + const questions = own(payload, "questions"); + if (typeof questions !== "object" || questions === null) return false; + for (const question of Object.values(questions)) { + if (typeof question !== "object" || question === null) continue; + if (valueHasToken(own(question, "instructions"), name)) return true; + if (valueHasToken(own(question, "criteria"), name)) return true; + } + return false; } function valueHasToken(value: unknown, name: string): boolean { @@ -247,12 +266,16 @@ function valueHasToken(value: unknown, name: string): boolean { function hasToken(text: string, name: string): boolean { if (name.length === 0) return false; + // `` is one generated reference, not a use of an asset named `image`. + const visible = text.replace(//g, (reference) => + " ".repeat(reference.length), + ); let from = 0; - while (from <= text.length) { - const at = text.indexOf(name, from); + while (from <= visible.length) { + const at = visible.indexOf(name, from); if (at < 0) return false; - const before = at === 0 ? "" : (text[at - 1] ?? ""); - const after = text[at + name.length] ?? ""; + const before = at === 0 ? "" : (visible[at - 1] ?? ""); + const after = visible[at + name.length] ?? ""; if (!IDENT.test(before) && !IDENT.test(after)) return true; from = at + 1; } diff --git a/src/client.ts b/src/client.ts index 07b0c56..6b834bf 100644 --- a/src/client.ts +++ b/src/client.ts @@ -40,6 +40,18 @@ export type PostEvaluateResponse = { httpStatus: number; }; +function bodyHasBoxQuestion(body: unknown): boolean { + if (typeof body !== "object" || body === null) return false; + if (!Object.hasOwn(body, "questions")) return false; + const questions = Object.getOwnPropertyDescriptor(body, "questions")?.value; + if (typeof questions !== "object" || questions === null) return false; + return Object.values(questions).some((question) => { + if (typeof question !== "object" || question === null) return false; + if (!Object.hasOwn(question, "type")) return false; + return Object.getOwnPropertyDescriptor(question, "type")?.value === "box"; + }); +} + function encodeMultipart( payload: unknown, parts: NonNullable, @@ -152,7 +164,10 @@ export async function postEvaluate( "content-type": contentType, accept: "application/json", }; - if (imageParts !== undefined && imageParts.length > 0) { + if ( + (imageParts !== undefined && imageParts.length > 0) || + bodyHasBoxQuestion(body) + ) { headers["TypeSafe-Danger-Zone-Images"] = "true"; } if (options.apiKey !== undefined && options.apiKey !== "") { diff --git a/src/decisions.test.ts b/src/decisions.test.ts index 022b498..b7d7a4f 100644 --- a/src/decisions.test.ts +++ b/src/decisions.test.ts @@ -292,6 +292,200 @@ describe("systemOne", () => { ), ).rejects.toBeInstanceOf(SystemOneError); }); + + test("asset names match only state, instructions, and criteria", async () => { + let fetched = false; + const transport = deps(async () => { + fetched = true; + return new Response("{}", { status: 200 }); + }); + const unreferenced: { + state: unknown; + questions: Record; + assets: Record; + }[] = [ + { + state: "plain", + questions: { q: choice("look", { a: null }) }, + assets: { choice: image(png) }, + }, + { + state: "plain", + questions: { q: noul("look") }, + assets: { noul: image(png) }, + }, + { + state: "plain", + questions: { q: score("look", ["none", "some"]) }, + assets: { score: image(png) }, + }, + { + state: "plain", + questions: { q: box("the region") }, + assets: { box: image(png) }, + }, + { + state: { shot: image(png) }, + questions: { q: noul("look") }, + assets: { image: image(jpeg) }, + }, + { + state: { photo: "x" }, + questions: { q: noul("look") }, + assets: { photo: image(png) }, + }, + { + state: { "a3f1c2e8-found": "x" }, + questions: { q: noul("look") }, + assets: { found: image(png) }, + }, + { + state: "photograph", + questions: { q: noul("look") }, + assets: { photo: image(png) }, + }, + { + state: "plain", + questions: { q: choice("look", { photo: null }) }, + assets: { photo: image(png) }, + }, + ]; + for (const input of unreferenced) { + await expect( + systemOne( + { ...input, config: { apiKey: "test-key" } }, + { deps: transport }, + ), + ).rejects.toThrow(/not referenced/); + } + expect(fetched).toBe(false); + + const photo = image(png); + const named = await systemOne( + { + state: withAssets`see image ${photo}`, + questions: { q: noul("use choice") }, + assets: { image: image(jpeg), choice: image(png) }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async () => + Promise.resolve( + new Response( + JSON.stringify({ answers: { q: { type: "noul", noul: 0.2 } } }), + { status: 200 }, + ), + ), + ), + }, + ); + if (named.fallback) throw new Error(named.reason); + expect(named.answers.q.noul).toBe(0.2); + }); + + test("sends the images header for a box question with no upload", async () => { + const seen = { header: "", contentType: "" }; + const result = await systemOne( + { + state: "a form", + questions: { region: box("the signature") }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async (_url, init) => { + const headers = new Headers(init?.headers); + seen.header = headers.get("TypeSafe-Danger-Zone-Images") ?? ""; + seen.contentType = headers.get("content-type") ?? ""; + return new Response( + JSON.stringify({ + answers: { + region: { + type: "box", + box: { top: 0.1, bottom: 0.4, left: 0.2, right: 0.5 }, + confidence: 0.4, + }, + }, + }), + { status: 200 }, + ); + }), + }, + ); + expect(seen.header).toBe("true"); + expect(seen.contentType).toBe("application/json"); + if (result.fallback) throw new Error(result.detail ?? result.reason); + + const choiceSeen = { header: "unset" }; + await systemOne( + { + state: "plain", + questions: { category: choice("What?", { a: null }) }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async (_url, init) => { + choiceSeen.header = + new Headers(init?.headers).get("TypeSafe-Danger-Zone-Images") ?? ""; + return new Response( + JSON.stringify({ + answers: { + category: { + type: "choice", + choice: "a", + probabilities: { a: 1 }, + confidence: 0.5, + }, + }, + }), + { status: 200 }, + ); + }), + }, + ); + expect(choiceSeen.header).toBe(""); + }); + + test("rejects an inverted box instead of returning a decision", async () => { + const ask = (edges: { + top: number; + bottom: number; + left: number; + right: number; + }) => + systemOne( + { + state: "a form", + questions: { region: box("the signature") }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async () => + Promise.resolve( + new Response( + JSON.stringify({ + answers: { + region: { type: "box", box: edges, confidence: 0.5 }, + }, + }), + { status: 200 }, + ), + ), + ), + }, + ); + for (const edges of [ + { top: 0.8, bottom: 0.1, left: 0.2, right: 0.5 }, + { top: 0.4, bottom: 0.4, left: 0.2, right: 0.5 }, + { top: 0.1, bottom: 0.4, left: 0.9, right: 0.2 }, + { top: 0.1, bottom: 0.4, left: 0.5, right: 0.5 }, + { top: -0.1, bottom: 0.4, left: 0.2, right: 0.5 }, + ]) { + const result = await ask(edges); + expect(result.fallback).toBe(true); + if (!result.fallback) throw new Error("expected fallback"); + expect(result.reason).toBe("parse-error"); + } + }); }); const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47]; diff --git a/src/evaluate.test.ts b/src/evaluate.test.ts index 86b4d14..3e4f816 100644 --- a/src/evaluate.test.ts +++ b/src/evaluate.test.ts @@ -412,3 +412,75 @@ describe("key material", () => { expect(JSON.stringify(events)).not.toContain(secret); }); }); + +describe("box answers", () => { + const boxQuestion: Question[] = [ + { id: "region", type: "box", instructions: "the signature" }, + ]; + + function boxDeps( + box: { top: number; bottom: number; left: number; right: number }, + seen?: { header: string | null; contentType: string | null }, + ): EvaluateDeps { + return { + fetch: (_url, init) => { + if (seen !== undefined) { + const headers = new Headers(init?.headers); + seen.header = headers.get("TypeSafe-Danger-Zone-Images"); + seen.contentType = headers.get("content-type"); + } + return Promise.resolve( + new Response( + JSON.stringify({ + answers: { region: { type: "box", box, confidence: 0.4 } }, + }), + { status: 200 }, + ), + ); + }, + scheduler: createDefaultScheduler(), + }; + } + + test("an inverted box is a parse fallback, not a decision", async () => { + for (const box of [ + { top: 0.8, bottom: 0.1, left: 0.2, right: 0.5 }, + { top: 0.4, bottom: 0.4, left: 0.1, right: 0.4 }, + { top: 0.1, bottom: 0.4, left: 0.9, right: 0.2 }, + { top: 0.1, bottom: 0.4, left: 0.5, right: 0.5 }, + ]) { + const result = await evaluate( + { state: "form", questions: boxQuestion, config: { apiKey: "key" } }, + { deps: boxDeps(box) }, + ); + expect(result.fallback).toBe(true); + if (!result.fallback) throw new Error("expected fallback"); + expect(result.reason).toBe("parse-error"); + } + const outside = await evaluate( + { state: "form", questions: boxQuestion, config: { apiKey: "key" } }, + { + deps: boxDeps({ top: -0.1, bottom: 0.4, left: 0.2, right: 0.5 }), + }, + ); + expect(outside).toMatchObject({ fallback: true, reason: "parse-error" }); + if (!outside.fallback) throw new Error("expected fallback"); + expect(outside.detail).toContain("outside [0, 1]"); + }); + + test("a box question with no image parts still opens the images gate", async () => { + const seen = { + header: null as string | null, + contentType: null as string | null, + }; + const result = await evaluate( + { state: "form", questions: boxQuestion, config: { apiKey: "key" } }, + { + deps: boxDeps({ top: 0.1, bottom: 0.4, left: 0.2, right: 0.5 }, seen), + }, + ); + expect(seen.header).toBe("true"); + expect(seen.contentType).toBe("application/json"); + expect(result.fallback).toBe(false); + }); +}); diff --git a/src/schemas.ts b/src/schemas.ts index 9e851f3..7d94a13 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -335,6 +335,12 @@ export function boxEdgesOutOfRange( return `answer "${id}" box.${edge} ${String(value)} outside [0, 1]`; } } + if (box.top >= box.bottom) { + return `answer "${id}" box.top ${String(box.top)} is not below box.bottom ${String(box.bottom)}`; + } + if (box.left >= box.right) { + return `answer "${id}" box.left ${String(box.left)} is not left of box.right ${String(box.right)}`; + } return undefined; } From 06bc6276f1f6eb5938ce7bd6288a6a0f56958940 Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 16:08:37 -0700 Subject: [PATCH 6/9] feat: add Clairic extraction beside keyed decisions --- package.json | 5 + src/decisions.test.ts | 1 + src/decisions.ts | 18 ++- src/extraction.test.ts | 157 +++++++++++++++++++ src/extraction.ts | 346 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 519 insertions(+), 8 deletions(-) create mode 100644 src/extraction.test.ts create mode 100644 src/extraction.ts diff --git a/package.json b/package.json index 22b619a..39861e4 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,11 @@ "corbits-system-one-src": "./src/decisions.ts", "types": "./dist/decisions.d.ts", "default": "./dist/decisions.js" + }, + "./extraction": { + "corbits-system-one-src": "./src/extraction.ts", + "types": "./dist/extraction.d.ts", + "default": "./dist/extraction.js" } }, "publishConfig": { diff --git a/src/decisions.test.ts b/src/decisions.test.ts index b7d7a4f..0f8860f 100644 --- a/src/decisions.test.ts +++ b/src/decisions.test.ts @@ -285,6 +285,7 @@ describe("systemOne", () => { systemOne( { state: "x", + // @ts-expect-error extraction questions are not decisions questions: { field: { type: "extract", instructions: "pull it" } }, config: { apiKey: "k" }, }, diff --git a/src/decisions.ts b/src/decisions.ts index 7470e22..6aa0d38 100644 --- a/src/decisions.ts +++ b/src/decisions.ts @@ -8,6 +8,7 @@ import { DecisionQuestionRecord, QuestionList, type Decision, + type DecisionQuestion, type Description, type EvaluateConfig, type FallbackResult, @@ -125,13 +126,14 @@ export type SystemOneResult> = | SystemOneSuccess | FallbackResult; -export type SystemOneInput> = { - state: unknown; - questions: Questions; - assets?: Record; - model?: string; - config?: EvaluateConfig; -}; +export type SystemOneInput> = + { + state: unknown; + questions: Questions; + assets?: Record; + model?: string; + config?: EvaluateConfig; + }; function withModel( config: EvaluateConfig | undefined, @@ -218,7 +220,7 @@ async function loadImages( * `fallback` and `reason` and do not throw. */ export async function systemOne< - const Questions extends Record, + const Questions extends Record, >( input: SystemOneInput, options?: Pick, diff --git a/src/extraction.test.ts b/src/extraction.test.ts new file mode 100644 index 0000000..2362e85 --- /dev/null +++ b/src/extraction.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, test } from "bun:test"; + +import { choice, systemOne } from "./decisions"; +import { date, estimate, extract, list, number, text } from "./extraction"; + +describe("extract", () => { + test("posts Clairic fields and maps answers", async () => { + const calls: { url: string; init: RequestInit | undefined }[] = []; + const result = await extract( + { + state: "Invoice INV-2291 issued 2026-09-02, net 30.", + fields: { + invoice_number: text(), + due_date: date("Issue date plus the net payment days"), + total: number({ + decimals: 2, + unit: "USD", + instructions: "Amount due including shipping", + }), + contacts: list("text"), + days_to_pay: estimate({ + min: 0, + max: 120, + step: 1, + instructions: "Days until paid", + }), + }, + }, + { + accountId: "acct", + token: "tok", + fetch: async (url, init) => { + calls.push({ url: String(url), init }); + return new Response( + JSON.stringify({ + result: { + answers: { + invoice_number: { value: "INV-2291", start: 8, end: 16 }, + due_date: { value: "2026-10-02" }, + total: { value: 1080, unit: "USD" }, + contacts: { value: ["Dana Ruiz"] }, + days_to_pay: { median: 30, interval: { p10: 12, p90: 45 } }, + }, + }, + }), + { status: 200 }, + ); + }, + }, + ); + + expect(calls).toHaveLength(1); + const call = calls[0]; + if (call === undefined) throw new Error("expected a fetch"); + expect(call.url).toBe( + "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clairic", + ); + expect(new Headers(call.init?.headers).get("authorization")).toBe( + "Bearer tok", + ); + expect(JSON.parse(String(call.init?.body))).toEqual({ + model: "clairic", + state: "Invoice INV-2291 issued 2026-09-02, net 30.", + fields: { + invoice_number: { type: "text" }, + due_date: { + type: "date", + instructions: "Issue date plus the net payment days", + }, + total: { + type: "number", + decimals: 2, + unit: "USD", + instructions: "Amount due including shipping", + }, + contacts: { type: "list", items: "text" }, + days_to_pay: { + type: "estimate", + min: 0, + max: 120, + step: 1, + instructions: "Days until paid", + }, + }, + }); + if (result.fallback) throw new Error("expected answers"); + expect(result.answers.invoice_number).toEqual({ + value: "INV-2291", + start: 8, + end: 16, + }); + expect(result.answers.due_date.value).toBe("2026-10-02"); + expect(result.answers.total.value).toBe(1080); + expect(result.answers.contacts.value).toEqual(["Dana Ruiz"]); + expect(result.answers.days_to_pay.median).toBe(30); + expect(result.answers.days_to_pay.interval).toEqual({ p10: 12, p90: 45 }); + }); + + test("falls back without credentials, on non-2xx, and on bad JSON", async () => { + const missing = await extract( + { state: "x", fields: { invoice_number: text() } }, + { + fetch: () => Promise.reject(new Error("fetch must not be called")), + }, + ); + expect(missing).toMatchObject({ fallback: true, reason: "no-key" }); + + const http = await extract( + { state: "x", fields: { invoice_number: text() } }, + { + accountId: "acct", + token: "tok", + fetch: async () => new Response("no", { status: 500 }), + }, + ); + expect(http).toMatchObject({ + fallback: true, + reason: "http-error", + httpStatus: 500, + }); + + const bad = await extract( + { state: "x", fields: { invoice_number: text() } }, + { + accountId: "acct", + token: "tok", + fetch: async () => new Response("not-json", { status: 200 }), + }, + ); + expect(bad).toMatchObject({ fallback: true, reason: "parse-error" }); + }); +}); + +describe("extraction and decisions stay disjoint", () => { + test("number is not a decision question and choice is not an extraction field", () => { + async function probe(): Promise { + await systemOne({ + state: "x", + questions: { + category: choice("What is this?", { billing: null }), + // @ts-expect-error number fields are not decision questions + total: number(), + }, + }); + + await extract({ + state: "x", + fields: { + invoice_number: text(), + // @ts-expect-error choice questions are not extraction fields + category: choice("What is this?", { billing: null }), + }, + }); + } + expect(probe).toBeFunction(); + }); +}); diff --git a/src/extraction.ts b/src/extraction.ts new file mode 100644 index 0000000..93c1318 --- /dev/null +++ b/src/extraction.ts @@ -0,0 +1,346 @@ +import type { FallbackReason, FallbackResult } from "./schemas.js"; + +const CLAIRIC_MODEL = "@cf/cloudflare/clairic"; + +export type TextField = { type: "text"; instructions?: string }; +export type DateField = { type: "date"; instructions?: string }; +export type NumberField = { + type: "number"; + decimals?: number; + unit?: string; + instructions?: string; +}; +export type ListField = { + type: "list"; + items: Items; +}; +export type EstimateField = { + type: "estimate"; + min?: number; + max?: number; + step?: number; + instructions?: string; +}; + +export type ExtractionField = + | TextField + | DateField + | NumberField + | ListField + | EstimateField; + +export type TextAnswer = { + value: string | null; + start?: number; + end?: number; +}; + +export type DateAnswer = { value: string | null }; + +export type NumberAnswer = { value: number | null; unit?: string }; + +export type ListAnswer = { value: string[] | null }; + +export type EstimateAnswer = { + median: number | null; + interval?: { p10: number; p90: number }; +}; + +type AnswerOf = Field extends { type: "text" } + ? TextAnswer + : Field extends { type: "date" } + ? DateAnswer + : Field extends { type: "number" } + ? NumberAnswer + : Field extends { type: "list" } + ? ListAnswer + : Field extends { type: "estimate" } + ? EstimateAnswer + : never; + +export type ExtractionAnswers> = + { + [Key in keyof Fields]: AnswerOf; + }; + +export type ExtractionSuccess> = + { + fallback: false; + answers: ExtractionAnswers; + modelId: string; + latencyMs: number; + }; + +export type ExtractionResult> = + | ExtractionSuccess + | FallbackResult; + +export type ExtractInput> = { + state: unknown; + fields: Fields; +}; + +export type ExtractFetch = ( + input: string | URL | Request, + init?: RequestInit, +) => Promise; + +export type ExtractOptions = { + accountId?: string; + token?: string; + fetch?: ExtractFetch; +}; + +function definedString(value: string | undefined): string | undefined { + if (value === undefined || value === "") return undefined; + return value; +} + +function text(instructions?: string): TextField { + return instructions === undefined + ? { type: "text" } + : { type: "text", instructions }; +} + +function date(instructions?: string): DateField { + return instructions === undefined + ? { type: "date" } + : { type: "date", instructions }; +} + +function number(options?: { + decimals?: number; + unit?: string; + instructions?: string; +}): NumberField { + const field: NumberField = { type: "number" }; + if (options?.decimals !== undefined) field.decimals = options.decimals; + if (options?.unit !== undefined) field.unit = options.unit; + if (options?.instructions !== undefined) + field.instructions = options.instructions; + return field; +} + +function list(items: Items): ListField { + return { type: "list", items }; +} + +function estimate(options?: { + min?: number; + max?: number; + step?: number; + instructions?: string; +}): EstimateField { + const field: EstimateField = { type: "estimate" }; + if (options?.min !== undefined) field.min = options.min; + if (options?.max !== undefined) field.max = options.max; + if (options?.step !== undefined) field.step = options.step; + if (options?.instructions !== undefined) + field.instructions = options.instructions; + return field; +} + +export { date, estimate, list, number, text }; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function numberOrUndefined(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) + ? value + : undefined; +} + +function textAnswer(raw: unknown): TextAnswer { + if (typeof raw === "string") return { value: raw }; + if (raw === null || raw === undefined) return { value: null }; + if (!isRecord(raw)) return { value: null }; + const direct = + typeof raw["value"] === "string" + ? raw["value"] + : typeof raw["text"] === "string" + ? raw["text"] + : null; + const offsets = isRecord(raw["offsets"]) ? raw["offsets"] : undefined; + const start = + numberOrUndefined(raw["start"]) ?? + (offsets !== undefined ? numberOrUndefined(offsets["start"]) : undefined); + const end = + numberOrUndefined(raw["end"]) ?? + (offsets !== undefined ? numberOrUndefined(offsets["end"]) : undefined); + const answer: TextAnswer = { value: direct }; + if (start !== undefined) answer.start = start; + if (end !== undefined) answer.end = end; + return answer; +} + +function dateAnswer(raw: unknown): DateAnswer { + if (typeof raw === "string") return { value: raw }; + if (raw === null || raw === undefined) return { value: null }; + if (!isRecord(raw)) return { value: null }; + const value = raw["value"] ?? raw["date"]; + return { value: typeof value === "string" ? value : null }; +} + +function numberAnswer(raw: unknown): NumberAnswer { + if (typeof raw === "number" && Number.isFinite(raw)) return { value: raw }; + if (raw === null || raw === undefined || !isRecord(raw)) + return { value: null }; + const value = numberOrUndefined(raw["value"] ?? raw["number"]); + const unit = typeof raw["unit"] === "string" ? raw["unit"] : undefined; + const answer: NumberAnswer = { value: value ?? null }; + if (unit !== undefined) answer.unit = unit; + return answer; +} + +function listAnswer(raw: unknown): ListAnswer { + if (Array.isArray(raw)) { + return { + value: raw.filter((item): item is string => typeof item === "string"), + }; + } + if (raw === null || raw === undefined || !isRecord(raw)) + return { value: null }; + const value = raw["value"] ?? raw["items"]; + if (value === null) return { value: null }; + if (!Array.isArray(value)) return { value: null }; + return { + value: value.filter((item): item is string => typeof item === "string"), + }; +} + +function estimateAnswer(raw: unknown): EstimateAnswer { + if (raw === null || raw === undefined || !isRecord(raw)) { + return { median: null }; + } + const median = numberOrUndefined(raw["median"] ?? raw["value"]); + const intervalRaw = raw["interval"]; + let interval: { p10: number; p90: number } | undefined; + if (Array.isArray(intervalRaw) && intervalRaw.length >= 2) { + const p10 = numberOrUndefined(intervalRaw[0]); + const p90 = numberOrUndefined(intervalRaw[1]); + if (p10 !== undefined && p90 !== undefined) interval = { p10, p90 }; + } else if (isRecord(intervalRaw)) { + const p10 = numberOrUndefined(intervalRaw["p10"] ?? intervalRaw["low"]); + const p90 = numberOrUndefined(intervalRaw["p90"] ?? intervalRaw["high"]); + if (p10 !== undefined && p90 !== undefined) interval = { p10, p90 }; + } + const answer: EstimateAnswer = { median: median ?? null }; + if (interval !== undefined) answer.interval = interval; + return answer; +} + +function mapAnswer(field: ExtractionField, raw: unknown): unknown { + if (field.type === "text") return textAnswer(raw); + if (field.type === "date") return dateAnswer(raw); + if (field.type === "number") return numberAnswer(raw); + if (field.type === "list") return listAnswer(raw); + return estimateAnswer(raw); +} + +function answersOf( + fields: Record, + raw: unknown, +): Record | undefined { + const envelope = + isRecord(raw) && isRecord(raw["result"]) ? raw["result"] : raw; + if (!isRecord(envelope) || !isRecord(envelope["answers"])) return undefined; + const source = envelope["answers"]; + const answers: Record = {}; + for (const [name, field] of Object.entries(fields)) { + answers[name] = mapAnswer(field, source[name]); + } + return answers; +} + +function fallback( + reason: FallbackReason, + started: number, + extra?: { httpStatus?: number; detail?: string }, +): FallbackResult { + const result: FallbackResult = { + fallback: true, + reason, + latencyMs: Math.max(0, Date.now() - started), + backendAttempted: "cloudflare-workers-ai", + }; + if (extra?.httpStatus !== undefined) result.httpStatus = extra.httpStatus; + if (extra?.detail !== undefined) result.detail = extra.detail; + return result; +} + +/** + * Keyed extraction against Cloudflare Workers AI Clairic. Missing credentials, + * non-2xx responses, and bodies that are not JSON return `fallback` and do + * not throw. + */ +export async function extract< + const Fields extends Record, +>( + input: ExtractInput, + options?: ExtractOptions, +): Promise> { + const started = Date.now(); + const accountId = + definedString(options?.accountId) ?? + definedString(process.env["CLOUDFLARE_ACCOUNT_ID"]); + const token = + definedString(options?.token) ?? + definedString(process.env["CLOUDFLARE_API_TOKEN"]); + if (accountId === undefined || token === undefined) { + return fallback("no-key", started, { + detail: + "no Cloudflare credentials; set accountId and token or CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN", + }); + } + + const url = `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/run/${CLAIRIC_MODEL}`; + const fetchImpl = options?.fetch ?? fetch; + let response: Response; + try { + response = await fetchImpl(url, { + method: "POST", + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + model: "clairic", + state: input.state, + fields: input.fields, + }), + }); + } catch (cause) { + return fallback("network", started, { + detail: cause instanceof Error ? cause.message : String(cause), + }); + } + + if (!response.ok) { + return fallback("http-error", started, { httpStatus: response.status }); + } + + let body: unknown; + try { + body = await response.json(); + } catch (cause) { + return fallback("parse-error", started, { + detail: cause instanceof Error ? cause.message : String(cause), + }); + } + + const answers = answersOf(input.fields, body); + if (answers === undefined) { + return fallback("parse-error", started, { + detail: "response has no answers", + }); + } + + return { + fallback: false, + answers: answers as ExtractionAnswers, + modelId: "clairic", + latencyMs: Math.max(0, Date.now() - started), + }; +} From c21f236dfb4213bd91a800d57a986d4de394826d Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 26 Oct 2026 20:46:03 -0700 Subject: [PATCH 7/9] fix: read Clairic spans, estimate bounds, and list items --- src/extraction.test.ts | 97 +++++++++++++ src/extraction.ts | 312 +++++++++++++++++++++++++++++++---------- 2 files changed, 334 insertions(+), 75 deletions(-) diff --git a/src/extraction.test.ts b/src/extraction.test.ts index 2362e85..97cddf0 100644 --- a/src/extraction.test.ts +++ b/src/extraction.test.ts @@ -96,6 +96,91 @@ describe("extract", () => { expect(result.answers.days_to_pay.interval).toEqual({ p10: 12, p90: 45 }); }); + test("maps documented spans, estimate bounds, and list items", async () => { + const result = await extract( + { + state: "Invoice INV-2291 issued 2026-09-02, net 30.", + fields: { + invoice_number: text(), + days_to_pay: estimate({ min: 0, max: 120 }), + amounts: list("number"), + due_dates: list("date"), + }, + }, + { + accountId: "acct", + token: "tok", + fetch: async () => + new Response( + JSON.stringify({ + result: { + answers: { + invoice_number: { + type: "text", + value: "INV-2291", + span: [8, 16], + confidence: 1, + p_null: 3.8e-10, + }, + days_to_pay: { + type: "estimate", + value: 30, + median: 30, + p10: 21, + p90: 38, + }, + amounts: { type: "list", value: [250, 80] }, + due_dates: { type: "list", value: ["2026-10-02"] }, + }, + }, + }), + { status: 200 }, + ), + }, + ); + + if (result.fallback) throw new Error(result.detail ?? "expected answers"); + expect(result.answers.invoice_number).toEqual({ + value: "INV-2291", + start: 8, + end: 16, + }); + expect(result.answers.days_to_pay).toEqual({ + median: 30, + interval: { p10: 21, p90: 38 }, + }); + const amounts: number[] | null = result.answers.amounts.value; + expect(amounts).toEqual([250, 80]); + expect(result.answers.due_dates.value).toEqual(["2026-10-02"]); + }); + + test("rejects a non-string text value instead of a successful null", async () => { + const result = await extract( + { state: "x", fields: { invoice_number: text() } }, + { + accountId: "acct", + token: "tok", + fetch: async () => + new Response( + JSON.stringify({ + result: { + answers: { + invoice_number: { type: "text", value: 1080 }, + }, + }, + }), + { status: 200 }, + ), + }, + ); + + expect(result).toMatchObject({ fallback: true, reason: "parse-error" }); + if (!result.fallback) { + const value: string | null = result.answers.invoice_number.value; + expect(value).not.toBeNull(); + } + }); + test("falls back without credentials, on non-2xx, and on bad JSON", async () => { const missing = await extract( { state: "x", fields: { invoice_number: text() } }, @@ -151,6 +236,18 @@ describe("extraction and decisions stay disjoint", () => { category: choice("What is this?", { billing: null }), }, }); + + const listed = await extract( + { state: "x", fields: { amounts: list("number") } }, + { fetch: () => Promise.reject(new Error("no fetch")) }, + ); + if (!listed.fallback) { + function stringsOnly(value: string[] | null): string[] | null { + return value; + } + // @ts-expect-error number lists are not string[] + stringsOnly(listed.answers.amounts.value); + } } expect(probe).toBeFunction(); }); diff --git a/src/extraction.ts b/src/extraction.ts index 93c1318..324e605 100644 --- a/src/extraction.ts +++ b/src/extraction.ts @@ -1,3 +1,5 @@ +import { type } from "arktype"; + import type { FallbackReason, FallbackResult } from "./schemas.js"; const CLAIRIC_MODEL = "@cf/cloudflare/clairic"; @@ -10,7 +12,9 @@ export type NumberField = { unit?: string; instructions?: string; }; -export type ListField = { +export type ListItemKind = "text" | "number" | "date"; + +export type ListField = { type: "list"; items: Items; }; @@ -39,7 +43,11 @@ export type DateAnswer = { value: string | null }; export type NumberAnswer = { value: number | null; unit?: string }; -export type ListAnswer = { value: string[] | null }; +type ListValue = Items extends "number" ? number : string; + +export type ListAnswer = { + value: ListValue[] | null; +}; export type EstimateAnswer = { median: number | null; @@ -52,8 +60,8 @@ type AnswerOf = Field extends { type: "text" } ? DateAnswer : Field extends { type: "number" } ? NumberAnswer - : Field extends { type: "list" } - ? ListAnswer + : Field extends { type: "list"; items: infer Items extends string } + ? ListAnswer : Field extends { type: "estimate" } ? EstimateAnswer : never; @@ -121,7 +129,9 @@ function number(options?: { return field; } -function list(items: Items): ListField { +function list( + items: Items, +): ListField { return { type: "list", items }; } @@ -146,110 +156,262 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } -function numberOrUndefined(value: unknown): number | undefined { - return typeof value === "number" && Number.isFinite(value) - ? value - : undefined; +type ParseError = { parseError: string }; + +const finiteNumber = type("number").narrow((value) => Number.isFinite(value)); + +const finiteOrNull = type("number | null").narrow( + (value) => value === null || Number.isFinite(value), +); + +const TextWire = type({ + "type?": "string", + "value?": "string | null", + "text?": "string | null", + "start?": finiteNumber, + "end?": finiteNumber, + "span?": [finiteNumber, finiteNumber], + "offsets?": { + "start?": finiteNumber, + "end?": finiteNumber, + }, +}); + +const DateWire = type("string").or( + type({ + "type?": "string", + "value?": "string | null", + "date?": "string | null", + }), +); + +const NumberWire = finiteNumber.or( + type({ + "type?": "string", + "value?": finiteOrNull, + "number?": finiteOrNull, + "unit?": "string", + }), +); + +const stringItems = type("string[]").or("null"); + +const numberItems = finiteNumber.array().or("null"); + +const StringListWire = type("string[]").or( + type({ + "type?": "string", + "value?": stringItems, + "items?": stringItems, + }), +); + +const NumberListWire = finiteNumber.array().or( + type({ + "type?": "string", + "value?": numberItems, + "items?": numberItems, + }), +); + +const EstimateInterval = type([finiteNumber, finiteNumber]).or( + type({ + "p10?": finiteNumber, + "low?": finiteNumber, + "p90?": finiteNumber, + "high?": finiteNumber, + }), +); + +const EstimateWire = type({ + "type?": "string", + "value?": finiteOrNull, + "median?": finiteOrNull, + "p10?": finiteNumber, + "p90?": finiteNumber, + "interval?": EstimateInterval, +}); + +function isParseError(value: unknown): value is ParseError { + return ( + typeof value === "object" && + value !== null && + "parseError" in value && + typeof value.parseError === "string" + ); } -function textAnswer(raw: unknown): TextAnswer { +function textAnswer(raw: unknown): TextAnswer | ParseError { if (typeof raw === "string") return { value: raw }; if (raw === null || raw === undefined) return { value: null }; - if (!isRecord(raw)) return { value: null }; - const direct = - typeof raw["value"] === "string" - ? raw["value"] - : typeof raw["text"] === "string" - ? raw["text"] + const parsed = TextWire(raw); + if (parsed instanceof type.errors) return { parseError: parsed.summary }; + const value = + typeof parsed.value === "string" + ? parsed.value + : typeof parsed.text === "string" + ? parsed.text : null; - const offsets = isRecord(raw["offsets"]) ? raw["offsets"] : undefined; - const start = - numberOrUndefined(raw["start"]) ?? - (offsets !== undefined ? numberOrUndefined(offsets["start"]) : undefined); - const end = - numberOrUndefined(raw["end"]) ?? - (offsets !== undefined ? numberOrUndefined(offsets["end"]) : undefined); - const answer: TextAnswer = { value: direct }; + const start = parsed.start ?? parsed.span?.[0] ?? parsed.offsets?.start; + const end = parsed.end ?? parsed.span?.[1] ?? parsed.offsets?.end; + const answer: TextAnswer = { value }; if (start !== undefined) answer.start = start; if (end !== undefined) answer.end = end; return answer; } -function dateAnswer(raw: unknown): DateAnswer { - if (typeof raw === "string") return { value: raw }; +function dateAnswer(raw: unknown): DateAnswer | ParseError { if (raw === null || raw === undefined) return { value: null }; - if (!isRecord(raw)) return { value: null }; - const value = raw["value"] ?? raw["date"]; - return { value: typeof value === "string" ? value : null }; + const parsed = DateWire(raw); + if (parsed instanceof type.errors) return { parseError: parsed.summary }; + if (typeof parsed === "string") return { value: parsed }; + const value = + typeof parsed.value === "string" + ? parsed.value + : typeof parsed.date === "string" + ? parsed.date + : null; + return { value }; } -function numberAnswer(raw: unknown): NumberAnswer { - if (typeof raw === "number" && Number.isFinite(raw)) return { value: raw }; - if (raw === null || raw === undefined || !isRecord(raw)) - return { value: null }; - const value = numberOrUndefined(raw["value"] ?? raw["number"]); - const unit = typeof raw["unit"] === "string" ? raw["unit"] : undefined; - const answer: NumberAnswer = { value: value ?? null }; - if (unit !== undefined) answer.unit = unit; +function numberAnswer(raw: unknown): NumberAnswer | ParseError { + if (raw === null || raw === undefined) return { value: null }; + const parsed = NumberWire(raw); + if (parsed instanceof type.errors) return { parseError: parsed.summary }; + if (typeof parsed === "number") return { value: parsed }; + const value = + typeof parsed.value === "number" + ? parsed.value + : typeof parsed.number === "number" + ? parsed.number + : null; + const answer: NumberAnswer = { value }; + if (parsed.unit !== undefined) answer.unit = parsed.unit; return answer; } -function listAnswer(raw: unknown): ListAnswer { - if (Array.isArray(raw)) { - return { - value: raw.filter((item): item is string => typeof item === "string"), - }; +function listValues( + parsed: + | { value?: string[] | number[] | null; items?: string[] | number[] | null } + | string[] + | number[], +): string[] | number[] | null { + if (Array.isArray(parsed)) return parsed; + if (parsed.value !== undefined) return parsed.value; + return parsed.items ?? null; +} + +function listAnswer( + items: ListItemKind, + raw: unknown, +): ListAnswer<"text"> | ListAnswer<"number"> | ParseError { + if (raw === null || raw === undefined) return { value: null }; + const parsed = (items === "number" ? NumberListWire : StringListWire)(raw); + if (parsed instanceof type.errors) return { parseError: parsed.summary }; + const values = listValues(parsed); + if (values === null) return { value: null }; + if (items === "number") { + if (!values.every((item) => typeof item === "number")) { + return { parseError: "list items must be numbers" }; + } + return { value: values }; } - if (raw === null || raw === undefined || !isRecord(raw)) - return { value: null }; - const value = raw["value"] ?? raw["items"]; - if (value === null) return { value: null }; - if (!Array.isArray(value)) return { value: null }; - return { - value: value.filter((item): item is string => typeof item === "string"), - }; + if (!values.every((item) => typeof item === "string")) { + return { parseError: "list items must be strings" }; + } + return { value: values }; } -function estimateAnswer(raw: unknown): EstimateAnswer { - if (raw === null || raw === undefined || !isRecord(raw)) { - return { median: null }; +function isNumberPair( + value: + | readonly [number, number] + | { p10?: number; low?: number; p90?: number; high?: number }, +): value is readonly [number, number] { + return Array.isArray(value); +} + +function estimateInterval(parsed: { + p10?: number; + p90?: number; + interval?: + | readonly [number, number] + | { p10?: number; low?: number; p90?: number; high?: number }; +}): { p10: number; p90: number } | undefined { + if (parsed.p10 !== undefined && parsed.p90 !== undefined) { + return { p10: parsed.p10, p90: parsed.p90 }; } - const median = numberOrUndefined(raw["median"] ?? raw["value"]); - const intervalRaw = raw["interval"]; - let interval: { p10: number; p90: number } | undefined; - if (Array.isArray(intervalRaw) && intervalRaw.length >= 2) { - const p10 = numberOrUndefined(intervalRaw[0]); - const p90 = numberOrUndefined(intervalRaw[1]); - if (p10 !== undefined && p90 !== undefined) interval = { p10, p90 }; - } else if (isRecord(intervalRaw)) { - const p10 = numberOrUndefined(intervalRaw["p10"] ?? intervalRaw["low"]); - const p90 = numberOrUndefined(intervalRaw["p90"] ?? intervalRaw["high"]); - if (p10 !== undefined && p90 !== undefined) interval = { p10, p90 }; + const interval = parsed.interval; + if (interval === undefined) return undefined; + if (isNumberPair(interval)) { + const low = interval[0]; + const high = interval[1]; + if (low === undefined || high === undefined) return undefined; + return { p10: low, p90: high }; } - const answer: EstimateAnswer = { median: median ?? null }; + const low = interval.p10 ?? interval.low; + const high = interval.p90 ?? interval.high; + if (low === undefined || high === undefined) return undefined; + return { p10: low, p90: high }; +} + +function estimateAnswer(raw: unknown): EstimateAnswer | ParseError { + if (raw === null || raw === undefined) return { median: null }; + const parsed = EstimateWire(raw); + if (parsed instanceof type.errors) return { parseError: parsed.summary }; + const median = parsed.median ?? parsed.value ?? null; + const answer: EstimateAnswer = { median }; + const interval = estimateInterval(parsed); if (interval !== undefined) answer.interval = interval; return answer; } -function mapAnswer(field: ExtractionField, raw: unknown): unknown { +function mapAnswer( + field: ExtractionField, + raw: unknown, +): + | TextAnswer + | DateAnswer + | NumberAnswer + | ListAnswer<"text"> + | ListAnswer<"number"> + | EstimateAnswer + | ParseError { if (field.type === "text") return textAnswer(raw); if (field.type === "date") return dateAnswer(raw); if (field.type === "number") return numberAnswer(raw); - if (field.type === "list") return listAnswer(raw); + if (field.type === "list") return listAnswer(field.items, raw); return estimateAnswer(raw); } -function answersOf( - fields: Record, +function isExtractionAnswers>( + fields: Fields, + answers: Record, +): answers is ExtractionAnswers { + return Object.keys(fields).every((key) => Object.hasOwn(answers, key)); +} + +function answersOf>( + fields: Fields, raw: unknown, -): Record | undefined { +): ExtractionAnswers | ParseError { const envelope = isRecord(raw) && isRecord(raw["result"]) ? raw["result"] : raw; - if (!isRecord(envelope) || !isRecord(envelope["answers"])) return undefined; + if (!isRecord(envelope) || !isRecord(envelope["answers"])) { + return { parseError: "response has no answers" }; + } const source = envelope["answers"]; const answers: Record = {}; for (const [name, field] of Object.entries(fields)) { - answers[name] = mapAnswer(field, source[name]); + const parsed = mapAnswer(field, source[name]); + if (isParseError(parsed)) { + return { + parseError: `answer "${name}" failed schema validation: ${parsed.parseError}`, + }; + } + answers[name] = parsed; + } + if (!isExtractionAnswers(fields, answers)) { + return { parseError: "response has no answers" }; } return answers; } @@ -331,15 +493,15 @@ export async function extract< } const answers = answersOf(input.fields, body); - if (answers === undefined) { + if (isParseError(answers)) { return fallback("parse-error", started, { - detail: "response has no answers", + detail: answers.parseError.slice(0, 240), }); } return { fallback: false, - answers: answers as ExtractionAnswers, + answers, modelId: "clairic", latencyMs: Math.max(0, Date.now() - started), }; From 1843d050fd8ac35f14ad04bdf146880db9e47365 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 26 Oct 2026 21:30:45 -0700 Subject: [PATCH 8/9] fix: type unreferenced asset cases as decision questions --- src/decisions.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/decisions.test.ts b/src/decisions.test.ts index 0f8860f..65f3084 100644 --- a/src/decisions.test.ts +++ b/src/decisions.test.ts @@ -5,6 +5,7 @@ import { createDefaultScheduler } from "@intx/inference"; import { Image, image, withAssets } from "./assets"; import { box, choice, noul, score, systemOne } from "./decisions"; import { SystemOneError } from "./errors"; +import type { DecisionQuestion } from "./schemas"; const png = Uint8Array.from([ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, @@ -302,7 +303,7 @@ describe("systemOne", () => { }); const unreferenced: { state: unknown; - questions: Record; + questions: Record; assets: Record; }[] = [ { From e95e4df2ac64034249beda388c6af39b381974b9 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 26 Oct 2026 21:30:46 -0700 Subject: [PATCH 9/9] chore(release): 0.4.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 39861e4..ae5563c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@corbits/system-one", - "version": "0.3.2", + "version": "0.4.0", "description": "Typed-decision evaluation client for System One (Jev-class) models: arktype question/decision schemas, endpoint configuration with gateway override, and timeout/fallback semantics.", "keywords": [ "corbits",