diff --git a/README.md b/README.md index e1090c0..cd2186c 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,10 @@ Boolean questions answer as `noul`, a probability from 0 to 1 that the answer is | Export | Description | | --------------------------------- | ----------------------------------------------------------------------------------- | | `evaluate(input, options?)` | Runs one evaluation. Returns `EvaluateResult`: decisions or a `FallbackResult`. | +| `systemOne(input, options?)` | Keyed decisions. Returns `answers` by question name, or the same fallback. | +| `choice`, `noul`, `score`, `box` | Build one keyed question. `box` locates a region and needs model `jev-preview`. | +| `image`, `Image.fromBase64` | PNG or JPEG bytes, a `Blob`, or base64. Other formats are rejected before send. | +| `withAssets` | Tag a template so interpolated images are sent. Also `withAssets.join`. | | `createSystemOneAdapter(config?)` | Returns a `ProviderAdapter` for `runInference`. `config` is an `EvaluateConfig`. | | `createSystemOneAdapterFactory` | Interchange `AdapterFactory` for `SIDECAR_ADAPTER_MANIFEST`; takes offering quirks. | | `SYSTEM_ONE_PROVIDER` | The `"corbits-system-one"` provider id. | @@ -82,24 +86,59 @@ Boolean questions answer as `noul`, a probability from 0 to 1 that the answer is ### Questions -| `type` | Fields | Decision fields | -| --------- | ------------------------------------------------------------------------------- | ------------------------------------------------ | -| `choice` | `instructions`, `criteria` (option name → description or `null`, 1–255 options) | `choice`, `probabilities`, `confidence` | -| `score` | `instructions`, `criteria` (2–10 descriptions) | `score`, `legend`, `probabilities`, `confidence` | -| `boolean` | `instructions`, optional `criteria: { true?, false? }` | `noul` | -| `noul` | Same as `boolean` | `noul` | +| `type` | Fields | Decision fields | +| --------- | ------------------------------------------------------------------------------- | ------------------------------------------------------------------- | +| `choice` | `instructions`, `criteria` (option name → description or `null`, 1–255 options) | `choice`, `probabilities`, `confidence` | +| `score` | `instructions`, `criteria` (2–10 descriptions) | `score`, `legend`, `probabilities`, `confidence` | +| `boolean` | `instructions`, optional `criteria: { true?, false? }` | `noul` | +| `noul` | Same as `boolean` | `noul` | +| `box` | `instructions` describing what to locate. Use model `jev-preview`. | `box` (`top`, `bottom`, `left`, `right` as fractions), `confidence` | -Every question has an `id`, unique within the list. +Every list question has an `id`, unique within the list. `box` edges outside `[0, 1]` are rejected. + +### Keyed questions + +`systemOne` takes the same state and a record of questions built with `choice`, `noul`, `score`, and `box`. The names are the answer keys. Those helpers compile to the list `evaluate` already sends. The same fallback reasons apply, and those failures do not throw. The helpers are also exported from `@corbits/system-one/decisions`. + +```ts +import { choice, noul, score, systemOne } from "@corbits/system-one"; + +const result = await systemOne({ + state: { document: "I was charged twice." }, + questions: { + category: choice("What is this?", { billing: null, technical: null }), + escalate: noul("Must this be escalated?"), + severity: score("How severe?", ["none", "cosmetic", "unusable"]), + }, +}); +``` + +On success, read `result.answers.category.choice` (plus `probabilities` and `confidence`), `result.answers.escalate.noul`, and `result.answers.severity.score`. + +`image()` accepts a `Blob` or `Uint8Array`. `Image.fromBase64()` accepts base64 file bytes. Only PNG and JPEG are sent. Place an image in `state`, instructions, or criteria, or interpolate it with `withAssets` (and `withAssets.join` for a list). An image in an ordinary template string is rejected before the request. `assets` maps a name to an image for plain strings; every name must appear as its own token in the request. A request with any image is `multipart/form-data` and sends `TypeSafe-Danger-Zone-Images: true`. Otherwise the body stays JSON. + +```ts +import { box, image, systemOne, withAssets } from "@corbits/system-one"; + +const photo = image(pngBytes); +const result = await systemOne({ + state: withAssets`Returned item: ${photo}`, + questions: { damage: box("The damaged area") }, + model: "jev-preview", +}); +``` + +`providerOptions.systemOne.questions` accepts this keyed record as well as a question list. ### Config `input.config` for `evaluate`, or the argument to `createSystemOneAdapter`: -| Field | Type | Default | Description | -| ----------- | ---------------- | ---------------------- | -------------------------------------------------------------------------------- | -| `endpoint` | `EndpointConfig` | `{ kind: "official" }` | `official`, `gateway`, or `custom` with a `url`. Each takes an optional `model`. | -| `timeoutMs` | `number` | `1500` | Bound for the whole round trip. | -| `apiKey` | `string` | from the environment | Overrides the environment key. | +| Field | Type | Default | Description | +| ----------- | ---------------- | ---------------------- | ----------------------------------------------------------------------------------------------------------------- | +| `endpoint` | `EndpointConfig` | `{ kind: "official" }` | `official`, `gateway`, or `custom` with a `url`. Each takes an optional `model`. `custom` may set `auth: "none"`. | +| `timeoutMs` | `number` | `1500` | Bound for the whole round trip. | +| `apiKey` | `string` | from the environment | Overrides the environment key. | ### Endpoints and keys @@ -107,19 +146,20 @@ Every question has an `id`, unique within the list. | ---------- | ---------------------------------------------------- | ----------------- | ---------------------------------------------- | | `official` | `https://api.typesafe.ai/v1/systemone` | `jev-latest` | `TYPESAFE_API_KEY` | | `gateway` | `https://ai-gateway.vercel.sh/typesafe/v1/systemone` | `typesafe-ai/jev` | `AI_GATEWAY_API_KEY`, then `VERCEL_OIDC_TOKEN` | -| `custom` | `url` | `typesafe-ai/jev` | `SYSTEM_ONE_API_KEY` | +| `custom` | `url` | `typesafe-ai/jev` | `SYSTEM_ONE_API_KEY`, unless `auth: "none"` | `SYSTEM_ONE_API_KEY` is also a supported alias on every endpoint, read after the endpoint's own key. +A custom endpoint with `auth: "none"` does not require a key and does not send `Authorization`. Official and gateway always require a key. Any other `auth` value is invalid input and throws. ### Fallback reasons -| `reason` | When | -| ------------- | ------------------------------------------------------------- | -| `no-key` | No key in `apiKey` or the environment. No request is sent. | -| `timeout` | The round trip, including the body read, exceeds `timeoutMs`. | -| `network` | The request fails before a response. | -| `http-error` | Non-2xx response. `httpStatus` holds the status. | -| `parse-error` | The body is not JSON or fails validation. | +| `reason` | When | +| ------------- | -------------------------------------------------------------------------------------------------------------- | +| `no-key` | No key in `apiKey` or the environment, and the endpoint is not custom with `auth: "none"`. No request is sent. | +| `timeout` | The round trip, including the body read, exceeds `timeoutMs`. | +| `network` | The request fails before a response. | +| `http-error` | Non-2xx response. `httpStatus` holds the status. | +| `parse-error` | The body is not JSON or fails validation. | `detail` carries the underlying message. @@ -208,7 +248,7 @@ The hub offering's `quirks` are validated and all optional: } ``` -Without `endpoint`, the adapter posts to the catalog provider's `baseURL` + `/systemone`, the host the offering's credential was issued for. A `model` quirk (top level or on `endpoint`) is the model sent on the wire, so one catalog model can be served by providers that name it differently. `questions` and `state` are per-call defaults; `providerOptions.systemOne` on a call wins field by field. Unknown keys throw. The adapter sends the bearer sentinel and the harness injects the offering's credential. +Without `endpoint`, the adapter posts to the catalog provider's `baseURL` + `/systemone`, the host the offering's credential was issued for. A `model` quirk (top level or on `endpoint`) is the model sent on the wire, so one catalog model can be served by providers that name it differently. `questions` and `state` are per-call defaults; `providerOptions.systemOne` on a call wins field by field. Unknown keys throw. The adapter sends the bearer sentinel and the harness injects the offering's credential, except a custom endpoint with `auth: "none"`, which omits `Authorization` so the host does not attach a key. | Provider | Provider `baseURL` | `model` quirk | | ----------------- | ------------------------------------------ | ----------------------------- | @@ -221,7 +261,7 @@ Without `endpoint`, the adapter posts to the catalog provider's `baseURL` + `/sy - `createSystemOneAdapterFactory` without an `endpoint` quirk now posts to the catalog provider's `baseURL` + `/systemone` instead of the official TypeSafe URL. Set the provider `baseURL` to the API root (for example `https://api.typesafe.ai/v1`), or keep the old behavior with `{"endpoint":{"kind":"official"}}`. The harness sends the offering's credential to whatever URL the adapter returns, so only set an explicit `endpoint` on a host that credential belongs to. - A `model` quirk now overrides the model the harness passes (the catalog canonical name). -- `createSystemOneAdapter` and `evaluate` are unchanged. +- `createSystemOneAdapter` and `evaluate` accept `auth: "none"` on a custom endpoint and omit `Authorization`. Any other `auth` value throws `SystemOneError` before a request is built. ## Upgrading from 0.1 diff --git a/package.json b/package.json index d181084..ae5563c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@corbits/system-one", - "version": "0.3.2", + "version": "0.4.0", "description": "Typed-decision evaluation client for System One (Jev-class) models: arktype question/decision schemas, endpoint configuration with gateway override, and timeout/fallback semantics.", "keywords": [ "corbits", @@ -32,6 +32,16 @@ "corbits-system-one-src": "./src/index.ts", "types": "./dist/index.d.ts", "default": "./dist/index.js" + }, + "./decisions": { + "corbits-system-one-src": "./src/decisions.ts", + "types": "./dist/decisions.d.ts", + "default": "./dist/decisions.js" + }, + "./extraction": { + "corbits-system-one-src": "./src/extraction.ts", + "types": "./dist/extraction.d.ts", + "default": "./dist/extraction.js" } }, "publishConfig": { diff --git a/src/adapter.test.ts b/src/adapter.test.ts index 41fefdb..ac9238d 100644 --- a/src/adapter.test.ts +++ b/src/adapter.test.ts @@ -9,6 +9,7 @@ import { createSystemOneAdapter, createSystemOneAdapterFactory, } from "./adapter"; +import { SystemOneError } from "./errors"; describe("adapter", () => { function turns(): ConversationTurn[] { @@ -122,6 +123,54 @@ describe("adapter factory", () => { { role: "user", timestamp: 0, content: [{ type: "text", text: "Hi" }] }, ]; + test("buildRequest omits Authorization for custom auth none and keeps the sentinel otherwise", () => { + const turns = (): ConversationTurn[] => [ + { role: "user", timestamp: 0, content: [{ type: "text", text: "Hi" }] }, + ]; + const keyless = createSystemOneAdapter({ + endpoint: { + kind: "custom", + url: "https://local.test/eval", + auth: "none", + }, + }).buildRequest(turns(), "jev-latest", {}); + expect(keyless.headers["authorization"]).toBeUndefined(); + expect(Object.keys(keyless.headers)).not.toContain("authorization"); + + const official = createSystemOneAdapter({ + endpoint: { kind: "official" }, + }).buildRequest(turns(), "jev-latest", {}); + expect(official.headers["authorization"]).toBe(BEARER_CREDENTIAL_SENTINEL); + + const gateway = createSystemOneAdapter({ + endpoint: { kind: "gateway" }, + }).buildRequest(turns(), "jev-latest", {}); + expect(gateway.headers["authorization"]).toBe(BEARER_CREDENTIAL_SENTINEL); + + const keyedCustom = createSystemOneAdapter({ + endpoint: { kind: "custom", url: "https://h.test/e" }, + }).buildRequest(turns(), "jev-latest", {}); + expect(keyedCustom.headers["authorization"]).toBe( + BEARER_CREDENTIAL_SENTINEL, + ); + }); + + test("invalid auth throws before a request is built", () => { + let thrown: unknown; + try { + createSystemOneAdapter({ + endpoint: { + kind: "custom", + url: "https://h.test/e", + auth: "bearer", + }, + } as unknown as Parameters[0]); + } catch (cause) { + thrown = cause; + } + expect(thrown).toBeInstanceOf(SystemOneError); + }); + test("no quirks routes to the provider baseURL with sentinel auth", () => { const request = createSystemOneAdapterFactory(source).buildRequest( turns(), @@ -183,4 +232,72 @@ describe("adapter factory", () => { /model must not be empty/, ); }); + + test("providerOptions.systemOne accepts keyed questions", () => { + const adapter = createSystemOneAdapter(); + const request = adapter.buildRequest([], "jev-latest", { + providerOptions: { + systemOne: { + state: "ticket", + questions: { + category: { + type: "choice", + instructions: "What?", + criteria: { a: null }, + }, + }, + }, + }, + }); + const body = JSON.parse(request.body) as { + questions: Record; + }; + expect(body.questions["category"]?.type).toBe("choice"); + }); + + test("parseJSONResponse rejects a box outside the image", () => { + const adapter = createSystemOneAdapter(); + expect(() => + adapter.parseJSONResponse( + JSON.stringify({ + answers: { + region: { + type: "box", + box: { top: -0.1, bottom: 0.4, left: 0.2, right: 0.5 }, + confidence: null, + }, + }, + }), + ), + ).toThrow(/outside \[0, 1\]/); + }); + + test("parseJSONResponse rejects an inverted box", () => { + const adapter = createSystemOneAdapter(); + const reject = (box: { + top: number; + bottom: number; + left: number; + right: number; + }) => + adapter.parseJSONResponse( + JSON.stringify({ + answers: { + region: { type: "box", box, confidence: null }, + }, + }), + ); + expect(() => + reject({ top: 0.8, bottom: 0.1, left: 0.2, right: 0.5 }), + ).toThrow(/not below/); + expect(() => + reject({ top: 0.2, bottom: 0.2, left: 0.1, right: 0.4 }), + ).toThrow(/not below/); + expect(() => + reject({ top: 0.1, bottom: 0.4, left: 0.9, right: 0.2 }), + ).toThrow(/not left of/); + expect(() => + reject({ top: 0.1, bottom: 0.4, left: 0.5, right: 0.5 }), + ).toThrow(/not left of/); + }); }); diff --git a/src/adapter.ts b/src/adapter.ts index ea67599..1ff8e1c 100644 --- a/src/adapter.ts +++ b/src/adapter.ts @@ -17,15 +17,18 @@ import type { } from "@intx/types/runtime"; import { resolveEndpoint } from "./config.js"; +import { SystemOneError } from "./errors.js"; import { Decision, + DecisionQuestionRecord, toDecision, toWireQuestions, WireResponseBody, + boxEdgesOutOfRange, EndpointConfig, + EvaluateConfig, QuestionList, State, - type EvaluateConfig, type WireUsage, } from "./schemas.js"; @@ -51,9 +54,11 @@ export const SYSTEM_ONE_PROVIDER = "corbits-system-one"; // `state` replaces the transcript-derived default; `questions` replaces the // single-question default. Both are validated, never cast — `QuestionList` // carries the same non-empty/unique-id contract as `EvaluateInput`. +const QuestionsInput = QuestionList.or(DecisionQuestionRecord); + const SystemOneProviderOptions = type({ "state?": State, - "questions?": QuestionList, + "questions?": QuestionsInput, "+": "reject", }); @@ -85,6 +90,24 @@ const DEFAULT_QUESTIONS: QuestionList = [ }, ]; +function asQuestionList( + questions: QuestionList | DecisionQuestionRecord, +): QuestionList { + if (Array.isArray(questions)) return questions; + const compiled = Object.entries(questions).map(([id, question]) => ({ + id, + ...question, + })); + const parsed = QuestionList(compiled); + if (parsed instanceof type.errors) { + throw new ProtocolMismatchError( + `${SYSTEM_ONE_PROVIDER} buildRequest: invalid providerOptions.systemOne questions: ${parsed.summary}`, + questions, + ); + } + return parsed; +} + function transcriptText(turns: ConversationTurn[]): string { const texts: string[] = []; for (const turn of turns) { @@ -142,6 +165,13 @@ function parseDecisions( ); } try { + if (answer.type === "box") { + if (answer.box === undefined) { + throw new Error(`answer "${id}" is missing box`); + } + const violation = boxEdgesOutOfRange(id, answer.box); + if (violation !== undefined) throw new Error(violation); + } return toDecision(id, answer); } catch (cause) { throw new ProtocolMismatchError( @@ -158,7 +188,9 @@ function parseDecisions( * host's single copy of `@intx/inference` (peer dependency, never vendored). * The optional `config` pins the endpoint (official by default, gateway, or * a custom URL) and the per-call model default; the inference call's own - * `model` argument wins for the request body. + * `model` argument wins for the request body. Official and gateway requests + * carry `authorization: BEARER_CREDENTIAL_SENTINEL` so the host injects a + * key. A custom endpoint with `auth: "none"` omits that header. */ function createSystemOneProviderAdapter( config?: EvaluateConfig, @@ -187,14 +219,18 @@ function createSystemOneProviderAdapter( ); } if (overrides.state !== undefined) state = overrides.state; - if (overrides.questions !== undefined) questions = overrides.questions; + if (overrides.questions !== undefined) { + questions = asQuestionList(overrides.questions); + } } const requestModel = model || endpoint.model; const headers: Record = { "content-type": "application/json", accept: "application/json", - authorization: BEARER_CREDENTIAL_SENTINEL, }; + if (endpoint.auth !== "none") { + headers["authorization"] = BEARER_CREDENTIAL_SENTINEL; + } return { url: endpoint.url, headers, @@ -264,7 +300,15 @@ function createSystemOneProviderAdapter( export function createSystemOneAdapter( config?: EvaluateConfig, ): ProviderAdapter { - return createSystemOneProviderAdapter(config); + if (config === undefined) return createSystemOneProviderAdapter(); + const parsed = EvaluateConfig(config); + if (parsed instanceof type.errors) { + throw new SystemOneError( + "parse-error", + `createSystemOneAdapter: invalid config: ${parsed.summary}`, + ); + } + return createSystemOneProviderAdapter(parsed); } // Deployment quirks a hub offering carries for this provider. `questions` diff --git a/src/assets.ts b/src/assets.ts new file mode 100644 index 0000000..0cf6a9d --- /dev/null +++ b/src/assets.ts @@ -0,0 +1,283 @@ +import { SystemOneError } from "./errors.js"; + +const CONTENT_TYPES = [ + ["image/png", [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]], + ["image/jpeg", [0xff, 0xd8, 0xff]], +] as const; + +const REFERENCE = //g; + +const IMAGE = Symbol.for("@typesafe-ai/sdk/image"); +const TEXT_WITH_ASSETS = Symbol.for("@typesafe-ai/sdk/text-with-assets"); + +const randomHex = (): string => + Array.from(crypto.getRandomValues(new Uint8Array(16)), (byte) => + byte.toString(16).padStart(2, "0"), + ).join(""); + +const isImage = (value: object): value is Image => IMAGE in value; + +const isTextWithAssets = (value: object): value is TextWithAssets => + TEXT_WITH_ASSETS in value; + +function fail(message: string): never { + throw new SystemOneError("parse-error", message); +} + +const toBytes = (data: ArrayBuffer | ArrayBufferView): Uint8Array => { + if ( + !ArrayBuffer.isView(data) && + Object.prototype.toString.call(data) !== "[object ArrayBuffer]" + ) { + fail("Image data must be file bytes or a Blob."); + } + return ArrayBuffer.isView(data) + ? new Uint8Array(data.buffer, data.byteOffset, data.byteLength) + : new Uint8Array(data); +}; + +export const contentTypeOf = (bytes: Uint8Array): string => { + for (const [contentType, signature] of CONTENT_TYPES) { + if (signature.every((byte, index) => bytes[index] === byte)) { + return contentType; + } + } + return "application/octet-stream"; +}; + +/** A PNG or JPEG image. Create one with `image` or `Image.fromBase64`. */ +export class Image { + readonly type = "image"; + readonly data: Uint8Array | Blob; + readonly #reference = ``; + + constructor(data: Blob | ArrayBuffer | ArrayBufferView) { + this.data = data instanceof Blob ? data : toBytes(data); + const size = + this.data instanceof Blob ? this.data.size : this.data.byteLength; + if (size === 0) fail("Image data is empty."); + } + + /** Decode base64 file bytes. Do not include a `data:` URL prefix. */ + static fromBase64(text: string): Image { + let binary: string; + try { + binary = atob(text); + } catch (cause) { + const error = new SystemOneError( + "parse-error", + "Image data must contain valid base64 text.", + ); + error.cause = cause; + throw error; + } + return new Image(Uint8Array.from(binary, (char) => char.charCodeAt(0))); + } + + toString(): string { + return this.#reference; + } + + toJSON(): string { + return this.#reference; + } +} + +Object.defineProperty(Image.prototype, IMAGE, { value: true }); + +class TextWithAssets { + readonly text: string; + readonly assets: Image[]; + + constructor(text: string, assets: Image[]) { + this.text = text; + this.assets = assets; + } + + toString(): string { + return this.text; + } + + toJSON(): string { + return this.text; + } +} + +Object.defineProperty(TextWithAssets.prototype, TEXT_WITH_ASSETS, { + value: true, +}); + +export type TextWithAssetsValue = TextWithAssets; + +/** Create an image from file bytes or a Blob. PNG and JPEG only, checked before send. */ +export const image = (data: Blob | ArrayBuffer | ArrayBufferView): Image => + new Image(data); + +const collectAssets = ( + value: unknown, + found: Map, + visited: Set, +): void => { + if (typeof value !== "object" || value === null || visited.has(value)) return; + visited.add(value); + if (isImage(value)) found.set(String(value), value); + else if (isTextWithAssets(value)) { + for (const asset of value.assets) found.set(String(asset), asset); + } else { + for (const item of Object.values(value)) + collectAssets(item, found, visited); + } +}; + +const textWithAssets = (text: string, values: unknown): TextWithAssets => { + const interpolated = new Map(); + collectAssets(values, interpolated, new Set()); + const assets = new Map(); + for (const [reference] of text.matchAll(REFERENCE)) { + const asset = interpolated.get(reference); + if (asset !== undefined) assets.set(reference, asset); + } + return new TextWithAssets(text, [...assets.values()]); +}; + +/** + * Tag a template so interpolated images are sent with the text. + * An image in an untagged template is rejected before the request is sent. + */ +export const withAssets = Object.assign( + (strings: TemplateStringsArray, ...values: unknown[]): TextWithAssets => + textWithAssets( + strings.reduce( + (joined, piece, index) => joined + String(values[index - 1]) + piece, + ), + values, + ), + { + join: (values: readonly unknown[], separator: string): TextWithAssets => + textWithAssets(values.join(separator), values), + }, +); + +export type SerializedRequest = { + json: string; + assets: Record | undefined; +}; + +/** Replace images with references and reject dangling or unused asset names. */ +export function serializeRequest( + payload: unknown, + assets: Record = {}, +): SerializedRequest { + const names = new Map( + Object.entries(assets).map(([name, asset]) => [asset, name]), + ); + const known = new Map( + Object.values(assets).map((asset) => [String(asset), asset]), + ); + const json = JSON.stringify(payload, function (key, value: unknown) { + if (typeof value === "number" && !Number.isFinite(value)) + return String(value); + const holder = this as Record; + const original = holder[key]; + if (typeof original !== "object" || original === null) return value; + if (isImage(original)) { + const name = names.get(original); + if (name !== undefined) return name; + known.set(String(original), original); + } else if (isTextWithAssets(original)) { + let text = original.text; + for (const asset of original.assets) { + const name = names.get(asset); + if (name === undefined) known.set(String(asset), asset); + else text = text.replaceAll(String(asset), name); + } + return text; + } + return value; + }); + const unreferenced = Object.keys(assets).filter( + (name) => !jsonValueHasToken(json, name), + ); + if (unreferenced.length > 0) { + fail( + `Assets not referenced in the request: ${unreferenced.join(", ")}. Write each name in \`assets\` into state, question instructions, or criteria.`, + ); + } + const sent = new Map(Object.entries(assets)); + const missing = new Set(); + for (const [reference] of json.matchAll(REFERENCE)) { + if (sent.has(reference)) continue; + const asset = known.get(reference); + if (asset === undefined) missing.add(reference); + else sent.set(reference, asset); + } + if (missing.size > 0) { + fail( + `Image references without an attached image: ${[...missing].join(", ")}. Tag template literals that interpolate images with \`withAssets\`.`, + ); + } + return { + json, + assets: sent.size === 0 ? undefined : Object.fromEntries(sent), + }; +} + +const IDENT = /[0-9A-Za-z_]/; + +/** True when `name` is its own token in state, instructions, or criteria. */ +function jsonValueHasToken(json: string, name: string): boolean { + let parsed: unknown; + try { + parsed = JSON.parse(json); + } catch { + return false; + } + return requestReferencesName(parsed, name); +} + +function own(value: object, key: string): unknown { + if (!Object.hasOwn(value, key)) return undefined; + return Object.getOwnPropertyDescriptor(value, key)?.value; +} + +/** Question `type` and other fields are not references. Keys are not either. */ +function requestReferencesName(payload: unknown, name: string): boolean { + if (typeof payload !== "object" || payload === null) return false; + if (valueHasToken(own(payload, "state"), name)) return true; + const questions = own(payload, "questions"); + if (typeof questions !== "object" || questions === null) return false; + for (const question of Object.values(questions)) { + if (typeof question !== "object" || question === null) continue; + if (valueHasToken(own(question, "instructions"), name)) return true; + if (valueHasToken(own(question, "criteria"), name)) return true; + } + return false; +} + +function valueHasToken(value: unknown, name: string): boolean { + if (typeof value === "string") return hasToken(value, name); + if (Array.isArray(value)) + return value.some((item) => valueHasToken(item, name)); + if (typeof value === "object" && value !== null) { + return Object.values(value).some((item) => valueHasToken(item, name)); + } + return false; +} + +function hasToken(text: string, name: string): boolean { + if (name.length === 0) return false; + // `` is one generated reference, not a use of an asset named `image`. + const visible = text.replace(//g, (reference) => + " ".repeat(reference.length), + ); + let from = 0; + while (from <= visible.length) { + const at = visible.indexOf(name, from); + if (at < 0) return false; + const before = at === 0 ? "" : (visible[at - 1] ?? ""); + const after = visible[at + name.length] ?? ""; + if (!IDENT.test(before) && !IDENT.test(after)) return true; + from = at + 1; + } + return false; +} diff --git a/src/client.ts b/src/client.ts index 0d0e1eb..6b834bf 100644 --- a/src/client.ts +++ b/src/client.ts @@ -23,6 +23,11 @@ import { SystemOneError, TransportError } from "./errors.js"; export type PostEvaluateOptions = { timeoutMs?: number; apiKey?: string; + imageParts?: { + name: string; + bytes: Uint8Array; + contentType: "image/png" | "image/jpeg"; + }[]; }; /** Transport dependencies: the `fetch` to POST with and the timeout scheduler. */ @@ -35,6 +40,73 @@ export type PostEvaluateResponse = { httpStatus: number; }; +function bodyHasBoxQuestion(body: unknown): boolean { + if (typeof body !== "object" || body === null) return false; + if (!Object.hasOwn(body, "questions")) return false; + const questions = Object.getOwnPropertyDescriptor(body, "questions")?.value; + if (typeof questions !== "object" || questions === null) return false; + return Object.values(questions).some((question) => { + if (typeof question !== "object" || question === null) return false; + if (!Object.hasOwn(question, "type")) return false; + return Object.getOwnPropertyDescriptor(question, "type")?.value === "box"; + }); +} + +function encodeMultipart( + payload: unknown, + parts: NonNullable, +): { body: Uint8Array; contentType: string } { + const boundary = Array.from( + crypto.getRandomValues(new Uint8Array(16)), + (byte) => byte.toString(16).padStart(2, "0"), + ).join(""); + const encoder = new TextEncoder(); + const chunks: Uint8Array[] = []; + const fields: { + name: string; + contentType: string; + content: Uint8Array; + filename?: string; + }[] = [ + { + name: "request", + contentType: "application/json", + content: encoder.encode(JSON.stringify(payload)), + }, + ...parts.map((part) => ({ + name: part.name, + contentType: part.contentType, + content: part.bytes, + filename: part.name, + })), + ]; + for (const field of fields) { + const disposition = + field.filename === undefined + ? `name="${field.name}"` + : `name="${field.name}"; filename="${field.filename}"`; + chunks.push( + encoder.encode( + `--${boundary}\r\nContent-Disposition: form-data; ${disposition}\r\nContent-Type: ${field.contentType}\r\n\r\n`, + ), + field.content, + encoder.encode("\r\n"), + ); + } + chunks.push(encoder.encode(`--${boundary}--\r\n`)); + const size = chunks.reduce((total, chunk) => total + chunk.byteLength, 0); + const body = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + body.set(chunk, offset); + offset += chunk.byteLength; + } + return { + body, + contentType: `multipart/form-data; boundary=${boundary}`, + }; +} + /** * POSTs an evaluation payload and returns the decoded JSON body. Sends * `Authorization: Bearer ` only when a key is present; aborts the @@ -52,9 +124,17 @@ export async function postEvaluate( options: PostEvaluateOptions, deps: EvaluateDeps, ): Promise { - let serialized: string; + let serialized: string | Uint8Array; + const imageParts = options.imageParts; + let contentType = "application/json"; try { - serialized = JSON.stringify(body); + if (imageParts !== undefined && imageParts.length > 0) { + const encoded = encodeMultipart(body, imageParts); + serialized = encoded.body; + contentType = encoded.contentType; + } else { + serialized = JSON.stringify(body); + } } catch { throw new SystemOneError( "parse-error", @@ -81,9 +161,15 @@ export async function postEvaluate( }; try { const headers: Record = { - "content-type": "application/json", + "content-type": contentType, accept: "application/json", }; + if ( + (imageParts !== undefined && imageParts.length > 0) || + bodyHasBoxQuestion(body) + ) { + headers["TypeSafe-Danger-Zone-Images"] = "true"; + } if (options.apiKey !== undefined && options.apiKey !== "") { headers["authorization"] = `Bearer ${options.apiKey}`; } diff --git a/src/config.ts b/src/config.ts index c496852..262b995 100644 --- a/src/config.ts +++ b/src/config.ts @@ -69,6 +69,8 @@ export type ResolvedEndpoint = { url: string; backend: "corbits-system-one" | "gateway" | "custom"; model: string; + /** `none` only when a custom endpoint explicitly opts out of a key. */ + auth: "key" | "none"; }; /** @@ -79,6 +81,8 @@ export type ResolvedEndpoint = { * distinguishes caller-routed traffic from the official endpoint — * `'gateway'` is reserved for the proxy quirks path. The model always * resolves here: a `custom` target without one sends `typesafe-ai/jev`. + * `auth` is `"none"` only when the caller set `auth: "none"` on a custom + * endpoint; every other target still requires a key. */ export function resolveEndpoint(endpoint?: EndpointConfig): ResolvedEndpoint { if (endpoint?.kind === "custom") { @@ -92,6 +96,7 @@ export function resolveEndpoint(endpoint?: EndpointConfig): ResolvedEndpoint { url: endpoint.url, backend: "custom", model: endpoint.model ?? GATEWAY_QUIRKS.model, + auth: endpoint.auth === "none" ? "none" : "key", }; } const kind = endpoint?.kind ?? "official"; @@ -108,5 +113,6 @@ export function resolveEndpoint(endpoint?: EndpointConfig): ResolvedEndpoint { url, backend: kind === "gateway" ? "gateway" : "corbits-system-one", model: endpoint?.model ?? quirks.model, + auth: "key", }; } diff --git a/src/decisions.test.ts b/src/decisions.test.ts new file mode 100644 index 0000000..65f3084 --- /dev/null +++ b/src/decisions.test.ts @@ -0,0 +1,573 @@ +import type { EvaluateDeps } from "./client"; +import { describe, expect, test } from "bun:test"; +import { createDefaultScheduler } from "@intx/inference"; + +import { Image, image, withAssets } from "./assets"; +import { box, choice, noul, score, systemOne } from "./decisions"; +import { SystemOneError } from "./errors"; +import type { DecisionQuestion } from "./schemas"; + +const png = Uint8Array.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, +]); +const jpeg = Uint8Array.from([0xff, 0xd8, 0xff, 0x00]); +const gif = Uint8Array.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61]); + +function deps(fetchImpl: EvaluateDeps["fetch"]): EvaluateDeps { + return { fetch: fetchImpl, scheduler: createDefaultScheduler() }; +} + +function okBody() { + return { + model: "jev-latest", + answers: { + category: { + type: "choice", + choice: "billing", + probabilities: { billing: 0.8, technical: 0.2 }, + confidence: 0.7, + }, + escalate: { type: "noul", noul: 0.2 }, + severity: { + type: "score", + score: 1, + legend: { "0": "none", "1": "cosmetic", "2": "unusable" }, + probabilities: { "0": 0.2, "1": 0.6, "2": 0.2 }, + confidence: 0.5, + }, + }, + }; +} + +describe("systemOne", () => { + test("returns named answers and falls back without throwing", async () => { + let calls = 0; + const result = await systemOne( + { + state: { document: "charged twice" }, + questions: { + category: choice("What is this?", { billing: null, technical: null }), + escalate: noul("Must this be escalated?"), + severity: score("How severe?", ["none", "cosmetic", "unusable"]), + }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async () => { + calls += 1; + return new Response(JSON.stringify(okBody()), { status: 200 }); + }), + }, + ); + expect(calls).toBe(1); + if (result.fallback) throw new Error("expected answers"); + expect(result.answers.category.choice).toBe("billing"); + expect(result.answers.category.probabilities.billing).toBe(0.8); + expect(result.answers.category.confidence).toBe(0.7); + expect(result.answers.escalate.noul).toBe(0.2); + expect(result.answers.severity.score).toBe(1); + + const fallback = await systemOne( + { + state: { document: "x" }, + questions: { escalate: noul("Must this be escalated?") }, + }, + { + deps: deps(() => Promise.reject(new Error("fetch must not be called"))), + }, + ); + expect(fallback.fallback).toBe(true); + if (!fallback.fallback) throw new Error("expected fallback"); + expect(fallback.reason).toBe("no-key"); + + const http = await systemOne( + { + state: { document: "x" }, + questions: { escalate: noul("Must this be escalated?") }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async () => new Response("no", { status: 500 })), + }, + ); + expect(http.fallback).toBe(true); + if (!http.fallback) throw new Error("expected fallback"); + expect(http.reason).toBe("http-error"); + }); + + test("sends images as multipart and rejects other formats first", async () => { + const bodies: { headers: Headers; text: string; raw: Uint8Array }[] = []; + const photo = image(png); + const result = await systemOne( + { + state: { photo }, + questions: { + region: box("The damaged area"), + damaged: noul("Is it damaged?"), + }, + model: "jev-preview", + config: { apiKey: "test-key" }, + }, + { + deps: deps(async (_url, init) => { + const headers = new Headers(init?.headers); + const raw = await bodyBytes(init?.body); + const text = new TextDecoder().decode(raw); + bodies.push({ headers, text, raw }); + return new Response( + JSON.stringify({ + answers: { + region: { + type: "box", + box: { top: 0.1, bottom: 0.4, left: 0.2, right: 0.5 }, + confidence: null, + }, + damaged: { type: "noul", noul: 0.9 }, + }, + }), + { status: 200 }, + ); + }), + }, + ); + if (result.fallback) throw new Error(result.reason); + expect(result.answers.region.box).toEqual({ + top: 0.1, + bottom: 0.4, + left: 0.2, + right: 0.5, + }); + expect(result.answers.region.confidence).toBeNull(); + const sent = bodies[0]; + if (sent === undefined) throw new Error("expected a request"); + expect(sent.headers.get("TypeSafe-Danger-Zone-Images")).toBe("true"); + expect(sent.headers.get("content-type")).toStartWith("multipart/form-data"); + const requestPart = multipartRequest(sent.headers, sent.raw); + const assetKey = Object.keys(requestPart.assets)[0]; + if (assetKey === undefined) throw new Error("expected an asset"); + expect(requestPart.assets[assetKey]?.part).toBe("asset-0"); + expect(requestPart.parts["asset-0"]).toEqual(png); + expect(requestPart.jsonBytes.includes(png[0] ?? 0)).toBe(false); + expect(sent.text).toContain("jev-preview"); + + let fetched = false; + let thrown: unknown; + try { + await systemOne( + { + state: image(gif), + questions: { damaged: noul("Is it damaged?") }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async () => { + fetched = true; + return new Response("{}", { status: 200 }); + }), + }, + ); + } catch (cause) { + thrown = cause; + } + expect(fetched).toBe(false); + expect(thrown).toBeInstanceOf(SystemOneError); + }); + + test("rejects a plain template image and requires every asset name", async () => { + const photo = image(jpeg); + let fetched = false; + const transport = deps(async () => { + fetched = true; + return new Response("{}", { status: 200 }); + }); + await expect( + systemOne( + { + state: `see ${photo}`, + questions: { damaged: noul("Is it damaged?") }, + config: { apiKey: "test-key" }, + }, + { deps: transport }, + ), + ).rejects.toBeInstanceOf(SystemOneError); + + await expect( + systemOne( + { + state: "no photo here", + questions: { damaged: noul("Is it damaged?") }, + assets: { "": image(png) }, + config: { apiKey: "test-key" }, + }, + { deps: transport }, + ), + ).rejects.toBeInstanceOf(SystemOneError); + expect(fetched).toBe(false); + + await expect( + systemOne( + { + state: { document: "state type" }, + questions: { damaged: noul("Is it damaged?") }, + assets: { t: image(png) }, + config: { apiKey: "test-key" }, + }, + { deps: transport }, + ), + ).rejects.toBeInstanceOf(SystemOneError); + + const taggedBodies: { headers: Headers; raw: Uint8Array }[] = []; + const tagged = await systemOne( + { + state: withAssets`compare ${photo}`, + questions: { + same: noul(withAssets`Does ${photo} match?`), + }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async (_url, init) => { + taggedBodies.push({ + headers: new Headers(init?.headers), + raw: await bodyBytes(init?.body), + }); + return new Response( + JSON.stringify({ + answers: { same: { type: "noul", noul: 0.4 } }, + }), + { status: 200 }, + ); + }), + }, + ); + if (tagged.fallback) throw new Error(tagged.reason); + expect(tagged.answers.same.noul).toBe(0.4); + const taggedSent = taggedBodies[0]; + if (taggedSent === undefined) throw new Error("expected a request"); + const taggedRequest = multipartRequest(taggedSent.headers, taggedSent.raw); + const taggedKey = Object.keys(taggedRequest.assets)[0]; + if (taggedKey === undefined) throw new Error("expected an asset"); + expect(taggedRequest.assets[taggedKey]?.part).toBe("asset-0"); + expect(taggedRequest.parts["asset-0"]).toEqual(jpeg); + expect(taggedRequest.jsonBytes.includes(jpeg[0] ?? 0)).toBe(false); + + const joined = withAssets.join([image(png), image(jpeg)], ", "); + expect(String(joined)).toContain(" please", + questions: { damaged: noul("Is damaged?") }, + assets: { + "": Image.fromBase64(Buffer.from(png).toString("base64")), + }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async (_url, init) => { + const text = await new Response(init?.body as Blob).text(); + expect(text).toContain(""); + expect(text).toContain("image/png"); + return new Response( + JSON.stringify({ + answers: { damaged: { type: "noul", noul: 0.1 } }, + }), + { status: 200 }, + ); + }), + }, + ); + if (named.fallback) throw new Error(named.reason); + expect(named.answers.damaged.noul).toBe(0.1); + }); + + test("rejects extraction questions", async () => { + await expect( + systemOne( + { + state: "x", + // @ts-expect-error extraction questions are not decisions + questions: { field: { type: "extract", instructions: "pull it" } }, + config: { apiKey: "k" }, + }, + { deps: deps(async () => new Response("{}", { status: 200 })) }, + ), + ).rejects.toBeInstanceOf(SystemOneError); + }); + + test("asset names match only state, instructions, and criteria", async () => { + let fetched = false; + const transport = deps(async () => { + fetched = true; + return new Response("{}", { status: 200 }); + }); + const unreferenced: { + state: unknown; + questions: Record; + assets: Record; + }[] = [ + { + state: "plain", + questions: { q: choice("look", { a: null }) }, + assets: { choice: image(png) }, + }, + { + state: "plain", + questions: { q: noul("look") }, + assets: { noul: image(png) }, + }, + { + state: "plain", + questions: { q: score("look", ["none", "some"]) }, + assets: { score: image(png) }, + }, + { + state: "plain", + questions: { q: box("the region") }, + assets: { box: image(png) }, + }, + { + state: { shot: image(png) }, + questions: { q: noul("look") }, + assets: { image: image(jpeg) }, + }, + { + state: { photo: "x" }, + questions: { q: noul("look") }, + assets: { photo: image(png) }, + }, + { + state: { "a3f1c2e8-found": "x" }, + questions: { q: noul("look") }, + assets: { found: image(png) }, + }, + { + state: "photograph", + questions: { q: noul("look") }, + assets: { photo: image(png) }, + }, + { + state: "plain", + questions: { q: choice("look", { photo: null }) }, + assets: { photo: image(png) }, + }, + ]; + for (const input of unreferenced) { + await expect( + systemOne( + { ...input, config: { apiKey: "test-key" } }, + { deps: transport }, + ), + ).rejects.toThrow(/not referenced/); + } + expect(fetched).toBe(false); + + const photo = image(png); + const named = await systemOne( + { + state: withAssets`see image ${photo}`, + questions: { q: noul("use choice") }, + assets: { image: image(jpeg), choice: image(png) }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async () => + Promise.resolve( + new Response( + JSON.stringify({ answers: { q: { type: "noul", noul: 0.2 } } }), + { status: 200 }, + ), + ), + ), + }, + ); + if (named.fallback) throw new Error(named.reason); + expect(named.answers.q.noul).toBe(0.2); + }); + + test("sends the images header for a box question with no upload", async () => { + const seen = { header: "", contentType: "" }; + const result = await systemOne( + { + state: "a form", + questions: { region: box("the signature") }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async (_url, init) => { + const headers = new Headers(init?.headers); + seen.header = headers.get("TypeSafe-Danger-Zone-Images") ?? ""; + seen.contentType = headers.get("content-type") ?? ""; + return new Response( + JSON.stringify({ + answers: { + region: { + type: "box", + box: { top: 0.1, bottom: 0.4, left: 0.2, right: 0.5 }, + confidence: 0.4, + }, + }, + }), + { status: 200 }, + ); + }), + }, + ); + expect(seen.header).toBe("true"); + expect(seen.contentType).toBe("application/json"); + if (result.fallback) throw new Error(result.detail ?? result.reason); + + const choiceSeen = { header: "unset" }; + await systemOne( + { + state: "plain", + questions: { category: choice("What?", { a: null }) }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async (_url, init) => { + choiceSeen.header = + new Headers(init?.headers).get("TypeSafe-Danger-Zone-Images") ?? ""; + return new Response( + JSON.stringify({ + answers: { + category: { + type: "choice", + choice: "a", + probabilities: { a: 1 }, + confidence: 0.5, + }, + }, + }), + { status: 200 }, + ); + }), + }, + ); + expect(choiceSeen.header).toBe(""); + }); + + test("rejects an inverted box instead of returning a decision", async () => { + const ask = (edges: { + top: number; + bottom: number; + left: number; + right: number; + }) => + systemOne( + { + state: "a form", + questions: { region: box("the signature") }, + config: { apiKey: "test-key" }, + }, + { + deps: deps(async () => + Promise.resolve( + new Response( + JSON.stringify({ + answers: { + region: { type: "box", box: edges, confidence: 0.5 }, + }, + }), + { status: 200 }, + ), + ), + ), + }, + ); + for (const edges of [ + { top: 0.8, bottom: 0.1, left: 0.2, right: 0.5 }, + { top: 0.4, bottom: 0.4, left: 0.2, right: 0.5 }, + { top: 0.1, bottom: 0.4, left: 0.9, right: 0.2 }, + { top: 0.1, bottom: 0.4, left: 0.5, right: 0.5 }, + { top: -0.1, bottom: 0.4, left: 0.2, right: 0.5 }, + ]) { + const result = await ask(edges); + expect(result.fallback).toBe(true); + if (!result.fallback) throw new Error("expected fallback"); + expect(result.reason).toBe("parse-error"); + } + }); +}); + +const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47]; + +async function bodyBytes(body: unknown): Promise { + if (body instanceof Uint8Array) return new Uint8Array(body); + if (typeof body === "string") return new TextEncoder().encode(body); + return new Uint8Array(await new Response(body as Blob).arrayBuffer()); +} + +function multipartRequest( + headers: Headers, + raw: Uint8Array, +): { + assets: Record; + parts: Record; + jsonBytes: Uint8Array; +} { + const contentType = headers.get("content-type") ?? ""; + const boundary = /boundary=([^;]+)/.exec(contentType)?.[1]; + if (boundary === undefined) throw new Error("missing multipart boundary"); + const marker = new TextEncoder().encode(`--${boundary}`); + const chunks: Uint8Array[] = []; + let from = 0; + while (from < raw.length) { + const at = indexOfBytes(raw, marker, from); + if (at < 0) break; + if (at > from) chunks.push(raw.subarray(from, at)); + from = at + marker.length; + } + const parts: Record = {}; + let requestJson = ""; + let jsonBytes = new Uint8Array(); + for (const chunk of chunks) { + const headerEnd = indexOfBytes( + chunk, + new TextEncoder().encode("\r\n\r\n"), + 0, + ); + if (headerEnd < 0) continue; + const header = new TextDecoder().decode(chunk.subarray(0, headerEnd)); + const name = /name="([^"]+)"/.exec(header)?.[1]; + if (name === undefined) continue; + let content = chunk.subarray(headerEnd + 4); + if ( + content.length >= 2 && + content[content.length - 2] === 13 && + content[content.length - 1] === 10 + ) { + content = content.subarray(0, content.length - 2); + } + if (name === "request") { + jsonBytes = new Uint8Array(content); + requestJson = new TextDecoder().decode(content); + } else { + parts[name] = new Uint8Array(content); + } + } + const payload = JSON.parse(requestJson) as { + assets?: Record; + }; + const pngAt = indexOfBytes(jsonBytes, Uint8Array.from(PNG_SIGNATURE), 0); + if (pngAt >= 0) throw new Error("PNG bytes leaked into the JSON part"); + return { assets: payload.assets ?? {}, parts, jsonBytes }; +} + +function indexOfBytes( + haystack: Uint8Array, + needle: Uint8Array, + from: number, +): number { + for (let index = from; index <= haystack.length - needle.length; index += 1) { + let match = true; + for (let offset = 0; offset < needle.length; offset += 1) { + if (haystack[index + offset] !== needle[offset]) { + match = false; + break; + } + } + if (match) return index; + } + return -1; +} diff --git a/src/decisions.ts b/src/decisions.ts new file mode 100644 index 0000000..6aa0d38 --- /dev/null +++ b/src/decisions.ts @@ -0,0 +1,306 @@ +import { type } from "arktype"; + +import { contentTypeOf, Image, serializeRequest } from "./assets.js"; +import type { EvaluateDeps } from "./client.js"; +import { evaluate, type EvaluateOptions } from "./evaluate.js"; +import { SystemOneError } from "./errors.js"; +import { + DecisionQuestionRecord, + QuestionList, + type Decision, + type DecisionQuestion, + type Description, + type EvaluateConfig, + type FallbackResult, + type State, +} from "./schemas.js"; + +type ChoiceCriteria = Record; + +type NoulCriteria = { + true?: Description; + false?: Description; +}; + +export function choice( + instructions: unknown, + criteria: Criteria, +): { type: "choice"; instructions: unknown; criteria: Criteria } { + if (Array.isArray(criteria)) { + throw new SystemOneError( + "parse-error", + "Choice criteria must be a map of labels to descriptions, not a list.", + ); + } + return { type: "choice", instructions, criteria }; +} + +export function noul( + instructions: unknown, + criteria?: NoulCriteria, +): { type: "noul"; instructions: unknown; criteria?: NoulCriteria } { + const question: { + type: "noul"; + instructions: unknown; + criteria?: NoulCriteria; + } = { type: "noul", instructions }; + if (criteria !== undefined) question.criteria = criteria; + return question; +} + +export function score( + instructions: unknown, + criteria: Criteria, +): { type: "score"; instructions: unknown; criteria: Criteria } { + if (!Array.isArray(criteria)) { + throw new SystemOneError( + "parse-error", + "Score criteria must be a list of descriptions indexed by score from zero, not a map.", + ); + } + return { type: "score", instructions, criteria }; +} + +/** + * Locate what `instructions` describe. The answer's `box` edges are fractions + * of the image. Box questions need the `jev-preview` model; pass it as + * `model` (or on the endpoint). This client does not reject other models. + */ +export function box(instructions: unknown): { + type: "box"; + instructions: unknown; +} { + return { type: "box", instructions }; +} + +type ChoiceAnswer = { + type: "choice"; + choice: keyof Criteria & string; + probabilities: Record; + confidence: number; +}; + +type NoulAnswer = { type: "noul"; noul: number }; + +type ScoreAnswer = { + type: "score"; + score: number; + legend: Record; + probabilities: Record; + confidence: number; +}; + +type BoxAnswer = { + type: "box"; + box: { top: number; bottom: number; left: number; right: number }; + confidence: number | null; +}; + +type AnswerOf = QuestionSpec extends { + type: "choice"; + criteria: infer Criteria extends ChoiceCriteria; +} + ? ChoiceAnswer + : QuestionSpec extends { type: "noul" } + ? NoulAnswer + : QuestionSpec extends { type: "score" } + ? ScoreAnswer + : QuestionSpec extends { type: "box" } + ? BoxAnswer + : never; + +export type SystemOneAnswers> = { + [Key in keyof Questions]: AnswerOf; +}; + +export type SystemOneSuccess> = { + fallback: false; + answers: SystemOneAnswers; + modelId: string; + backend: "corbits-system-one" | "gateway" | "custom"; + latencyMs: number; + usage?: { inputTokens: number; outputTokens: number }; +}; + +export type SystemOneResult> = + | SystemOneSuccess + | FallbackResult; + +export type SystemOneInput> = + { + state: unknown; + questions: Questions; + assets?: Record; + model?: string; + config?: EvaluateConfig; + }; + +function withModel( + config: EvaluateConfig | undefined, + model: string | undefined, +): EvaluateConfig { + const next: EvaluateConfig = {}; + if (config?.apiKey !== undefined) next.apiKey = config.apiKey; + if (config?.timeoutMs !== undefined) next.timeoutMs = config.timeoutMs; + const endpoint = config?.endpoint; + if (model !== undefined) { + next.endpoint = { ...(endpoint ?? { kind: "official" }), model }; + } else if (endpoint !== undefined) { + next.endpoint = endpoint; + } + return next; +} + +function answersFrom(decisions: Decision[]): Record { + const answers: Record = {}; + for (const decision of decisions) { + if (decision.type === "choice") { + answers[decision.id] = { + type: "choice", + choice: decision.choice, + probabilities: decision.probabilities, + confidence: decision.confidence, + }; + } else if (decision.type === "noul") { + answers[decision.id] = { type: "noul", noul: decision.noul }; + } else if (decision.type === "score") { + answers[decision.id] = { + type: "score", + score: decision.score, + legend: decision.legend, + probabilities: decision.probabilities, + confidence: decision.confidence, + }; + } else { + answers[decision.id] = { + type: "box", + box: decision.box, + confidence: decision.confidence, + }; + } + } + return answers; +} + +async function loadImages( + assets: Record, +): Promise> { + const parts: NonNullable["parts"] = []; + const wire: NonNullable["assets"] = {}; + let index = 0; + for (const [alias, asset] of Object.entries(assets)) { + let bytes: Uint8Array; + if (asset.data instanceof Blob) { + bytes = new Uint8Array(await asset.data.arrayBuffer()); + } else { + bytes = asset.data; + } + const contentType = contentTypeOf(bytes); + if (contentType !== "image/png" && contentType !== "image/jpeg") { + throw new SystemOneError( + "parse-error", + `Image "${alias}" must be PNG or JPEG.`, + ); + } + const name = `asset-${index}`; + parts.push({ + name, + bytes, + contentType, + }); + wire[alias] = { type: "image", part: name }; + index += 1; + } + return { assets: wire, parts }; +} + +/** + * Keyed decisions call. `choice`, `noul`, `score`, and `box` compile to the + * question list `evaluate` already sends. Transport failures return + * `fallback` and `reason` and do not throw. + */ +export async function systemOne< + const Questions extends Record, +>( + input: SystemOneInput, + options?: Pick, +): Promise> { + const parsed = DecisionQuestionRecord(input.questions); + if (parsed instanceof type.errors) { + throw new SystemOneError( + "parse-error", + `systemOne: invalid questions: ${parsed.summary}`, + ); + } + const serialized = serializeRequest( + { state: input.state, questions: input.questions }, + input.assets, + ); + let payload: { state?: unknown; questions?: unknown }; + try { + payload = JSON.parse(serialized.json) as { + state?: unknown; + questions?: unknown; + }; + } catch (cause) { + throw new SystemOneError( + "parse-error", + `systemOne: request is not JSON: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + } + const questions = QuestionList( + Object.entries( + payload.questions !== null && + typeof payload.questions === "object" && + !Array.isArray(payload.questions) + ? payload.questions + : {}, + ).map(([id, question]) => + typeof question === "object" && question !== null + ? { id, ...question } + : { id }, + ), + ); + if (questions instanceof type.errors) { + throw new SystemOneError( + "parse-error", + `systemOne: invalid questions: ${questions.summary}`, + ); + } + const state = payload.state; + if (state === undefined) { + throw new SystemOneError("parse-error", "systemOne: state is required"); + } + const images = + serialized.assets === undefined + ? undefined + : await loadImages(serialized.assets); + const config = withModel(input.config, input.model); + const result = await evaluate( + { + state: state as State, + questions, + ...(input.config !== undefined || input.model !== undefined + ? { config } + : {}), + }, + { + ...(options?.deps !== undefined ? { deps: options.deps } : {}), + ...(options?.onTelemetry !== undefined + ? { onTelemetry: options.onTelemetry } + : {}), + ...(images !== undefined ? { images } : {}), + }, + ); + if (result.fallback) return result; + return { + fallback: false, + answers: answersFrom(result.decisions) as SystemOneAnswers, + modelId: result.modelId, + backend: result.backend, + latencyMs: result.latencyMs, + ...(result.usage !== undefined ? { usage: result.usage } : {}), + }; +} + +export type { EvaluateDeps }; diff --git a/src/evaluate.test.ts b/src/evaluate.test.ts index 6db0e63..3e4f816 100644 --- a/src/evaluate.test.ts +++ b/src/evaluate.test.ts @@ -230,6 +230,113 @@ async function bearerFor( return seen.authorization; } +describe("custom auth none", () => { + const keyless: EndpointConfig = { + kind: "custom", + url: "https://local.test/eval", + auth: "none", + }; + + test("skips no-key and sends no Authorization, even with a key present", async () => { + process.env["SYSTEM_ONE_API_KEY"] = "legacy"; + let authorization: string | null = "unset"; + let called = false; + const result = await evaluate( + { + state: {}, + questions, + config: { endpoint: keyless, apiKey: "should-not-send" }, + }, + { + deps: { + fetch: (_input, init) => { + called = true; + authorization = new Headers(init?.headers).get("authorization"); + return Promise.resolve( + new Response(JSON.stringify({ answers: answers() }), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + }, + scheduler: createDefaultScheduler(), + }, + }, + ); + expect(called).toBe(true); + expect(authorization).toBeNull(); + if (result.fallback !== false) { + throw new Error(`expected decisions, got ${JSON.stringify(result)}`); + } + expect(result.decisions).toHaveLength(3); + }); + + test("official and gateway still fail closed with no-key and do not fetch", async () => { + const deps = { + fetch: () => Promise.reject(new Error("fetch must not be called")), + scheduler: createDefaultScheduler(), + }; + const official = await evaluate( + { state: {}, questions, config: { endpoint: { kind: "official" } } }, + { deps }, + ); + const gateway = await evaluate( + { state: {}, questions, config: { endpoint: { kind: "gateway" } } }, + { deps }, + ); + const custom = await evaluate( + { + state: {}, + questions, + config: { endpoint: { kind: "custom", url: "https://h.test/e" } }, + }, + { deps }, + ); + expect(official).toMatchObject({ fallback: true, reason: "no-key" }); + expect(gateway).toMatchObject({ fallback: true, reason: "no-key" }); + expect(custom).toMatchObject({ fallback: true, reason: "no-key" }); + }); + + test("auth other than none is invalid input and throws", async () => { + let thrown: unknown; + try { + await evaluate({ + state: {}, + questions, + config: { + endpoint: { + kind: "custom", + url: "https://h.test/e", + auth: "bearer", + } as unknown as EndpointConfig, + }, + }); + } catch (cause) { + thrown = cause; + } + expect(thrown).toBeInstanceOf(SystemOneError); + }); + + test("official rejects an auth field", async () => { + let thrown: unknown; + try { + await evaluate({ + state: {}, + questions, + config: { + endpoint: { + kind: "official", + auth: "none", + } as unknown as EndpointConfig, + }, + }); + } catch (cause) { + thrown = cause; + } + expect(thrown).toBeInstanceOf(SystemOneError); + }); +}); + describe("credential precedence", () => { test("official prefers TYPESAFE_API_KEY, then SYSTEM_ONE_API_KEY", async () => { process.env["SYSTEM_ONE_API_KEY"] = "legacy"; @@ -305,3 +412,75 @@ describe("key material", () => { expect(JSON.stringify(events)).not.toContain(secret); }); }); + +describe("box answers", () => { + const boxQuestion: Question[] = [ + { id: "region", type: "box", instructions: "the signature" }, + ]; + + function boxDeps( + box: { top: number; bottom: number; left: number; right: number }, + seen?: { header: string | null; contentType: string | null }, + ): EvaluateDeps { + return { + fetch: (_url, init) => { + if (seen !== undefined) { + const headers = new Headers(init?.headers); + seen.header = headers.get("TypeSafe-Danger-Zone-Images"); + seen.contentType = headers.get("content-type"); + } + return Promise.resolve( + new Response( + JSON.stringify({ + answers: { region: { type: "box", box, confidence: 0.4 } }, + }), + { status: 200 }, + ), + ); + }, + scheduler: createDefaultScheduler(), + }; + } + + test("an inverted box is a parse fallback, not a decision", async () => { + for (const box of [ + { top: 0.8, bottom: 0.1, left: 0.2, right: 0.5 }, + { top: 0.4, bottom: 0.4, left: 0.1, right: 0.4 }, + { top: 0.1, bottom: 0.4, left: 0.9, right: 0.2 }, + { top: 0.1, bottom: 0.4, left: 0.5, right: 0.5 }, + ]) { + const result = await evaluate( + { state: "form", questions: boxQuestion, config: { apiKey: "key" } }, + { deps: boxDeps(box) }, + ); + expect(result.fallback).toBe(true); + if (!result.fallback) throw new Error("expected fallback"); + expect(result.reason).toBe("parse-error"); + } + const outside = await evaluate( + { state: "form", questions: boxQuestion, config: { apiKey: "key" } }, + { + deps: boxDeps({ top: -0.1, bottom: 0.4, left: 0.2, right: 0.5 }), + }, + ); + expect(outside).toMatchObject({ fallback: true, reason: "parse-error" }); + if (!outside.fallback) throw new Error("expected fallback"); + expect(outside.detail).toContain("outside [0, 1]"); + }); + + test("a box question with no image parts still opens the images gate", async () => { + const seen = { + header: null as string | null, + contentType: null as string | null, + }; + const result = await evaluate( + { state: "form", questions: boxQuestion, config: { apiKey: "key" } }, + { + deps: boxDeps({ top: 0.1, bottom: 0.4, left: 0.2, right: 0.5 }, seen), + }, + ); + expect(seen.header).toBe("true"); + expect(seen.contentType).toBe("application/json"); + expect(result.fallback).toBe(false); + }); +}); diff --git a/src/evaluate.ts b/src/evaluate.ts index a0c096f..55e6d1e 100644 --- a/src/evaluate.ts +++ b/src/evaluate.ts @@ -23,6 +23,7 @@ import { toWireQuestions, WireRequest, WireResponseBody, + boxEdgesOutOfRange, type Decision, type EvaluateResult, type FallbackReason, @@ -120,7 +121,11 @@ function checkAnswer( if (answer.noul === undefined) return `answer "${id}" is missing noul`; return undefined; } - if (answer.confidence === undefined) { + if (question.type === "box" && answer.type === "box") { + if (answer.box === undefined) return `answer "${id}" is missing box`; + return boxEdgesOutOfRange(id, answer.box); + } + if (answer.confidence == null) { return `answer "${id}" is missing confidence`; } if (question.type === "choice" && answer.type === "choice") { @@ -225,6 +230,15 @@ function envKey(name: string): string | undefined { export type EvaluateOptions = { onTelemetry?: (event: SystemOneTelemetryEvent) => void; deps?: EvaluateDeps; + /** Image parts already checked as PNG or JPEG. Omitted for JSON requests. */ + images?: { + assets: Record; + parts: { + name: string; + bytes: Uint8Array; + contentType: "image/png" | "image/jpeg"; + }[]; + }; }; /** @@ -236,6 +250,9 @@ export type EvaluateOptions = { * `TYPESAFE_API_KEY` for official, `AI_GATEWAY_API_KEY` then * `VERCEL_OIDC_TOKEN` for gateway — then the `SYSTEM_ONE_API_KEY` alias, * then a fail-closed `'no-key'` fallback with zero fetch calls. + * A custom endpoint with `auth: "none"` skips that check and sends no + * `Authorization` header, even if a key is present. Official and gateway + * always require a key; a custom endpoint without `auth: "none"` does too. * A `timeoutMs` that is not a finite number >= 0 falls back to * `DEFAULT_TIMEOUT_MS`. Transport failures map to matching fallback reasons * (`'timeout'`, `'network'`, `'http-error'`, or `'parse-error'` for a 2xx @@ -282,7 +299,11 @@ export async function evaluate( const rawApiKey = parsed.config?.apiKey; let apiKey: string | undefined; let keySources: string; - if (rawApiKey !== undefined && rawApiKey !== "") { + const keyless = endpoint.auth === "none"; + if (keyless) { + apiKey = undefined; + keySources = ""; + } else if (rawApiKey !== undefined && rawApiKey !== "") { apiKey = rawApiKey; keySources = "config.apiKey"; } else if (endpoint.backend === "corbits-system-one") { @@ -332,16 +353,20 @@ export async function evaluate( }; emit(startEvent); - if (apiKey === undefined || apiKey === "") { + if (!keyless && (apiKey === undefined || apiKey === "")) { return fallback("no-key", { detail: `no API key supplied; set config.apiKey or ${keySources}`, }); } + const images = options?.images; const wire = WireRequest({ model, state: parsed.state, questions: toWireQuestions(parsed.questions), + ...(images !== undefined && images.parts.length > 0 + ? { assets: images.assets } + : {}), }); if (wire instanceof type.errors) { throw new SystemOneError( @@ -356,7 +381,13 @@ export async function evaluate( const response = await postEvaluate( endpoint.url, wire, - { timeoutMs, apiKey }, + { + timeoutMs, + ...(apiKey === undefined ? {} : { apiKey }), + ...(images !== undefined && images.parts.length > 0 + ? { imageParts: images.parts } + : {}), + }, deps, ); data = response.data; diff --git a/src/extraction.test.ts b/src/extraction.test.ts new file mode 100644 index 0000000..97cddf0 --- /dev/null +++ b/src/extraction.test.ts @@ -0,0 +1,254 @@ +import { describe, expect, test } from "bun:test"; + +import { choice, systemOne } from "./decisions"; +import { date, estimate, extract, list, number, text } from "./extraction"; + +describe("extract", () => { + test("posts Clairic fields and maps answers", async () => { + const calls: { url: string; init: RequestInit | undefined }[] = []; + const result = await extract( + { + state: "Invoice INV-2291 issued 2026-09-02, net 30.", + fields: { + invoice_number: text(), + due_date: date("Issue date plus the net payment days"), + total: number({ + decimals: 2, + unit: "USD", + instructions: "Amount due including shipping", + }), + contacts: list("text"), + days_to_pay: estimate({ + min: 0, + max: 120, + step: 1, + instructions: "Days until paid", + }), + }, + }, + { + accountId: "acct", + token: "tok", + fetch: async (url, init) => { + calls.push({ url: String(url), init }); + return new Response( + JSON.stringify({ + result: { + answers: { + invoice_number: { value: "INV-2291", start: 8, end: 16 }, + due_date: { value: "2026-10-02" }, + total: { value: 1080, unit: "USD" }, + contacts: { value: ["Dana Ruiz"] }, + days_to_pay: { median: 30, interval: { p10: 12, p90: 45 } }, + }, + }, + }), + { status: 200 }, + ); + }, + }, + ); + + expect(calls).toHaveLength(1); + const call = calls[0]; + if (call === undefined) throw new Error("expected a fetch"); + expect(call.url).toBe( + "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clairic", + ); + expect(new Headers(call.init?.headers).get("authorization")).toBe( + "Bearer tok", + ); + expect(JSON.parse(String(call.init?.body))).toEqual({ + model: "clairic", + state: "Invoice INV-2291 issued 2026-09-02, net 30.", + fields: { + invoice_number: { type: "text" }, + due_date: { + type: "date", + instructions: "Issue date plus the net payment days", + }, + total: { + type: "number", + decimals: 2, + unit: "USD", + instructions: "Amount due including shipping", + }, + contacts: { type: "list", items: "text" }, + days_to_pay: { + type: "estimate", + min: 0, + max: 120, + step: 1, + instructions: "Days until paid", + }, + }, + }); + if (result.fallback) throw new Error("expected answers"); + expect(result.answers.invoice_number).toEqual({ + value: "INV-2291", + start: 8, + end: 16, + }); + expect(result.answers.due_date.value).toBe("2026-10-02"); + expect(result.answers.total.value).toBe(1080); + expect(result.answers.contacts.value).toEqual(["Dana Ruiz"]); + expect(result.answers.days_to_pay.median).toBe(30); + expect(result.answers.days_to_pay.interval).toEqual({ p10: 12, p90: 45 }); + }); + + test("maps documented spans, estimate bounds, and list items", async () => { + const result = await extract( + { + state: "Invoice INV-2291 issued 2026-09-02, net 30.", + fields: { + invoice_number: text(), + days_to_pay: estimate({ min: 0, max: 120 }), + amounts: list("number"), + due_dates: list("date"), + }, + }, + { + accountId: "acct", + token: "tok", + fetch: async () => + new Response( + JSON.stringify({ + result: { + answers: { + invoice_number: { + type: "text", + value: "INV-2291", + span: [8, 16], + confidence: 1, + p_null: 3.8e-10, + }, + days_to_pay: { + type: "estimate", + value: 30, + median: 30, + p10: 21, + p90: 38, + }, + amounts: { type: "list", value: [250, 80] }, + due_dates: { type: "list", value: ["2026-10-02"] }, + }, + }, + }), + { status: 200 }, + ), + }, + ); + + if (result.fallback) throw new Error(result.detail ?? "expected answers"); + expect(result.answers.invoice_number).toEqual({ + value: "INV-2291", + start: 8, + end: 16, + }); + expect(result.answers.days_to_pay).toEqual({ + median: 30, + interval: { p10: 21, p90: 38 }, + }); + const amounts: number[] | null = result.answers.amounts.value; + expect(amounts).toEqual([250, 80]); + expect(result.answers.due_dates.value).toEqual(["2026-10-02"]); + }); + + test("rejects a non-string text value instead of a successful null", async () => { + const result = await extract( + { state: "x", fields: { invoice_number: text() } }, + { + accountId: "acct", + token: "tok", + fetch: async () => + new Response( + JSON.stringify({ + result: { + answers: { + invoice_number: { type: "text", value: 1080 }, + }, + }, + }), + { status: 200 }, + ), + }, + ); + + expect(result).toMatchObject({ fallback: true, reason: "parse-error" }); + if (!result.fallback) { + const value: string | null = result.answers.invoice_number.value; + expect(value).not.toBeNull(); + } + }); + + test("falls back without credentials, on non-2xx, and on bad JSON", async () => { + const missing = await extract( + { state: "x", fields: { invoice_number: text() } }, + { + fetch: () => Promise.reject(new Error("fetch must not be called")), + }, + ); + expect(missing).toMatchObject({ fallback: true, reason: "no-key" }); + + const http = await extract( + { state: "x", fields: { invoice_number: text() } }, + { + accountId: "acct", + token: "tok", + fetch: async () => new Response("no", { status: 500 }), + }, + ); + expect(http).toMatchObject({ + fallback: true, + reason: "http-error", + httpStatus: 500, + }); + + const bad = await extract( + { state: "x", fields: { invoice_number: text() } }, + { + accountId: "acct", + token: "tok", + fetch: async () => new Response("not-json", { status: 200 }), + }, + ); + expect(bad).toMatchObject({ fallback: true, reason: "parse-error" }); + }); +}); + +describe("extraction and decisions stay disjoint", () => { + test("number is not a decision question and choice is not an extraction field", () => { + async function probe(): Promise { + await systemOne({ + state: "x", + questions: { + category: choice("What is this?", { billing: null }), + // @ts-expect-error number fields are not decision questions + total: number(), + }, + }); + + await extract({ + state: "x", + fields: { + invoice_number: text(), + // @ts-expect-error choice questions are not extraction fields + category: choice("What is this?", { billing: null }), + }, + }); + + const listed = await extract( + { state: "x", fields: { amounts: list("number") } }, + { fetch: () => Promise.reject(new Error("no fetch")) }, + ); + if (!listed.fallback) { + function stringsOnly(value: string[] | null): string[] | null { + return value; + } + // @ts-expect-error number lists are not string[] + stringsOnly(listed.answers.amounts.value); + } + } + expect(probe).toBeFunction(); + }); +}); diff --git a/src/extraction.ts b/src/extraction.ts new file mode 100644 index 0000000..324e605 --- /dev/null +++ b/src/extraction.ts @@ -0,0 +1,508 @@ +import { type } from "arktype"; + +import type { FallbackReason, FallbackResult } from "./schemas.js"; + +const CLAIRIC_MODEL = "@cf/cloudflare/clairic"; + +export type TextField = { type: "text"; instructions?: string }; +export type DateField = { type: "date"; instructions?: string }; +export type NumberField = { + type: "number"; + decimals?: number; + unit?: string; + instructions?: string; +}; +export type ListItemKind = "text" | "number" | "date"; + +export type ListField = { + type: "list"; + items: Items; +}; +export type EstimateField = { + type: "estimate"; + min?: number; + max?: number; + step?: number; + instructions?: string; +}; + +export type ExtractionField = + | TextField + | DateField + | NumberField + | ListField + | EstimateField; + +export type TextAnswer = { + value: string | null; + start?: number; + end?: number; +}; + +export type DateAnswer = { value: string | null }; + +export type NumberAnswer = { value: number | null; unit?: string }; + +type ListValue = Items extends "number" ? number : string; + +export type ListAnswer = { + value: ListValue[] | null; +}; + +export type EstimateAnswer = { + median: number | null; + interval?: { p10: number; p90: number }; +}; + +type AnswerOf = Field extends { type: "text" } + ? TextAnswer + : Field extends { type: "date" } + ? DateAnswer + : Field extends { type: "number" } + ? NumberAnswer + : Field extends { type: "list"; items: infer Items extends string } + ? ListAnswer + : Field extends { type: "estimate" } + ? EstimateAnswer + : never; + +export type ExtractionAnswers> = + { + [Key in keyof Fields]: AnswerOf; + }; + +export type ExtractionSuccess> = + { + fallback: false; + answers: ExtractionAnswers; + modelId: string; + latencyMs: number; + }; + +export type ExtractionResult> = + | ExtractionSuccess + | FallbackResult; + +export type ExtractInput> = { + state: unknown; + fields: Fields; +}; + +export type ExtractFetch = ( + input: string | URL | Request, + init?: RequestInit, +) => Promise; + +export type ExtractOptions = { + accountId?: string; + token?: string; + fetch?: ExtractFetch; +}; + +function definedString(value: string | undefined): string | undefined { + if (value === undefined || value === "") return undefined; + return value; +} + +function text(instructions?: string): TextField { + return instructions === undefined + ? { type: "text" } + : { type: "text", instructions }; +} + +function date(instructions?: string): DateField { + return instructions === undefined + ? { type: "date" } + : { type: "date", instructions }; +} + +function number(options?: { + decimals?: number; + unit?: string; + instructions?: string; +}): NumberField { + const field: NumberField = { type: "number" }; + if (options?.decimals !== undefined) field.decimals = options.decimals; + if (options?.unit !== undefined) field.unit = options.unit; + if (options?.instructions !== undefined) + field.instructions = options.instructions; + return field; +} + +function list( + items: Items, +): ListField { + return { type: "list", items }; +} + +function estimate(options?: { + min?: number; + max?: number; + step?: number; + instructions?: string; +}): EstimateField { + const field: EstimateField = { type: "estimate" }; + if (options?.min !== undefined) field.min = options.min; + if (options?.max !== undefined) field.max = options.max; + if (options?.step !== undefined) field.step = options.step; + if (options?.instructions !== undefined) + field.instructions = options.instructions; + return field; +} + +export { date, estimate, list, number, text }; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +type ParseError = { parseError: string }; + +const finiteNumber = type("number").narrow((value) => Number.isFinite(value)); + +const finiteOrNull = type("number | null").narrow( + (value) => value === null || Number.isFinite(value), +); + +const TextWire = type({ + "type?": "string", + "value?": "string | null", + "text?": "string | null", + "start?": finiteNumber, + "end?": finiteNumber, + "span?": [finiteNumber, finiteNumber], + "offsets?": { + "start?": finiteNumber, + "end?": finiteNumber, + }, +}); + +const DateWire = type("string").or( + type({ + "type?": "string", + "value?": "string | null", + "date?": "string | null", + }), +); + +const NumberWire = finiteNumber.or( + type({ + "type?": "string", + "value?": finiteOrNull, + "number?": finiteOrNull, + "unit?": "string", + }), +); + +const stringItems = type("string[]").or("null"); + +const numberItems = finiteNumber.array().or("null"); + +const StringListWire = type("string[]").or( + type({ + "type?": "string", + "value?": stringItems, + "items?": stringItems, + }), +); + +const NumberListWire = finiteNumber.array().or( + type({ + "type?": "string", + "value?": numberItems, + "items?": numberItems, + }), +); + +const EstimateInterval = type([finiteNumber, finiteNumber]).or( + type({ + "p10?": finiteNumber, + "low?": finiteNumber, + "p90?": finiteNumber, + "high?": finiteNumber, + }), +); + +const EstimateWire = type({ + "type?": "string", + "value?": finiteOrNull, + "median?": finiteOrNull, + "p10?": finiteNumber, + "p90?": finiteNumber, + "interval?": EstimateInterval, +}); + +function isParseError(value: unknown): value is ParseError { + return ( + typeof value === "object" && + value !== null && + "parseError" in value && + typeof value.parseError === "string" + ); +} + +function textAnswer(raw: unknown): TextAnswer | ParseError { + if (typeof raw === "string") return { value: raw }; + if (raw === null || raw === undefined) return { value: null }; + const parsed = TextWire(raw); + if (parsed instanceof type.errors) return { parseError: parsed.summary }; + const value = + typeof parsed.value === "string" + ? parsed.value + : typeof parsed.text === "string" + ? parsed.text + : null; + const start = parsed.start ?? parsed.span?.[0] ?? parsed.offsets?.start; + const end = parsed.end ?? parsed.span?.[1] ?? parsed.offsets?.end; + const answer: TextAnswer = { value }; + if (start !== undefined) answer.start = start; + if (end !== undefined) answer.end = end; + return answer; +} + +function dateAnswer(raw: unknown): DateAnswer | ParseError { + if (raw === null || raw === undefined) return { value: null }; + const parsed = DateWire(raw); + if (parsed instanceof type.errors) return { parseError: parsed.summary }; + if (typeof parsed === "string") return { value: parsed }; + const value = + typeof parsed.value === "string" + ? parsed.value + : typeof parsed.date === "string" + ? parsed.date + : null; + return { value }; +} + +function numberAnswer(raw: unknown): NumberAnswer | ParseError { + if (raw === null || raw === undefined) return { value: null }; + const parsed = NumberWire(raw); + if (parsed instanceof type.errors) return { parseError: parsed.summary }; + if (typeof parsed === "number") return { value: parsed }; + const value = + typeof parsed.value === "number" + ? parsed.value + : typeof parsed.number === "number" + ? parsed.number + : null; + const answer: NumberAnswer = { value }; + if (parsed.unit !== undefined) answer.unit = parsed.unit; + return answer; +} + +function listValues( + parsed: + | { value?: string[] | number[] | null; items?: string[] | number[] | null } + | string[] + | number[], +): string[] | number[] | null { + if (Array.isArray(parsed)) return parsed; + if (parsed.value !== undefined) return parsed.value; + return parsed.items ?? null; +} + +function listAnswer( + items: ListItemKind, + raw: unknown, +): ListAnswer<"text"> | ListAnswer<"number"> | ParseError { + if (raw === null || raw === undefined) return { value: null }; + const parsed = (items === "number" ? NumberListWire : StringListWire)(raw); + if (parsed instanceof type.errors) return { parseError: parsed.summary }; + const values = listValues(parsed); + if (values === null) return { value: null }; + if (items === "number") { + if (!values.every((item) => typeof item === "number")) { + return { parseError: "list items must be numbers" }; + } + return { value: values }; + } + if (!values.every((item) => typeof item === "string")) { + return { parseError: "list items must be strings" }; + } + return { value: values }; +} + +function isNumberPair( + value: + | readonly [number, number] + | { p10?: number; low?: number; p90?: number; high?: number }, +): value is readonly [number, number] { + return Array.isArray(value); +} + +function estimateInterval(parsed: { + p10?: number; + p90?: number; + interval?: + | readonly [number, number] + | { p10?: number; low?: number; p90?: number; high?: number }; +}): { p10: number; p90: number } | undefined { + if (parsed.p10 !== undefined && parsed.p90 !== undefined) { + return { p10: parsed.p10, p90: parsed.p90 }; + } + const interval = parsed.interval; + if (interval === undefined) return undefined; + if (isNumberPair(interval)) { + const low = interval[0]; + const high = interval[1]; + if (low === undefined || high === undefined) return undefined; + return { p10: low, p90: high }; + } + const low = interval.p10 ?? interval.low; + const high = interval.p90 ?? interval.high; + if (low === undefined || high === undefined) return undefined; + return { p10: low, p90: high }; +} + +function estimateAnswer(raw: unknown): EstimateAnswer | ParseError { + if (raw === null || raw === undefined) return { median: null }; + const parsed = EstimateWire(raw); + if (parsed instanceof type.errors) return { parseError: parsed.summary }; + const median = parsed.median ?? parsed.value ?? null; + const answer: EstimateAnswer = { median }; + const interval = estimateInterval(parsed); + if (interval !== undefined) answer.interval = interval; + return answer; +} + +function mapAnswer( + field: ExtractionField, + raw: unknown, +): + | TextAnswer + | DateAnswer + | NumberAnswer + | ListAnswer<"text"> + | ListAnswer<"number"> + | EstimateAnswer + | ParseError { + if (field.type === "text") return textAnswer(raw); + if (field.type === "date") return dateAnswer(raw); + if (field.type === "number") return numberAnswer(raw); + if (field.type === "list") return listAnswer(field.items, raw); + return estimateAnswer(raw); +} + +function isExtractionAnswers>( + fields: Fields, + answers: Record, +): answers is ExtractionAnswers { + return Object.keys(fields).every((key) => Object.hasOwn(answers, key)); +} + +function answersOf>( + fields: Fields, + raw: unknown, +): ExtractionAnswers | ParseError { + const envelope = + isRecord(raw) && isRecord(raw["result"]) ? raw["result"] : raw; + if (!isRecord(envelope) || !isRecord(envelope["answers"])) { + return { parseError: "response has no answers" }; + } + const source = envelope["answers"]; + const answers: Record = {}; + for (const [name, field] of Object.entries(fields)) { + const parsed = mapAnswer(field, source[name]); + if (isParseError(parsed)) { + return { + parseError: `answer "${name}" failed schema validation: ${parsed.parseError}`, + }; + } + answers[name] = parsed; + } + if (!isExtractionAnswers(fields, answers)) { + return { parseError: "response has no answers" }; + } + return answers; +} + +function fallback( + reason: FallbackReason, + started: number, + extra?: { httpStatus?: number; detail?: string }, +): FallbackResult { + const result: FallbackResult = { + fallback: true, + reason, + latencyMs: Math.max(0, Date.now() - started), + backendAttempted: "cloudflare-workers-ai", + }; + if (extra?.httpStatus !== undefined) result.httpStatus = extra.httpStatus; + if (extra?.detail !== undefined) result.detail = extra.detail; + return result; +} + +/** + * Keyed extraction against Cloudflare Workers AI Clairic. Missing credentials, + * non-2xx responses, and bodies that are not JSON return `fallback` and do + * not throw. + */ +export async function extract< + const Fields extends Record, +>( + input: ExtractInput, + options?: ExtractOptions, +): Promise> { + const started = Date.now(); + const accountId = + definedString(options?.accountId) ?? + definedString(process.env["CLOUDFLARE_ACCOUNT_ID"]); + const token = + definedString(options?.token) ?? + definedString(process.env["CLOUDFLARE_API_TOKEN"]); + if (accountId === undefined || token === undefined) { + return fallback("no-key", started, { + detail: + "no Cloudflare credentials; set accountId and token or CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN", + }); + } + + const url = `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/run/${CLAIRIC_MODEL}`; + const fetchImpl = options?.fetch ?? fetch; + let response: Response; + try { + response = await fetchImpl(url, { + method: "POST", + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + model: "clairic", + state: input.state, + fields: input.fields, + }), + }); + } catch (cause) { + return fallback("network", started, { + detail: cause instanceof Error ? cause.message : String(cause), + }); + } + + if (!response.ok) { + return fallback("http-error", started, { httpStatus: response.status }); + } + + let body: unknown; + try { + body = await response.json(); + } catch (cause) { + return fallback("parse-error", started, { + detail: cause instanceof Error ? cause.message : String(cause), + }); + } + + const answers = answersOf(input.fields, body); + if (isParseError(answers)) { + return fallback("parse-error", started, { + detail: answers.parseError.slice(0, 240), + }); + } + + return { + fallback: false, + answers, + modelId: "clairic", + latencyMs: Math.max(0, Date.now() - started), + }; +} diff --git a/src/index.ts b/src/index.ts index 7d7b4a0..9196de8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,9 @@ export { DEFAULT_TIMEOUT_MS } from "./config.js"; export type { EvaluateDeps } from "./client.js"; export { evaluate } from "./evaluate.js"; export type { EvaluateOptions } from "./evaluate.js"; +export { choice, noul, score, box, systemOne } from "./decisions.js"; +export type { SystemOneResult, SystemOneSuccess } from "./decisions.js"; +export { Image, image, withAssets } from "./assets.js"; export { createSystemOneAdapter, createSystemOneAdapterFactory, @@ -14,6 +17,8 @@ export { BooleanQuestion, ChoiceQuestion, Decision, + DecisionQuestion, + DecisionQuestionRecord, Description, EndpointConfig, EvaluateConfig, diff --git a/src/schemas.ts b/src/schemas.ts index a9ab389..7d94a13 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -114,11 +114,60 @@ export const BooleanQuestion = type({ }).narrow((q) => nonEmpty(q.instructions)); export type BooleanQuestion = typeof BooleanQuestion.infer; +// Locates a region in an image. Edges come back as fractions of the image. +// Callers should set the model to `jev-preview`; the client does not require it. +export const BoxQuestion = type({ + id: "string", + type: "'box'", + instructions: Description, + "+": "reject", +}).narrow((q) => nonEmpty(q.instructions)); +export type BoxQuestion = typeof BoxQuestion.infer; + +// Keyed decisions questions: choice, noul, score, or box only. Extraction-style +// helpers are not part of this contract and fail the union. +export const DecisionQuestion = type({ + type: "'choice'", + instructions: "unknown", + criteria: "unknown", + "+": "reject", +}) + .or( + type({ + type: "'noul'", + "instructions?": "unknown", + "criteria?": "unknown", + "+": "reject", + }), + ) + .or( + type({ + type: "'score'", + instructions: "unknown", + criteria: "unknown", + "+": "reject", + }), + ) + .or( + type({ + type: "'box'", + instructions: "unknown", + "+": "reject", + }), + ); +export type DecisionQuestion = typeof DecisionQuestion.infer; + +export const DecisionQuestionRecord = type({ + "[string]": DecisionQuestion, +}).narrow((qs) => Object.keys(qs).length >= 1); +export type DecisionQuestionRecord = typeof DecisionQuestionRecord.infer; + // The question union a caller may ask in one evaluation. `.or` chains keep // each branch's literal `type`, so the union still discriminates on it. export const Question = ChoiceQuestion.or(ScoreQuestion) .or(NoulQuestion) - .or(BooleanQuestion); + .or(BooleanQuestion) + .or(BoxQuestion); export type Question = typeof Question.infer; // A non-empty question list with unique ids: a duplicate would make the @@ -138,10 +187,13 @@ export type QuestionList = typeof QuestionList.infer; // `gateway` the proxy endpoint, `custom` a caller-supplied URL. `url` only // exists on `custom` — a `url` on another kind would be silently ignored, // so the schema rejects it instead of accepting misleading config. +// `auth: "none"` exists only on `custom`: it is the explicit opt-out from +// the fail-closed key requirement. Official and gateway cannot carry it. export const EndpointConfig = type({ kind: "'custom'", url: "string", "model?": "string", + "auth?": "'none'", "+": "reject", }).or( type({ @@ -186,18 +238,25 @@ export type EvaluateInput = typeof EvaluateInput.infer; /** One question in wire form: no id (the map key carries it). */ export const WireQuestion = type({ - type: "'noul' | 'choice' | 'score'", + type: "'noul' | 'choice' | 'score' | 'box'", instructions: Description, "criteria?": "unknown", "+": "reject", }); export type WireQuestion = typeof WireQuestion.infer; -/** The exact POST body: model + state + id-keyed questions. */ +const WireAsset = type({ + type: "'image'", + part: "string", + "+": "reject", +}); + +/** The exact POST body: model + state + id-keyed questions, plus image parts. */ export const WireRequest = type({ model: "string", state: State, questions: { "[string]": WireQuestion }, + "assets?": { "[string]": WireAsset }, "+": "reject", }).narrow((body) => Object.keys(body.questions).length >= 1); export type WireRequest = typeof WireRequest.infer; @@ -215,7 +274,9 @@ export function toWireQuestions( type: question.type === "boolean" ? "noul" : question.type, instructions: question.instructions, }; - if (question.criteria !== undefined) mapped.criteria = question.criteria; + if ("criteria" in question && question.criteria !== undefined) { + mapped.criteria = question.criteria; + } wire[question.id] = mapped; } return wire; @@ -247,16 +308,42 @@ export const APPROXIMATE_SUM_TOLERANCE = 0.02; // `type` must carry. No `"+": "reject"`: the backend may attach per-answer // metadata that `toDecision` strips. export const WireAnswer = type({ - type: "'noul' | 'choice' | 'score'", + type: "'noul' | 'choice' | 'score' | 'box'", "choice?": "string", "score?": "number", "noul?": Confidence, "probabilities?": ProbabilityMap, - "confidence?": Confidence, + "confidence?": Confidence.or("null"), "legend?": { "[string]": "string" }, + "box?": { + top: "number", + bottom: "number", + left: "number", + right: "number", + }, }); export type WireAnswer = typeof WireAnswer.infer; +/** Box edges are fractions of the image. Shared by `evaluate` and the adapter. */ +export function boxEdgesOutOfRange( + id: string, + box: { top: number; bottom: number; left: number; right: number }, +): string | undefined { + for (const edge of ["top", "bottom", "left", "right"] as const) { + const value = box[edge]; + if (!Number.isFinite(value) || value < 0 || value > 1) { + return `answer "${id}" box.${edge} ${String(value)} outside [0, 1]`; + } + } + if (box.top >= box.bottom) { + return `answer "${id}" box.top ${String(box.top)} is not below box.bottom ${String(box.bottom)}`; + } + if (box.left >= box.right) { + return `answer "${id}" box.left ${String(box.left)} is not left of box.right ${String(box.right)}`; + } + return undefined; +} + // Token usage reported on the response envelope. export const WireUsage = type({ input_tokens: "number", @@ -305,6 +392,21 @@ export const Decision = type({ confidence: Confidence, "+": "reject", }), + ) + .or( + type({ + id: "string", + type: "'box'", + box: { + top: "number", + bottom: "number", + left: "number", + right: "number", + "+": "reject", + }, + confidence: Confidence.or("null"), + "+": "reject", + }), ); export type Decision = typeof Decision.infer; @@ -326,6 +428,13 @@ export function toDecision(id: string, answer: WireAnswer): Decision { probabilities: answer.probabilities, confidence: answer.confidence, }; + } else if (answer.type === "box") { + candidate = { + id, + type: "box", + box: answer.box, + confidence: answer.confidence ?? null, + }; } else { candidate = { id,