diff --git a/package.json b/package.json index 22b619a..39861e4 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,11 @@ "corbits-system-one-src": "./src/decisions.ts", "types": "./dist/decisions.d.ts", "default": "./dist/decisions.js" + }, + "./extraction": { + "corbits-system-one-src": "./src/extraction.ts", + "types": "./dist/extraction.d.ts", + "default": "./dist/extraction.js" } }, "publishConfig": { diff --git a/src/decisions.test.ts b/src/decisions.test.ts index 022b498..c202aa9 100644 --- a/src/decisions.test.ts +++ b/src/decisions.test.ts @@ -285,6 +285,7 @@ describe("systemOne", () => { systemOne( { state: "x", + // @ts-expect-error extraction questions are not decisions questions: { field: { type: "extract", instructions: "pull it" } }, config: { apiKey: "k" }, }, diff --git a/src/decisions.ts b/src/decisions.ts index 7470e22..6aa0d38 100644 --- a/src/decisions.ts +++ b/src/decisions.ts @@ -8,6 +8,7 @@ import { DecisionQuestionRecord, QuestionList, type Decision, + type DecisionQuestion, type Description, type EvaluateConfig, type FallbackResult, @@ -125,13 +126,14 @@ export type SystemOneResult> = | SystemOneSuccess | FallbackResult; -export type SystemOneInput> = { - state: unknown; - questions: Questions; - assets?: Record; - model?: string; - config?: EvaluateConfig; -}; +export type SystemOneInput> = + { + state: unknown; + questions: Questions; + assets?: Record; + model?: string; + config?: EvaluateConfig; + }; function withModel( config: EvaluateConfig | undefined, @@ -218,7 +220,7 @@ async function loadImages( * `fallback` and `reason` and do not throw. */ export async function systemOne< - const Questions extends Record, + const Questions extends Record, >( input: SystemOneInput, options?: Pick, diff --git a/src/extraction.test.ts b/src/extraction.test.ts new file mode 100644 index 0000000..97cddf0 --- /dev/null +++ b/src/extraction.test.ts @@ -0,0 +1,254 @@ +import { describe, expect, test } from "bun:test"; + +import { choice, systemOne } from "./decisions"; +import { date, estimate, extract, list, number, text } from "./extraction"; + +describe("extract", () => { + test("posts Clairic fields and maps answers", async () => { + const calls: { url: string; init: RequestInit | undefined }[] = []; + const result = await extract( + { + state: "Invoice INV-2291 issued 2026-09-02, net 30.", + fields: { + invoice_number: text(), + due_date: date("Issue date plus the net payment days"), + total: number({ + decimals: 2, + unit: "USD", + instructions: "Amount due including shipping", + }), + contacts: list("text"), + days_to_pay: estimate({ + min: 0, + max: 120, + step: 1, + instructions: "Days until paid", + }), + }, + }, + { + accountId: "acct", + token: "tok", + fetch: async (url, init) => { + calls.push({ url: String(url), init }); + return new Response( + JSON.stringify({ + result: { + answers: { + invoice_number: { value: "INV-2291", start: 8, end: 16 }, + due_date: { value: "2026-10-02" }, + total: { value: 1080, unit: "USD" }, + contacts: { value: ["Dana Ruiz"] }, + days_to_pay: { median: 30, interval: { p10: 12, p90: 45 } }, + }, + }, + }), + { status: 200 }, + ); + }, + }, + ); + + expect(calls).toHaveLength(1); + const call = calls[0]; + if (call === undefined) throw new Error("expected a fetch"); + expect(call.url).toBe( + "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clairic", + ); + expect(new Headers(call.init?.headers).get("authorization")).toBe( + "Bearer tok", + ); + expect(JSON.parse(String(call.init?.body))).toEqual({ + model: "clairic", + state: "Invoice INV-2291 issued 2026-09-02, net 30.", + fields: { + invoice_number: { type: "text" }, + due_date: { + type: "date", + instructions: "Issue date plus the net payment days", + }, + total: { + type: "number", + decimals: 2, + unit: "USD", + instructions: "Amount due including shipping", + }, + contacts: { type: "list", items: "text" }, + days_to_pay: { + type: "estimate", + min: 0, + max: 120, + step: 1, + instructions: "Days until paid", + }, + }, + }); + if (result.fallback) throw new Error("expected answers"); + expect(result.answers.invoice_number).toEqual({ + value: "INV-2291", + start: 8, + end: 16, + }); + expect(result.answers.due_date.value).toBe("2026-10-02"); + expect(result.answers.total.value).toBe(1080); + expect(result.answers.contacts.value).toEqual(["Dana Ruiz"]); + expect(result.answers.days_to_pay.median).toBe(30); + expect(result.answers.days_to_pay.interval).toEqual({ p10: 12, p90: 45 }); + }); + + test("maps documented spans, estimate bounds, and list items", async () => { + const result = await extract( + { + state: "Invoice INV-2291 issued 2026-09-02, net 30.", + fields: { + invoice_number: text(), + days_to_pay: estimate({ min: 0, max: 120 }), + amounts: list("number"), + due_dates: list("date"), + }, + }, + { + accountId: "acct", + token: "tok", + fetch: async () => + new Response( + JSON.stringify({ + result: { + answers: { + invoice_number: { + type: "text", + value: "INV-2291", + span: [8, 16], + confidence: 1, + p_null: 3.8e-10, + }, + days_to_pay: { + type: "estimate", + value: 30, + median: 30, + p10: 21, + p90: 38, + }, + amounts: { type: "list", value: [250, 80] }, + due_dates: { type: "list", value: ["2026-10-02"] }, + }, + }, + }), + { status: 200 }, + ), + }, + ); + + if (result.fallback) throw new Error(result.detail ?? "expected answers"); + expect(result.answers.invoice_number).toEqual({ + value: "INV-2291", + start: 8, + end: 16, + }); + expect(result.answers.days_to_pay).toEqual({ + median: 30, + interval: { p10: 21, p90: 38 }, + }); + const amounts: number[] | null = result.answers.amounts.value; + expect(amounts).toEqual([250, 80]); + expect(result.answers.due_dates.value).toEqual(["2026-10-02"]); + }); + + test("rejects a non-string text value instead of a successful null", async () => { + const result = await extract( + { state: "x", fields: { invoice_number: text() } }, + { + accountId: "acct", + token: "tok", + fetch: async () => + new Response( + JSON.stringify({ + result: { + answers: { + invoice_number: { type: "text", value: 1080 }, + }, + }, + }), + { status: 200 }, + ), + }, + ); + + expect(result).toMatchObject({ fallback: true, reason: "parse-error" }); + if (!result.fallback) { + const value: string | null = result.answers.invoice_number.value; + expect(value).not.toBeNull(); + } + }); + + test("falls back without credentials, on non-2xx, and on bad JSON", async () => { + const missing = await extract( + { state: "x", fields: { invoice_number: text() } }, + { + fetch: () => Promise.reject(new Error("fetch must not be called")), + }, + ); + expect(missing).toMatchObject({ fallback: true, reason: "no-key" }); + + const http = await extract( + { state: "x", fields: { invoice_number: text() } }, + { + accountId: "acct", + token: "tok", + fetch: async () => new Response("no", { status: 500 }), + }, + ); + expect(http).toMatchObject({ + fallback: true, + reason: "http-error", + httpStatus: 500, + }); + + const bad = await extract( + { state: "x", fields: { invoice_number: text() } }, + { + accountId: "acct", + token: "tok", + fetch: async () => new Response("not-json", { status: 200 }), + }, + ); + expect(bad).toMatchObject({ fallback: true, reason: "parse-error" }); + }); +}); + +describe("extraction and decisions stay disjoint", () => { + test("number is not a decision question and choice is not an extraction field", () => { + async function probe(): Promise { + await systemOne({ + state: "x", + questions: { + category: choice("What is this?", { billing: null }), + // @ts-expect-error number fields are not decision questions + total: number(), + }, + }); + + await extract({ + state: "x", + fields: { + invoice_number: text(), + // @ts-expect-error choice questions are not extraction fields + category: choice("What is this?", { billing: null }), + }, + }); + + const listed = await extract( + { state: "x", fields: { amounts: list("number") } }, + { fetch: () => Promise.reject(new Error("no fetch")) }, + ); + if (!listed.fallback) { + function stringsOnly(value: string[] | null): string[] | null { + return value; + } + // @ts-expect-error number lists are not string[] + stringsOnly(listed.answers.amounts.value); + } + } + expect(probe).toBeFunction(); + }); +}); diff --git a/src/extraction.ts b/src/extraction.ts new file mode 100644 index 0000000..324e605 --- /dev/null +++ b/src/extraction.ts @@ -0,0 +1,508 @@ +import { type } from "arktype"; + +import type { FallbackReason, FallbackResult } from "./schemas.js"; + +const CLAIRIC_MODEL = "@cf/cloudflare/clairic"; + +export type TextField = { type: "text"; instructions?: string }; +export type DateField = { type: "date"; instructions?: string }; +export type NumberField = { + type: "number"; + decimals?: number; + unit?: string; + instructions?: string; +}; +export type ListItemKind = "text" | "number" | "date"; + +export type ListField = { + type: "list"; + items: Items; +}; +export type EstimateField = { + type: "estimate"; + min?: number; + max?: number; + step?: number; + instructions?: string; +}; + +export type ExtractionField = + | TextField + | DateField + | NumberField + | ListField + | EstimateField; + +export type TextAnswer = { + value: string | null; + start?: number; + end?: number; +}; + +export type DateAnswer = { value: string | null }; + +export type NumberAnswer = { value: number | null; unit?: string }; + +type ListValue = Items extends "number" ? number : string; + +export type ListAnswer = { + value: ListValue[] | null; +}; + +export type EstimateAnswer = { + median: number | null; + interval?: { p10: number; p90: number }; +}; + +type AnswerOf = Field extends { type: "text" } + ? TextAnswer + : Field extends { type: "date" } + ? DateAnswer + : Field extends { type: "number" } + ? NumberAnswer + : Field extends { type: "list"; items: infer Items extends string } + ? ListAnswer + : Field extends { type: "estimate" } + ? EstimateAnswer + : never; + +export type ExtractionAnswers> = + { + [Key in keyof Fields]: AnswerOf; + }; + +export type ExtractionSuccess> = + { + fallback: false; + answers: ExtractionAnswers; + modelId: string; + latencyMs: number; + }; + +export type ExtractionResult> = + | ExtractionSuccess + | FallbackResult; + +export type ExtractInput> = { + state: unknown; + fields: Fields; +}; + +export type ExtractFetch = ( + input: string | URL | Request, + init?: RequestInit, +) => Promise; + +export type ExtractOptions = { + accountId?: string; + token?: string; + fetch?: ExtractFetch; +}; + +function definedString(value: string | undefined): string | undefined { + if (value === undefined || value === "") return undefined; + return value; +} + +function text(instructions?: string): TextField { + return instructions === undefined + ? { type: "text" } + : { type: "text", instructions }; +} + +function date(instructions?: string): DateField { + return instructions === undefined + ? { type: "date" } + : { type: "date", instructions }; +} + +function number(options?: { + decimals?: number; + unit?: string; + instructions?: string; +}): NumberField { + const field: NumberField = { type: "number" }; + if (options?.decimals !== undefined) field.decimals = options.decimals; + if (options?.unit !== undefined) field.unit = options.unit; + if (options?.instructions !== undefined) + field.instructions = options.instructions; + return field; +} + +function list( + items: Items, +): ListField { + return { type: "list", items }; +} + +function estimate(options?: { + min?: number; + max?: number; + step?: number; + instructions?: string; +}): EstimateField { + const field: EstimateField = { type: "estimate" }; + if (options?.min !== undefined) field.min = options.min; + if (options?.max !== undefined) field.max = options.max; + if (options?.step !== undefined) field.step = options.step; + if (options?.instructions !== undefined) + field.instructions = options.instructions; + return field; +} + +export { date, estimate, list, number, text }; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +type ParseError = { parseError: string }; + +const finiteNumber = type("number").narrow((value) => Number.isFinite(value)); + +const finiteOrNull = type("number | null").narrow( + (value) => value === null || Number.isFinite(value), +); + +const TextWire = type({ + "type?": "string", + "value?": "string | null", + "text?": "string | null", + "start?": finiteNumber, + "end?": finiteNumber, + "span?": [finiteNumber, finiteNumber], + "offsets?": { + "start?": finiteNumber, + "end?": finiteNumber, + }, +}); + +const DateWire = type("string").or( + type({ + "type?": "string", + "value?": "string | null", + "date?": "string | null", + }), +); + +const NumberWire = finiteNumber.or( + type({ + "type?": "string", + "value?": finiteOrNull, + "number?": finiteOrNull, + "unit?": "string", + }), +); + +const stringItems = type("string[]").or("null"); + +const numberItems = finiteNumber.array().or("null"); + +const StringListWire = type("string[]").or( + type({ + "type?": "string", + "value?": stringItems, + "items?": stringItems, + }), +); + +const NumberListWire = finiteNumber.array().or( + type({ + "type?": "string", + "value?": numberItems, + "items?": numberItems, + }), +); + +const EstimateInterval = type([finiteNumber, finiteNumber]).or( + type({ + "p10?": finiteNumber, + "low?": finiteNumber, + "p90?": finiteNumber, + "high?": finiteNumber, + }), +); + +const EstimateWire = type({ + "type?": "string", + "value?": finiteOrNull, + "median?": finiteOrNull, + "p10?": finiteNumber, + "p90?": finiteNumber, + "interval?": EstimateInterval, +}); + +function isParseError(value: unknown): value is ParseError { + return ( + typeof value === "object" && + value !== null && + "parseError" in value && + typeof value.parseError === "string" + ); +} + +function textAnswer(raw: unknown): TextAnswer | ParseError { + if (typeof raw === "string") return { value: raw }; + if (raw === null || raw === undefined) return { value: null }; + const parsed = TextWire(raw); + if (parsed instanceof type.errors) return { parseError: parsed.summary }; + const value = + typeof parsed.value === "string" + ? parsed.value + : typeof parsed.text === "string" + ? parsed.text + : null; + const start = parsed.start ?? parsed.span?.[0] ?? parsed.offsets?.start; + const end = parsed.end ?? parsed.span?.[1] ?? parsed.offsets?.end; + const answer: TextAnswer = { value }; + if (start !== undefined) answer.start = start; + if (end !== undefined) answer.end = end; + return answer; +} + +function dateAnswer(raw: unknown): DateAnswer | ParseError { + if (raw === null || raw === undefined) return { value: null }; + const parsed = DateWire(raw); + if (parsed instanceof type.errors) return { parseError: parsed.summary }; + if (typeof parsed === "string") return { value: parsed }; + const value = + typeof parsed.value === "string" + ? parsed.value + : typeof parsed.date === "string" + ? parsed.date + : null; + return { value }; +} + +function numberAnswer(raw: unknown): NumberAnswer | ParseError { + if (raw === null || raw === undefined) return { value: null }; + const parsed = NumberWire(raw); + if (parsed instanceof type.errors) return { parseError: parsed.summary }; + if (typeof parsed === "number") return { value: parsed }; + const value = + typeof parsed.value === "number" + ? parsed.value + : typeof parsed.number === "number" + ? parsed.number + : null; + const answer: NumberAnswer = { value }; + if (parsed.unit !== undefined) answer.unit = parsed.unit; + return answer; +} + +function listValues( + parsed: + | { value?: string[] | number[] | null; items?: string[] | number[] | null } + | string[] + | number[], +): string[] | number[] | null { + if (Array.isArray(parsed)) return parsed; + if (parsed.value !== undefined) return parsed.value; + return parsed.items ?? null; +} + +function listAnswer( + items: ListItemKind, + raw: unknown, +): ListAnswer<"text"> | ListAnswer<"number"> | ParseError { + if (raw === null || raw === undefined) return { value: null }; + const parsed = (items === "number" ? NumberListWire : StringListWire)(raw); + if (parsed instanceof type.errors) return { parseError: parsed.summary }; + const values = listValues(parsed); + if (values === null) return { value: null }; + if (items === "number") { + if (!values.every((item) => typeof item === "number")) { + return { parseError: "list items must be numbers" }; + } + return { value: values }; + } + if (!values.every((item) => typeof item === "string")) { + return { parseError: "list items must be strings" }; + } + return { value: values }; +} + +function isNumberPair( + value: + | readonly [number, number] + | { p10?: number; low?: number; p90?: number; high?: number }, +): value is readonly [number, number] { + return Array.isArray(value); +} + +function estimateInterval(parsed: { + p10?: number; + p90?: number; + interval?: + | readonly [number, number] + | { p10?: number; low?: number; p90?: number; high?: number }; +}): { p10: number; p90: number } | undefined { + if (parsed.p10 !== undefined && parsed.p90 !== undefined) { + return { p10: parsed.p10, p90: parsed.p90 }; + } + const interval = parsed.interval; + if (interval === undefined) return undefined; + if (isNumberPair(interval)) { + const low = interval[0]; + const high = interval[1]; + if (low === undefined || high === undefined) return undefined; + return { p10: low, p90: high }; + } + const low = interval.p10 ?? interval.low; + const high = interval.p90 ?? interval.high; + if (low === undefined || high === undefined) return undefined; + return { p10: low, p90: high }; +} + +function estimateAnswer(raw: unknown): EstimateAnswer | ParseError { + if (raw === null || raw === undefined) return { median: null }; + const parsed = EstimateWire(raw); + if (parsed instanceof type.errors) return { parseError: parsed.summary }; + const median = parsed.median ?? parsed.value ?? null; + const answer: EstimateAnswer = { median }; + const interval = estimateInterval(parsed); + if (interval !== undefined) answer.interval = interval; + return answer; +} + +function mapAnswer( + field: ExtractionField, + raw: unknown, +): + | TextAnswer + | DateAnswer + | NumberAnswer + | ListAnswer<"text"> + | ListAnswer<"number"> + | EstimateAnswer + | ParseError { + if (field.type === "text") return textAnswer(raw); + if (field.type === "date") return dateAnswer(raw); + if (field.type === "number") return numberAnswer(raw); + if (field.type === "list") return listAnswer(field.items, raw); + return estimateAnswer(raw); +} + +function isExtractionAnswers>( + fields: Fields, + answers: Record, +): answers is ExtractionAnswers { + return Object.keys(fields).every((key) => Object.hasOwn(answers, key)); +} + +function answersOf>( + fields: Fields, + raw: unknown, +): ExtractionAnswers | ParseError { + const envelope = + isRecord(raw) && isRecord(raw["result"]) ? raw["result"] : raw; + if (!isRecord(envelope) || !isRecord(envelope["answers"])) { + return { parseError: "response has no answers" }; + } + const source = envelope["answers"]; + const answers: Record = {}; + for (const [name, field] of Object.entries(fields)) { + const parsed = mapAnswer(field, source[name]); + if (isParseError(parsed)) { + return { + parseError: `answer "${name}" failed schema validation: ${parsed.parseError}`, + }; + } + answers[name] = parsed; + } + if (!isExtractionAnswers(fields, answers)) { + return { parseError: "response has no answers" }; + } + return answers; +} + +function fallback( + reason: FallbackReason, + started: number, + extra?: { httpStatus?: number; detail?: string }, +): FallbackResult { + const result: FallbackResult = { + fallback: true, + reason, + latencyMs: Math.max(0, Date.now() - started), + backendAttempted: "cloudflare-workers-ai", + }; + if (extra?.httpStatus !== undefined) result.httpStatus = extra.httpStatus; + if (extra?.detail !== undefined) result.detail = extra.detail; + return result; +} + +/** + * Keyed extraction against Cloudflare Workers AI Clairic. Missing credentials, + * non-2xx responses, and bodies that are not JSON return `fallback` and do + * not throw. + */ +export async function extract< + const Fields extends Record, +>( + input: ExtractInput, + options?: ExtractOptions, +): Promise> { + const started = Date.now(); + const accountId = + definedString(options?.accountId) ?? + definedString(process.env["CLOUDFLARE_ACCOUNT_ID"]); + const token = + definedString(options?.token) ?? + definedString(process.env["CLOUDFLARE_API_TOKEN"]); + if (accountId === undefined || token === undefined) { + return fallback("no-key", started, { + detail: + "no Cloudflare credentials; set accountId and token or CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN", + }); + } + + const url = `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/run/${CLAIRIC_MODEL}`; + const fetchImpl = options?.fetch ?? fetch; + let response: Response; + try { + response = await fetchImpl(url, { + method: "POST", + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + }, + body: JSON.stringify({ + model: "clairic", + state: input.state, + fields: input.fields, + }), + }); + } catch (cause) { + return fallback("network", started, { + detail: cause instanceof Error ? cause.message : String(cause), + }); + } + + if (!response.ok) { + return fallback("http-error", started, { httpStatus: response.status }); + } + + let body: unknown; + try { + body = await response.json(); + } catch (cause) { + return fallback("parse-error", started, { + detail: cause instanceof Error ? cause.message : String(cause), + }); + } + + const answers = answersOf(input.fields, body); + if (isParseError(answers)) { + return fallback("parse-error", started, { + detail: answers.parseError.slice(0, 240), + }); + } + + return { + fallback: false, + answers, + modelId: "clairic", + latencyMs: Math.max(0, Date.now() - started), + }; +}