From adcfc9323fc2b5afa1130ed6a121c78a5d02a0a8 Mon Sep 17 00:00:00 2001 From: "typeagent-bot[bot]" <274150370+typeagent-bot[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:53:38 +0000 Subject: [PATCH 1/4] Implement accepted Issue #75 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- typescript/metadata.md | 61 ++++++++ typescript/src/model.ts | 227 +++++++++++++-------------- typescript/src/result.ts | 84 +++++++++- typescript/src/typechat.ts | 18 ++- typescript/test/metadata.test.ts | 258 +++++++++++++++++++++++++++++++ 5 files changed, 515 insertions(+), 133 deletions(-) create mode 100644 typescript/metadata.md create mode 100644 typescript/test/metadata.test.ts diff --git a/typescript/metadata.md b/typescript/metadata.md new file mode 100644 index 00000000..395d5afb --- /dev/null +++ b/typescript/metadata.md @@ -0,0 +1,61 @@ +# Response metadata and reported token usage + +Model and translation `Result` envelopes have optional `metadata` on **success and failure**. +Successful model data remains text; translation data remains the validated application object. +Metadata never enters application data, schemas, or repair prompts. Validation, `getData`, and +the default limit of one repair are unchanged. + +```ts +const result = await translator.translate(request); +const tokens = result.metadata?.usage?.inputTokens; // undefined: unknown; 0: reported zero +const coverage = result.metadata?.usageReporting?.inputTokens; // "complete" or "partial" +if (result.success) { + const applicationData = result.data; +} else { + const message = result.message; + const records = result.metadata?.responses; // retained on failure +} +``` + +`metadata.responses` contains ordered records with optional `usage` and `filtering` across initial +and repair responses, including failures. A metadata-free completion contributes `{}` when another +has metadata; if all metadata is absent, the result stays unchanged. Built-in clients also retain +metadata from JSON HTTP errors and responses preceding HTTP retries, without extra service or +moderation requests. An unreadable JSON HTTP error body adds a record's `error` diagnostic (the +error class, not the body), preserving status messages and HTTP retry behavior. Filtered/refused +output remains failure, even if it contains valid-looking JSON. + +`metadata.usage` sums **reported counts only**, once per record: input/output/total 13/7/20 plus +5/3/8 yields 18/10/28. Each corresponding `metadata.usageReporting` field is `"complete"` if every +record reports that count, otherwise `"partial"`. If no record reports a count, its total and +coverage remain absent. Missing total tokens are never inferred from input/output counts. Built-in +clients accept nonnegative safe integers; missing/null counts are unknown, not zero. Even complete +reporting excludes unreported requests, failed transports, and usage the service omits. These +totals are not estimates of all billable usage; TypeChat does not calculate prices. + +| Existing API | `usage` fields mapped to `inputTokens` / `outputTokens` / `totalTokens` | Filtering sources | +| --- | --- | --- | +| OpenAI Chat Completions | `prompt_tokens` / `completion_tokens` / `total_tokens` | First choice's `finish_reason: "content_filter"`, `message.refusal` | +| OpenAI Responses | `input_tokens` / `output_tokens` / `total_tokens` | `incomplete_details.reason: "content_filter"`, output content `type: "refusal"` and its `refusal` text | +| Azure OpenAI Chat Completions | Same as Chat Completions | Same, plus `prompt_filter_results` (or `prompt_annotations`), first choice's `content_filter_results`, HTTP error `error.innererror.content_filter_result`, `error.code`, `error.innererror.code` | + +Filtering exposes optional `blocked`, `reason`, `refusals`, `input`, and `output`. `input` retains +prompt filtering details, `output` completion filtering details; both are `unknown` provider-returned +filtering subtrees, never complete response bodies. Compatible Chat Completions endpoints can return +the same fields. Reported filtering flags, filtering finish reasons, and refusals prevent accepting +filtered output. Missing details remain optional, not evidence that no filtering occurred. Filtering +details and refusals may be sensitive; do not log them automatically. + +Custom models returning metadata-free `Promise>` remain compatible. To attach +metadata without altering text, use `withResponseMetadata(result, [{ usage, filtering }])`; it +computes reported totals from those records. Translation recomputes from records, not existing +totals, so each response contributes exactly once and separate translations stay isolated. + +Run the offline mocked regression suite from `typescript/` with `npm test -- out/metadata.test.js`. +This explicitly adds metadata tests to the existing test command without changing package scripts. +No live provider access or credentials are required. + +Thanks to [intuity-hans](https://github.com/intuity-hans) for the prior work in +[#80](https://github.com/microsoft/TypeChat/pull/80) at +[4f3e378](https://github.com/microsoft/TypeChat/commit/4f3e37840bd5007aeaa05c75589b1e4c16505a80). +That contribution was closed without merging, not an available feature of earlier releases. diff --git a/typescript/src/model.ts b/typescript/src/model.ts index 6c9d6978..7e686be0 100644 --- a/typescript/src/model.ts +++ b/typescript/src/model.ts @@ -1,4 +1,4 @@ -import { Result, success, error } from "./result"; +import { Result, success, error, withResponseMetadata, TypeChatResponseMetadata, TypeChatTokenUsage } from "./result"; /** * Represents a section of an LLM prompt with an associated role. TypeChat uses the "user" role for @@ -87,6 +87,7 @@ export interface TypeChatLanguageModel { * Obtains a completion from the language model for the given prompt. * @param prompt A prompt string or an array of prompt sections. If a string is specified, * it is converted into a single "user" role prompt section. + * Optional result metadata contains service-reported usage and filtering information. */ complete(prompt: string | PromptSection[]): Promise>; } @@ -195,10 +196,7 @@ export function createOpenAILanguageModel(apiKey: string, model: string, endPoin "OpenAI-Organization": org }; const proxy = resolveProxySettings(options); - if ((options?.useResponsesApi ?? isResponsesApiUrl(endPoint))) { - return createResponsesFetchLanguageModel(endPoint, headers, { model }, proxy); - } - return createFetchLanguageModel(endPoint, headers, { model }, proxy); + return createFetchLanguageModel(endPoint, headers, { model }, proxy, options?.useResponsesApi ?? isResponsesApiUrl(endPoint)); } /** @@ -237,7 +235,7 @@ const defaultMaxResponseBytes = 100 * 1024 * 1024; /** * Common OpenAI REST API endpoint encapsulation using the fetch API. */ -function createFetchLanguageModel(url: string, headers: object, defaultParams: object, proxy?: ProxySettings) { +function createFetchLanguageModel(url: string, headers: object, defaultParams: object, proxy?: ProxySettings, responsesApi = false) { let dispatcherPromise: Promise | undefined; const model: TypeChatLanguageModel = { complete @@ -245,12 +243,14 @@ function createFetchLanguageModel(url: string, headers: object, defaultParams: o return model; async function complete(prompt: string | PromptSection[]) { + const responses: TypeChatResponseMetadata[] = []; + const finish = (result: Result) => withResponseMetadata(result, responses); dispatcherPromise ??= resolveProxyDispatcher(proxy); let dispatcher: RequestInit["dispatcher"]; try { dispatcher = await dispatcherPromise; } catch (e) { - return error(`REST API fetch error: ${getErrorMessage(e)}`); + return finish(error(`REST API fetch error: ${getErrorMessage(e)}`)); } let retryCount = 0; const retryMaxAttempts = model.retryMaxAttempts ?? 3; @@ -261,7 +261,11 @@ function createFetchLanguageModel(url: string, headers: object, defaultParams: o while (true) { const options: RequestInit = { method: "POST", - body: JSON.stringify({ + body: JSON.stringify(responsesApi ? { + ...defaultParams, + input: prompt, + temperature: 0 + } : { ...defaultParams, messages, temperature: 0, @@ -280,28 +284,38 @@ function createFetchLanguageModel(url: string, headers: object, defaultParams: o response = await fetchWithTimeout(url, options, timeoutMs); } catch (e) { if (retryCount >= retryMaxAttempts) { - return error(`REST API fetch error: ${getErrorMessage(e)}`); + return finish(error(`REST API fetch error: ${getErrorMessage(e)}`)); } await sleep(retryPauseMs); retryCount++; continue; } + const parsed = await readModelResponse(response, maxResponseBytes, responsesApi, responses); + if (!parsed.success && response.ok) { + return finish(parsed); + } + const json = parsed.success ? parsed.data : undefined; if (response.ok) { - let json: { choices?: { message?: PromptSection }[] } | null; - try { - json = await readResponseJson(response, maxResponseBytes) as { choices?: { message?: PromptSection }[] } | null; - } catch (e) { - return error(`REST API response error: ${getErrorMessage(e)}`); + let content: unknown; + if (responsesApi) { + const output = record(json)?.output; + const message = Array.isArray(output) ? record(output.find(o => record(o)?.type === "message")) : undefined; + const parts = message?.content; + content = Array.isArray(parts) ? record(parts.find(c => record(c)?.type === "output_text"))?.text : undefined; + } else { + const choices = record(json)?.choices; + const choice = record(Array.isArray(choices) ? choices[0] : undefined); + content = record(choice?.message)?.content; } - const content = json?.choices?.[0]?.message?.content; - if (typeof content === "string") { - return success(content); + if (typeof content === "string" && !responses[responses.length - 1].filtering?.blocked) { + return finish(success(content)); } else { - return error(`REST API unexpected response format: ${JSON.stringify(json)}`); + return finish(error(`REST API unexpected response format: ${JSON.stringify(json)}`, + typeof content === "string" ? { responseText: content } : undefined)); } } if (!isTransientHttpError(response.status) || retryCount >= retryMaxAttempts) { - return error(`REST API error ${response.status}: ${response.statusText}`); + return finish(error(`REST API error ${response.status}: ${response.statusText}`)); } await sleep(getRetryDelayMs(response, retryPauseMs, retryPauseMs * retryMaxAttempts)); retryCount++; @@ -309,116 +323,83 @@ function createFetchLanguageModel(url: string, headers: object, defaultParams: o } } -/** - * OpenAI Responses API endpoint encapsulation using the fetch API. - * - * The Responses API uses a different request and response shape from Chat Completions: - * - **Request body**: `input` (string or array of `PromptSection`) instead of `messages`. - * - **Response body**: text is returned inside `output[n].content[m].text` where the matching - * output item has `type === "message"` and the content item has `type === "output_text"`. - * - * Example successful response: - * ```json - * { - * "id": "resp_...", - * "output": [ - * { - * "type": "message", - * "role": "assistant", - * "content": [{ "type": "output_text", "text": "Hello!" }] - * } - * ] - * } - * ``` - * - * @param url The Responses API endpoint URL (path should end with `/responses`). - * @param headers HTTP headers to include in every request (e.g. `Authorization`). - * @param defaultParams Additional JSON body parameters merged into every request (e.g. `{ model }`). - */ -function createResponsesFetchLanguageModel(url: string, headers: object, defaultParams: object, proxy?: ProxySettings) { - let dispatcherPromise: Promise | undefined; - const model: TypeChatLanguageModel = { - complete - }; - return model; +function record(value: unknown): Record | undefined { + return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : undefined; +} - async function complete(prompt: string | PromptSection[]) { - dispatcherPromise ??= resolveProxyDispatcher(proxy); - let dispatcher: RequestInit["dispatcher"]; - try { - dispatcher = await dispatcherPromise; - } catch (e) { - return error(`REST API fetch error: ${getErrorMessage(e)}`); +function responseMetadata(json: unknown, responsesApi: boolean): TypeChatResponseMetadata { + const body = record(json); + const metadata: TypeChatResponseMetadata = {}; + const reported = record(body?.usage); + const usage: TypeChatTokenUsage = {}; + const sourceKeys = responsesApi ? ["input_tokens", "output_tokens", "total_tokens"] : ["prompt_tokens", "completion_tokens", "total_tokens"]; + for (const [index, key] of (["inputTokens", "outputTokens", "totalTokens"] as const).entries()) { + const count = reported?.[sourceKeys[index]]; + if (typeof count === "number" && Number.isSafeInteger(count) && count >= 0) { + usage[key] = count; } - let retryCount = 0; - const retryMaxAttempts = model.retryMaxAttempts ?? 3; - const retryPauseMs = model.retryPauseMs ?? 1000; - const timeoutMs = model.timeoutMs ?? defaultTimeoutMs; - const maxResponseBytes = model.maxResponseBytes ?? defaultMaxResponseBytes; - const input = typeof prompt === "string" ? prompt : (prompt as PromptSection[]); - while (true) { - const options: RequestInit = { - method: "POST", - body: JSON.stringify({ - ...defaultParams, - input, - temperature: 0, - }), - headers: { - "content-type": "application/json", - ...headers - } - }; - if (dispatcher) { - options.dispatcher = dispatcher; - } - let response: Response; - try { - response = await fetchWithTimeout(url, options, timeoutMs); - } catch (e) { - if (retryCount >= retryMaxAttempts) { - return error(`REST API fetch error: ${getErrorMessage(e)}`); - } - await sleep(retryPauseMs); - retryCount++; - continue; - } - if (response.ok) { - type ResponsesAPIOutputItem = { - type: string; - role?: string; - content: { type: string; text: string }[]; - }; - let json: unknown; - try { - json = await readResponseJson(response, maxResponseBytes); - } catch (e) { - return error(`REST API response error: ${getErrorMessage(e)}`); - } - if ( - json === null || - typeof json !== "object" || - !("output" in json) || - !Array.isArray((json as { output?: unknown }).output) - ) { - return error(`REST API unexpected response format: ${JSON.stringify(json)}`); - } - const output = (json as { output: ResponsesAPIOutputItem[] }).output; - const message = output.find(o => o.type === "message"); - const textContent = message?.content?.find(c => c.type === "output_text"); - if (textContent?.text !== undefined) { - return success(textContent.text); - } else { - return error(`REST API unexpected response format: ${JSON.stringify(json)}`); + } + if (Object.keys(usage).length) { + metadata.usage = usage; + } + const choices = body?.choices; + const choice = record(Array.isArray(choices) ? choices[0] : undefined); + const failure = record(body?.error); + const inner = record(failure?.innererror); + const input = body?.prompt_filter_results ?? body?.prompt_annotations ?? inner?.content_filter_result; + const output = choice?.content_filter_results; + const filtering: NonNullable = {}; + if (input !== undefined) filtering.input = input; + if (output !== undefined) filtering.output = output; + const filtered = (rule: unknown) => record(rule)?.filtered === true; + const blocked = (details: unknown) => Object.values(record(details) ?? {}).some(rule => + filtered(rule) || Array.isArray(rule) && rule.some(filtered)); + if (blocked(output) || (Array.isArray(input) ? input.some(item => blocked(record(item)?.content_filter_results)) : blocked(input))) { + filtering.blocked = true; + } + const reason = responsesApi ? record(body?.incomplete_details)?.reason : choice?.finish_reason; + const code = failure?.code; + if (reason === "content_filter" || code === "content_filter" || code === "content_policy_violation" || inner?.code === "ResponsibleAIPolicyViolation") { + filtering.reason = typeof code === "string" ? code : "content_filter"; + filtering.blocked = true; + } + const refusals: string[] = []; + const refusal = record(choice?.message)?.refusal; + if (typeof refusal === "string" && refusal) refusals.push(refusal); + if (responsesApi && Array.isArray(body?.output)) { + for (const item of body.output) { + const content = record(item)?.content; + if (Array.isArray(content)) { + for (const part of content) { + const value = record(part); + if (value?.type === "refusal") { + filtering.blocked = true; + if (typeof value.refusal === "string") refusals.push(value.refusal); + } } } - if (!isTransientHttpError(response.status) || retryCount >= retryMaxAttempts) { - return error(`REST API error ${response.status}: ${response.statusText}`); - } - await sleep(getRetryDelayMs(response, retryPauseMs, retryPauseMs * retryMaxAttempts)); - retryCount++; } } + if (refusals.length) { + filtering.refusals = refusals; + filtering.blocked = true; + } + if (Object.keys(filtering).length) metadata.filtering = filtering; + return metadata; +} + +async function readModelResponse(response: Response, maxBytes: number, responsesApi: boolean, responses: TypeChatResponseMetadata[]): Promise> { + let json: unknown; + try { + if (response.ok || response.headers.get("content-type")?.toLowerCase().includes("json")) { + json = await readResponseJson(response, maxBytes); + } + } catch (e) { + responses.push(response.ok ? {} : { error: e instanceof Error ? e.name : "ResponseReadError" }); + return error(`REST API response error: ${getErrorMessage(e)}`); + } + responses.push(responseMetadata(json, responsesApi)); + return success(json); } /** diff --git a/typescript/src/result.ts b/typescript/src/result.ts index d2b8f4c0..12ea7de6 100644 --- a/typescript/src/result.ts +++ b/typescript/src/result.ts @@ -1,7 +1,40 @@ +/** Service-reported counts. Missing counts are unknown, not zero. */ +export interface TypeChatTokenUsage { + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; +} + +/** Metadata for one response, excluding generated text and raw response bodies. */ +export interface TypeChatResponseMetadata { + usage?: TypeChatTokenUsage; + /** Error class when an HTTP error body's metadata could not be read. */ + error?: string; + filtering?: { + blocked?: boolean; + reason?: string; + refusals?: string[]; + /** Provider-returned filtering details, not application data. */ + input?: unknown; + output?: unknown; + }; +} + +/** + * Ordered response records and reported-only totals, including repair responses. + * @see https://github.com/microsoft/TypeChat/blob/main/typescript/metadata.md + */ +export interface TypeChatMetadata { + responses: TypeChatResponseMetadata[]; + usage?: TypeChatTokenUsage; + /** Coverage of each reported total; absent when no response reported that count. */ + usageReporting?: { [K in keyof TypeChatTokenUsage]?: "complete" | "partial" }; +} + /** * An object representing a successful operation with a result of type `T`. */ -export type Success = { success: true, data: T }; +export type Success = { success: true, data: T, metadata?: TypeChatMetadata }; /** * An object representing an operation that failed for the reason given in `message`. @@ -19,6 +52,7 @@ export type Error = { * `data` if a repair response couldn't be parsed. */ responseText?: string; + metadata?: TypeChatMetadata; }; /** @@ -38,10 +72,10 @@ export function success(data: T): Success { /** * Returns an `Error` object. * @param message The value for the `message` property of the result. - * @param details Optional unvalidated data and original model response text. + * @param details Optional unvalidated data, original model response text, and metadata. * @returns An `Error` object. */ -export function error(message: string, details?: Pick): Error { +export function error(message: string, details?: Pick): Error { const result: Error = { success: false, message }; if (details) { if ("data" in details) { @@ -51,10 +85,54 @@ export function error(message: string, details?: Pick(result: Result, responses: TypeChatResponseMetadata[]): Result { + if (!responses.some(response => response.usage !== undefined || response.filtering !== undefined || response.error !== undefined)) { + return result; + } + const metadata: TypeChatMetadata = { responses: [...responses] }; + const usage: TypeChatTokenUsage = {}; + const reporting: NonNullable = {}; + for (const key of ["inputTokens", "outputTokens", "totalTokens"] as const) { + let total = 0; + let reported = 0; + for (const response of responses) { + const count = response.usage?.[key]; + if (count !== undefined) { + total += count; + reported++; + } + } + if (reported) { + usage[key] = total; + reporting[key] = reported === responses.length ? "complete" : "partial"; + } + } + if (Object.keys(usage).length) { + metadata.usage = usage; + metadata.usageReporting = reporting; + } + if (result.success) { + return { ...result, success: true, data: result.data, metadata }; + } + return { + ...result, + success: false, + message: result.message, + ...("data" in result ? { data: result.data } : {}), + ...(result.responseText !== undefined ? { responseText: result.responseText } : {}), + metadata + }; +} + /** * Obtains the value associated with a successful `Result` or throws an exception if * the result is an error. diff --git a/typescript/src/typechat.ts b/typescript/src/typechat.ts index af7a6a8b..164b0a49 100644 --- a/typescript/src/typechat.ts +++ b/typescript/src/typechat.ts @@ -1,4 +1,4 @@ -import { Result, success, error, Error as TypeChatError } from "./result"; +import { Result, success, error, withResponseMetadata, TypeChatResponseMetadata, Error as TypeChatError } from "./result"; import { TypeChatLanguageModel, PromptSection, PromptContent } from "./model"; /** @@ -57,6 +57,7 @@ export interface TypeChatJsonTranslator { * diagnostics produced for the first attempt. This often helps produce a valid instance. * A failure retains the latest rejected candidate and available model response text. These may * come from different attempts if a repair response couldn't be parsed. + * Optional result metadata retains response records and reported usage across both attempts. * @param request The natural language request. * @param promptPreamble An optional string or array of prompt sections to prepend to the generated * prompt. If a string is specified, it is converted into a single "user" role prompt section. @@ -128,24 +129,27 @@ export function createJsonTranslator(model: TypeChatLanguageMo let prompt: PromptSection[] = [...preamble, { role: "user", content: typeChat.createRequestPrompt(request) }]; let attemptRepair = typeChat.attemptRepair; let failureContext: (Pick & { responseText: string }) | undefined; + const responses: TypeChatResponseMetadata[] = []; + const finish = (result: Result) => withResponseMetadata(result, responses); while (true) { const response = await model.complete(prompt); + responses.push(...(response.metadata?.responses ?? [{}])); if (!response.success) { - return failureContext ? { + return finish(failureContext ? { ...failureContext, ...response, success: false, message: response.message, ...("data" in response ? { data: response.data } : {}), responseText: response.responseText ?? failureContext.responseText - } : response; + } : response); } const responseText = response.data; failureContext = { ...failureContext, responseText }; const startIndex = responseText.indexOf("{"); const endIndex = responseText.lastIndexOf("}"); if (!(startIndex >= 0 && endIndex > startIndex)) { - return error(`Response is not JSON:\n${responseText}`, failureContext); + return finish(error(`Response is not JSON:\n${responseText}`, failureContext)); } const jsonText = responseText.slice(startIndex, endIndex + 1); let jsonObject; @@ -153,7 +157,7 @@ export function createJsonTranslator(model: TypeChatLanguageMo jsonObject = JSON.parse(jsonText) as object; } catch (e) { - return error(e instanceof SyntaxError ? e.message : "JSON parse error", failureContext); + return finish(error(e instanceof SyntaxError ? e.message : "JSON parse error", failureContext)); } if (typeChat.stripNulls) { stripNulls(jsonObject); @@ -161,7 +165,7 @@ export function createJsonTranslator(model: TypeChatLanguageMo const schemaValidation = validator.validate(jsonObject); const validation = schemaValidation.success ? typeChat.validateInstance(schemaValidation.data) : schemaValidation; if (validation.success) { - return validation; + return finish(validation); } const failure: TypeChatError & { responseText: string } = { ...validation, @@ -172,7 +176,7 @@ export function createJsonTranslator(model: TypeChatLanguageMo }; failureContext = { data: failure.data, responseText: failure.responseText }; if (!attemptRepair) { - return { ...failure, message: `JSON validation failed: ${failure.message}\n${jsonText}` }; + return finish({ ...failure, message: `JSON validation failed: ${failure.message}\n${jsonText}` }); } prompt.push({ role: "assistant", content: responseText }); prompt.push({ role: "user", content: typeChat.createRepairPrompt(failure.message) }); diff --git a/typescript/test/metadata.test.ts b/typescript/test/metadata.test.ts new file mode 100644 index 00000000..7d63df9e --- /dev/null +++ b/typescript/test/metadata.test.ts @@ -0,0 +1,258 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + createOpenAILanguageModel, createAzureOpenAILanguageModel, createJsonTranslator, + success, error, getData, withResponseMetadata, + Result, TypeChatLanguageModel, TypeChatJsonValidator, TypeChatResponseMetadata, PromptSection +} from "../dist/index.js"; +import { createTypeScriptJsonValidator } from "../dist/ts/index.js"; + +type Payload = { value: number }; +const valid = '{"value":42}'; +const invalid = '{"value":"bad"}'; +const first = { inputTokens: 13, outputTokens: 7, totalTokens: 20 }; +const second = { inputTokens: 5, outputTokens: 3, totalTokens: 8 }; +const validator: TypeChatJsonValidator = { + getSchemaText: () => "export interface Payload { value: number; }", + getTypeName: () => "Payload", + validate: value => "value" in value && typeof value.value === "number" ? + success({ value: value.value }) : error("bad value") +}; +const tagged = (result: Result, metadata: TypeChatResponseMetadata) => withResponseMetadata(result, [metadata]); + +function translatorFor(results: Result[], schema = validator) { + let calls = 0; + const prompts: (string | PromptSection[])[] = []; + // The pre-metadata custom-model signature remains assignable. + const model: TypeChatLanguageModel = { + async complete(prompt): Promise> { + prompts.push(typeof prompt === "string" ? prompt : [...prompt]); + const result = results[calls++]; + assert.ok(result, "unexpected model call"); + return result; + } + }; + return { translator: createJsonTranslator(model, schema), calls: () => calls, prompts }; +} + +for (const api of ["chat", "responses", "azure"] as const) { + const factory = () => api === "azure" ? createAzureOpenAILanguageModel("test", "https://example.test/chat/completions") : + createOpenAILanguageModel("test", "test", `https://example.test/${api === "responses" ? "responses" : "chat/completions"}`); + const body = (text: unknown, usage?: unknown) => ({ + id: "raw-body-not-metadata", usage, + ...(api === "responses" ? { output: [{ type: "message", content: [{ type: "output_text", text }] }] } : + { choices: [{ message: { content: text } }] }) + }); + const wireUsage = (input: number, output: number, total: number) => api === "responses" ? + { input_tokens: input, output_tokens: output, total_tokens: total } : + { prompt_tokens: input, completion_tokens: output, total_tokens: total }; + + test(`${api}: reported counts, zeros, absent and partial metadata`, async t => { + for (const [usage, expected] of [ + [wireUsage(13, 7, 20), first], [wireUsage(0, 0, 0), { inputTokens: 0, outputTokens: 0, totalTokens: 0 }], + [api === "responses" ? { input_tokens: 0 } : { prompt_tokens: 0 }, { inputTokens: 0 }], + [undefined, undefined], [{}, undefined], [null, undefined] + ] as const) { + const fetch = t.mock.method(globalThis, "fetch", async () => Response.json(body(valid, usage))); + const result = await factory().complete("request"); + assert.ok(result.success); + const text: string = result.data; + assert.equal(text, valid); + assert.deepEqual(result.metadata?.usage, expected); + if (expected) { + assert.equal(result.metadata?.responses[0].filtering, undefined); + assert.deepEqual(result.metadata?.responses, [{ usage: expected }]); + assert.ok(!JSON.stringify(result.metadata).includes("raw-body-not-metadata")); + } else assert.ok(!("metadata" in result)); + assert.equal(fetch.mock.callCount(), 1); + fetch.mock.restore(); + } + }); + + test(`${api}: filtering and refusal failures retain usage`, async t => { + const json = body(valid, wireUsage(13, 7, 20)); + const filtered = api === "responses" ? { + ...json, incomplete_details: { reason: "content_filter" }, + output: [{ type: "message", content: [{ type: "refusal", refusal: "declined" }] }] + } : { ...json, choices: [{ finish_reason: "content_filter", message: { content: valid, refusal: "declined" } }] }; + const fetch = t.mock.method(globalThis, "fetch", async () => Response.json(filtered)); + const translator = createJsonTranslator(factory(), validator); + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.match(result.message, /unexpected response format/); + assert.deepEqual(result.metadata?.usage, first); + assert.deepEqual(result.metadata?.responses[0].filtering, { reason: "content_filter", blocked: true, refusals: ["declined"] }); + assert.equal(result.responseText, api === "responses" ? undefined : valid); + assert.throws(() => getData(result), /unexpected response format/); + assert.equal(fetch.mock.callCount(), 1); + }); + + test(`${api}: malformed successful response retains metadata`, async t => { + t.mock.method(globalThis, "fetch", async () => Response.json({ usage: wireUsage(13, 7, 20) })); + const result = await factory().complete("request"); + assert.ok(!result.success); + assert.deepEqual(result.metadata?.usage, first); + }); +} + +test("Azure filtering details and HTTP content-filter errors are bounded metadata", async t => { + const input = [{ prompt_index: 0, content_filter_results: { hate: { filtered: false, severity: "safe" } } }]; + const output = { violence: { filtered: false, severity: "safe" } }; + const payloads = [ + { prompt_filter_results: input, choices: [{ message: { content: valid }, content_filter_results: output }] }, + { choices: [{ message: { content: valid }, content_filter_results: { hate: { filtered: true } } }] }, + { choices: [{ message: { content: valid }, content_filter_results: { custom_blocklists: [{ filtered: true }] } }] }, + { error: { code: "content_filter", message: "private error text", innererror: { content_filter_result: output } } } + ]; + t.mock.method(globalThis, "fetch", async () => Response.json(payloads.shift(), { status: payloads.length ? 200 : 400, statusText: "Bad Request" })); + const model = createAzureOpenAILanguageModel("test", "https://example.test/chat/completions"); + const ok = await model.complete("request"); + assert.ok(ok.success); + assert.deepEqual(ok.metadata?.responses, [{ filtering: { input, output } }]); + assert.equal(ok.metadata?.usage, undefined); + for (let index = 0; index < 2; index++) { + const blocked = await model.complete("request"); + assert.ok(!blocked.success); + assert.equal(blocked.metadata?.responses[0].filtering?.blocked, true); + assert.equal(blocked.responseText, valid); + } + const failed = await model.complete("request"); + assert.ok(!failed.success); + assert.equal(failed.message, "REST API error 400: Bad Request"); + assert.deepEqual(failed.metadata?.responses, [{ filtering: { input: output, reason: "content_filter", blocked: true } }]); + assert.ok(!JSON.stringify(failed.metadata).includes("private error text")); +}); + +test("repair totals count each ordered response once and stay outside validated data", async () => { + const { translator, calls, prompts } = translatorFor([tagged(success(invalid), { usage: first }), tagged(success(valid), { usage: second })]); + const result = await translator.translate("request"); + assert.ok(result.success); + const data: Payload = getData(result); + assert.deepEqual(data, { value: 42 }); + assert.deepEqual(result.metadata, { + responses: [{ usage: first }, { usage: second }], + usage: { inputTokens: 18, outputTokens: 10, totalTokens: 28 }, + usageReporting: { inputTokens: "complete", outputTokens: "complete", totalTokens: "complete" } + }); + assert.equal(calls(), 2); + assert.ok(!JSON.stringify(prompts).includes("inputTokens")); + assert.ok(!translator.validator.getSchemaText().includes("metadata")); +}); + +test("partial reporting distinguishes missing counts, zero, and missing metadata", async () => { + for (const tail of [{ usage: { inputTokens: 0 } }, {}, { filtering: { input: [] } }]) { + const { translator } = translatorFor([ + tagged(success(invalid), { usage: { inputTokens: 13, outputTokens: 7 } }), tagged(success(valid), tail) + ]); + const result = await translator.translate("request"); + assert.deepEqual(result.metadata?.usage, { inputTokens: 13, outputTokens: 7 }); + assert.deepEqual(result.metadata?.usageReporting, { inputTokens: "usage" in tail ? "complete" : "partial", outputTokens: "partial" }); + assert.deepEqual(result.metadata?.responses[1], tail); + assert.ok(!("totalTokens" in (result.metadata?.usage ?? {}))); + } +}); + +test("parse, schema and custom failures preserve messages, rejected data and responseText", async () => { + const schema = createTypeScriptJsonValidator(validator.getSchemaText(), "Payload"); + try { + for (const text of ["not JSON", '{"value":}', invalid, valid]) { + const plain = translatorFor([success(text)], schema).translator; + const { translator, calls } = translatorFor([tagged(success(text), { usage: first })], schema); + plain.attemptRepair = translator.attemptRepair = false; + if (text === valid) { + plain.validateInstance = translator.validateInstance = () => error("custom", { data: null, responseText: "" }); + } + const expected = await plain.translate("request"); + const { metadata, ...actual } = await translator.translate("request"); + assert.ok(!actual.success); + assert.deepEqual(actual, expected); + assert.deepEqual(metadata?.usage, first); + assert.equal(calls(), 1); + assert.throws(() => getData(actual)); + if (text === valid) { + assert.equal(actual.data, null); + assert.equal(actual.responseText, ""); + } + } + } finally { schema.close(); } +}); + +test("HTTP retries retain reported metadata, including a later transport failure", async t => { + for (const fail of [false, true]) { + let calls = 0; + const fetch = t.mock.method(globalThis, "fetch", async () => { + if (calls++ === 0) return Response.json({ usage: { prompt_tokens: 13 } }, { status: 503 }); + if (fail) throw new Error("offline transport failure"); + return Response.json({ choices: [{ message: { content: valid } }], usage: { prompt_tokens: 5 } }); + }); + const model = createOpenAILanguageModel("test", "test"); + model.retryMaxAttempts = 1; + model.retryPauseMs = 0; + const result = await model.complete("request"); + assert.equal(result.success, !fail); + assert.deepEqual(result.metadata?.usage, { inputTokens: fail ? 13 : 18 }); + assert.equal(result.metadata?.responses.length, fail ? 1 : 2); + assert.equal(calls, 2); + fetch.mock.restore(); + } +}); + +test("unreadable error metadata preserves HTTP messages and retry limits", async t => { + for (const status of [400, 503]) { + let calls = 0; + const fetch = t.mock.method(globalThis, "fetch", async () => ++calls === 1 ? + new Response("not JSON", { status, statusText: "Rejected", headers: { "content-type": "application/json" } }) : + Response.json({ choices: [{ message: { content: valid } }], usage: { prompt_tokens: 0 } })); + const model = createOpenAILanguageModel("test", "test"); + model.retryMaxAttempts = 1; + model.retryPauseMs = 0; + const result = await model.complete("request"); + assert.equal(result.success, status === 503); + if (!result.success) assert.equal(result.message, "REST API error 400: Rejected"); + assert.deepEqual(result.metadata?.responses[0], { error: "SyntaxError" }); + assert.deepEqual(result.metadata?.usage, status === 503 ? { inputTokens: 0 } : undefined); + assert.equal(result.metadata?.usageReporting?.inputTokens, status === 503 ? "partial" : undefined); + assert.equal(calls, status === 503 ? 2 : 1); + fetch.mock.restore(); + } +}); + +test("initial and repair failures retain all metadata without extending repair limits", async () => { + const initial = tagged(error("initial failed"), { usage: first }); + assert.deepEqual(await translatorFor([initial]).translator.translate("request"), initial); + for (const tail of [tagged(error("repair failed"), { usage: second }), error("repair failed"), + tagged(success("not JSON"), { usage: second }), tagged(success('{"value":}'), { usage: second }), tagged(success(invalid), { usage: second })]) { + const { translator, calls } = translatorFor([tagged(success(invalid), { usage: first }), tail]); + const result = await translator.translate("request"); + assert.ok(!result.success); + assert.deepEqual(result.data, { value: "bad" }); + assert.equal(result.responseText, tail.success ? tail.data : invalid); + assert.equal(result.metadata?.responses.length, 2); + assert.deepEqual(result.metadata?.usage, tail.metadata ? { inputTokens: 18, outputTokens: 10, totalTokens: 28 } : first); + assert.equal(result.metadata?.usageReporting?.inputTokens, tail.metadata ? "complete" : "partial"); + assert.equal(calls(), 2); + } +}); + +test("translations are isolated and metadata-free custom models retain their old shape", async () => { + const { translator } = translatorFor([tagged(success(valid), { usage: first }), tagged(success(valid), { usage: second }), success(valid)]); + const [a, b] = await Promise.all([translator.translate("a"), translator.translate("b")]); + assert.deepEqual(a.metadata?.usage, first); + assert.deepEqual(b.metadata?.usage, second); + assert.notEqual(a.metadata?.responses, b.metadata?.responses); + assert.deepEqual(await translator.translate("c"), success({ value: 42 })); +}); + +test("metadata preserves inherited envelope fields and leaves the original unchanged", () => { + const original = new class { + get success(): false { return false; } + get message() { return "custom"; } + get data(): unknown { return undefined; } + get responseText() { return ""; } + }; + const { metadata, ...result } = withResponseMetadata(original, [{ usage: first }]); + assert.deepEqual(result, { success: false, message: "custom", data: undefined, responseText: "" }); + assert.deepEqual(metadata?.usage, first); + assert.ok(!("metadata" in original)); + assert.deepEqual(error("custom", { metadata }), { success: false, message: "custom", metadata }); +}); From 58ee0f8c285227e4096194a95f25e9f9d38f24a2 Mon Sep 17 00:00:00 2001 From: Robert Gruen Date: Mon, 2 Nov 2026 19:03:14 -0700 Subject: [PATCH 2/4] Run metadata regressions in the default TypeScript test command Include the metadata suite in ordinary npm test and CI instead of only compiling it. Update the documented command. This owner-approved follow-up leaves the original automated publication evidence unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- typescript/metadata.md | 4 ++-- typescript/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/typescript/metadata.md b/typescript/metadata.md index 395d5afb..6bd3b26c 100644 --- a/typescript/metadata.md +++ b/typescript/metadata.md @@ -51,8 +51,8 @@ metadata without altering text, use `withResponseMetadata(result, [{ usage, filt computes reported totals from those records. Translation recomputes from records, not existing totals, so each response contributes exactly once and separate translations stay isolated. -Run the offline mocked regression suite from `typescript/` with `npm test -- out/metadata.test.js`. -This explicitly adds metadata tests to the existing test command without changing package scripts. +Run the offline mocked regression suite from `typescript/` with `npm test`. +The default test command includes the metadata, repair-accounting, and failure-retention tests. No live provider access or credentials are required. Thanks to [intuity-hans](https://github.com/intuity-hans) for the prior work in diff --git a/typescript/package.json b/typescript/package.json index 49150810..ed11ebbe 100644 --- a/typescript/package.json +++ b/typescript/package.json @@ -21,7 +21,7 @@ }, "scripts": { "build": "tsc -p src", - "test": "npm run build && tsc -p test && node --test out/validate.test.js out/zod.test.js out/program.test.js out/failure.test.js out/interactive.test.js out/interactive-stdin.test.js tests/model.test.mjs tests/examples.test.mjs tests/package.test.mjs", + "test": "npm run build && tsc -p test && node --test out/validate.test.js out/zod.test.js out/program.test.js out/failure.test.js out/metadata.test.js out/interactive.test.js out/interactive-stdin.test.js tests/model.test.mjs tests/examples.test.mjs tests/package.test.mjs", "build-all": "npm run build && npm run build --workspaces", "prepare": "npm run build-all", "prepublishOnly": "node -e \"require('fs').copyFileSync('../SECURITY.md','SECURITY.md')\"", From fa9d14c7f6786b9ad22de42f4effc67339ae94bb Mon Sep 17 00:00:00 2001 From: Robert Gruen Date: Mon, 2 Nov 2026 20:26:56 -0700 Subject: [PATCH 3/4] Report filtered replies safely and guard token aggregate overflow Separate filtered and refused replies from response-format errors without dumping provider bodies into messages. Preserve response records and application results while explicitly marking unrepresentable token totals as overflow. Add boundary, retry, repair, and error-message regressions and document the owner-approved behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- typescript/metadata.md | 12 ++++- typescript/src/model.ts | 9 ++-- typescript/src/result.ts | 23 +++++++--- typescript/test/metadata.test.ts | 75 +++++++++++++++++++++++++++++++- 4 files changed, 107 insertions(+), 12 deletions(-) diff --git a/typescript/metadata.md b/typescript/metadata.md index 6bd3b26c..a5c89b3e 100644 --- a/typescript/metadata.md +++ b/typescript/metadata.md @@ -8,7 +8,7 @@ the default limit of one repair are unchanged. ```ts const result = await translator.translate(request); const tokens = result.metadata?.usage?.inputTokens; // undefined: unknown; 0: reported zero -const coverage = result.metadata?.usageReporting?.inputTokens; // "complete" or "partial" +const coverage = result.metadata?.usageReporting?.inputTokens; // "complete", "partial", or "overflow" if (result.success) { const applicationData = result.data; } else { @@ -23,7 +23,10 @@ has metadata; if all metadata is absent, the result stays unchanged. Built-in cl metadata from JSON HTTP errors and responses preceding HTTP retries, without extra service or moderation requests. An unreadable JSON HTTP error body adds a record's `error` diagnostic (the error class, not the body), preserving status messages and HTTP retry behavior. Filtered/refused -output remains failure, even if it contains valid-looking JSON. +output remains failure, even if it contains valid-looking JSON. An otherwise successful HTTP +response that is filtered or refused returns a concise error rather than putting the complete +provider body or refusal text in the error message. Filtering details remain in metadata, and +available text remains in `responseText`. `metadata.usage` sums **reported counts only**, once per record: input/output/total 13/7/20 plus 5/3/8 yields 18/10/28. Each corresponding `metadata.usageReporting` field is `"complete"` if every @@ -33,6 +36,11 @@ clients accept nonnegative safe integers; missing/null counts are unknown, not z reporting excludes unreported requests, failed transports, and usage the service omits. These totals are not estimates of all billable usage; TypeChat does not calculate prices. +If a reported aggregate would exceed JavaScript's safe-integer range, that count is omitted from +`metadata.usage` and its `metadata.usageReporting` status is `"overflow"`. Individual response +records, other representable totals, and the application validation result remain intact; the +aggregate is never rounded, clamped, or represented as zero. + | Existing API | `usage` fields mapped to `inputTokens` / `outputTokens` / `totalTokens` | Filtering sources | | --- | --- | --- | | OpenAI Chat Completions | `prompt_tokens` / `completion_tokens` / `total_tokens` | First choice's `finish_reason: "content_filter"`, `message.refusal` | diff --git a/typescript/src/model.ts b/typescript/src/model.ts index 7e686be0..e550f66d 100644 --- a/typescript/src/model.ts +++ b/typescript/src/model.ts @@ -307,11 +307,14 @@ function createFetchLanguageModel(url: string, headers: object, defaultParams: o const choice = record(Array.isArray(choices) ? choices[0] : undefined); content = record(choice?.message)?.content; } - if (typeof content === "string" && !responses[responses.length - 1].filtering?.blocked) { + if (responses[responses.length - 1].filtering?.blocked) { + return finish(error("REST API response was filtered or refused.", + typeof content === "string" ? { responseText: content } : undefined)); + } + if (typeof content === "string") { return finish(success(content)); } else { - return finish(error(`REST API unexpected response format: ${JSON.stringify(json)}`, - typeof content === "string" ? { responseText: content } : undefined)); + return finish(error(`REST API unexpected response format: ${JSON.stringify(json)}`)); } } if (!isTransientHttpError(response.status) || retryCount >= retryMaxAttempts) { diff --git a/typescript/src/result.ts b/typescript/src/result.ts index 12ea7de6..1bb15a26 100644 --- a/typescript/src/result.ts +++ b/typescript/src/result.ts @@ -27,8 +27,8 @@ export interface TypeChatResponseMetadata { export interface TypeChatMetadata { responses: TypeChatResponseMetadata[]; usage?: TypeChatTokenUsage; - /** Coverage of each reported total; absent when no response reported that count. */ - usageReporting?: { [K in keyof TypeChatTokenUsage]?: "complete" | "partial" }; + /** Coverage of each total; overflow marks an unrepresentable total omitted from usage. */ + usageReporting?: { [K in keyof TypeChatTokenUsage]?: "complete" | "partial" | "overflow" }; } /** @@ -104,20 +104,33 @@ export function withResponseMetadata(result: Result, responses: TypeChatRe for (const key of ["inputTokens", "outputTokens", "totalTokens"] as const) { let total = 0; let reported = 0; + let overflow = false; for (const response of responses) { const count = response.usage?.[key]; if (count !== undefined) { - total += count; reported++; + if (!overflow) { + if (count > Number.MAX_SAFE_INTEGER - total) { + overflow = true; + } else { + total += count; + } + } } } if (reported) { - usage[key] = total; - reporting[key] = reported === responses.length ? "complete" : "partial"; + if (overflow) { + reporting[key] = "overflow"; + } else { + usage[key] = total; + reporting[key] = reported === responses.length ? "complete" : "partial"; + } } } if (Object.keys(usage).length) { metadata.usage = usage; + } + if (Object.keys(reporting).length) { metadata.usageReporting = reporting; } if (result.success) { diff --git a/typescript/test/metadata.test.ts b/typescript/test/metadata.test.ts index 7d63df9e..af63dc84 100644 --- a/typescript/test/metadata.test.ts +++ b/typescript/test/metadata.test.ts @@ -79,11 +79,13 @@ for (const api of ["chat", "responses", "azure"] as const) { const translator = createJsonTranslator(factory(), validator); const result = await translator.translate("request"); assert.ok(!result.success); - assert.match(result.message, /unexpected response format/); + assert.match(result.message, /filtered or refused/); + assert.ok(!result.message.includes("raw-body-not-metadata")); + assert.ok(!result.message.includes("declined")); assert.deepEqual(result.metadata?.usage, first); assert.deepEqual(result.metadata?.responses[0].filtering, { reason: "content_filter", blocked: true, refusals: ["declined"] }); assert.equal(result.responseText, api === "responses" ? undefined : valid); - assert.throws(() => getData(result), /unexpected response format/); + assert.throws(() => getData(result), /filtered or refused/); assert.equal(fetch.mock.callCount(), 1); }); @@ -91,6 +93,7 @@ for (const api of ["chat", "responses", "azure"] as const) { t.mock.method(globalThis, "fetch", async () => Response.json({ usage: wireUsage(13, 7, 20) })); const result = await factory().complete("request"); assert.ok(!result.success); + assert.match(result.message, /unexpected response format/); assert.deepEqual(result.metadata?.usage, first); }); } @@ -152,6 +155,74 @@ test("partial reporting distinguishes missing counts, zero, and missing metadata } }); +test("checked usage totals preserve records and valid data when an aggregate overflows", () => { + const records = [ + { usage: { inputTokens: Number.MAX_SAFE_INTEGER, outputTokens: 1 } }, + { usage: { inputTokens: 2, outputTokens: 2 } } + ]; + const original = success({ value: 42 }); + const result = withResponseMetadata(original, records); + assert.deepEqual(getData(result), { value: 42 }); + assert.deepEqual(original, success({ value: 42 })); + assert.deepEqual(result.metadata?.responses, records); + assert.deepEqual(result.metadata?.usage, { outputTokens: 3 }); + assert.deepEqual(result.metadata?.usageReporting, { inputTokens: "overflow", outputTokens: "complete" }); + for (const counts of [[Number.MAX_SAFE_INTEGER - 1, 1], [Number.MAX_SAFE_INTEGER, 0]]) { + const boundary = withResponseMetadata(success(valid), counts.map(inputTokens => ({ usage: { inputTokens } }))); + assert.deepEqual(boundary.metadata?.usage, { inputTokens: Number.MAX_SAFE_INTEGER }); + assert.deepEqual(boundary.metadata?.usageReporting, { inputTokens: "complete" }); + } +}); + +test("HTTP retries mark overflow even when no aggregate can be represented", async t => { + let calls = 0; + t.mock.method(globalThis, "fetch", async () => { + calls++; + return Response.json({ + usage: { prompt_tokens: calls === 1 ? Number.MAX_SAFE_INTEGER : 2 }, + ...(calls === 1 ? {} : { choices: [{ message: { content: valid } }] }) + }, { status: calls === 1 ? 503 : 200 }); + }); + const model = createOpenAILanguageModel("test", "test"); + model.retryMaxAttempts = 1; + model.retryPauseMs = 0; + const result = await model.complete("request"); + assert.ok(result.success); + assert.equal(result.data, valid); + assert.equal(calls, 2); + assert.deepEqual(result.metadata?.responses, [ + { usage: { inputTokens: Number.MAX_SAFE_INTEGER } }, + { usage: { inputTokens: 2 } } + ]); + assert.equal(result.metadata?.usage, undefined); + assert.deepEqual(result.metadata?.usageReporting, { inputTokens: "overflow" }); +}); + +test("repair overflow is explicit on success and failure without discarding responses", async () => { + for (const tail of [success(valid), error("repair failed")]) { + const { translator, calls } = translatorFor([ + tagged(success(invalid), { usage: { inputTokens: Number.MAX_SAFE_INTEGER } }), + tagged(tail, { usage: { inputTokens: 2 } }) + ]); + const result = await translator.translate("request"); + assert.equal(result.success, tail.success); + if (result.success) { + assert.deepEqual(result.data, { value: 42 }); + } else { + assert.equal(result.message, "repair failed"); + assert.deepEqual(result.data, { value: "bad" }); + assert.equal(result.responseText, invalid); + } + assert.equal(calls(), 2); + assert.deepEqual(result.metadata?.responses, [ + { usage: { inputTokens: Number.MAX_SAFE_INTEGER } }, + { usage: { inputTokens: 2 } } + ]); + assert.equal(result.metadata?.usage, undefined); + assert.deepEqual(result.metadata?.usageReporting, { inputTokens: "overflow" }); + } +}); + test("parse, schema and custom failures preserve messages, rejected data and responseText", async () => { const schema = createTypeScriptJsonValidator(validator.getSchemaText(), "Payload"); try { From 1c1abe6f9566ecd7d691596344911ce1f30f83bd Mon Sep 17 00:00:00 2001 From: Robert Gruen Date: Mon, 2 Nov 2026 21:39:50 -0700 Subject: [PATCH 4/4] docs: ship and link response metadata guide Include the guide in npm packages, link it from the published README, and verify the exact packaged guide and link target in the existing clean-consumer regression test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- typescript/README.md | 7 +++++++ typescript/package.json | 1 + typescript/tests/package.test.mjs | 10 ++++++++++ 3 files changed, 18 insertions(+) diff --git a/typescript/README.md b/typescript/README.md index 06f37b22..defb1fcd 100644 --- a/typescript/README.md +++ b/typescript/README.md @@ -73,6 +73,13 @@ or `undefined`, so check property presence when that distinction matters. Rejected values and response text may contain sensitive output. Do not log them automatically. +## Accessing response metadata + +Model and translation results can include optional `metadata` on success and failure, +separate from application data. See the [response metadata guide](https://github.com/microsoft/TypeChat/blob/main/typescript/metadata.md) +for reported token usage, initial and repair response records, and available filtering details. +The guide is also included as `metadata.md` in the npm package. + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/typescript/package.json b/typescript/package.json index ed11ebbe..26f0daee 100644 --- a/typescript/package.json +++ b/typescript/package.json @@ -44,6 +44,7 @@ "dist", "LICENSE", "README.md", + "metadata.md", "SECURITY.md" ], "peerDependencies": { diff --git a/typescript/tests/package.test.mjs b/typescript/tests/package.test.mjs index 4f37b99b..6a7a5e0a 100644 --- a/typescript/tests/package.test.mjs +++ b/typescript/tests/package.test.mjs @@ -87,6 +87,16 @@ test("published package works for a clean TypeScript 7 consumer", () => { stdio: "ignore", timeout: 120_000, }); + assert.equal( + readFileSync(path.join(consumerPackage, "metadata.md"), "utf8"), + readFileSync(path.join(packageRoot, "metadata.md"), "utf8") + ); + const readme = readFileSync(path.join(consumerPackage, "README.md"), "utf8"); + assert.equal( + readme.match(/\[response metadata guide\]\(([^)]+)\)/)?.[1], + "https://github.com/microsoft/TypeChat/blob/main/typescript/metadata.md", + "Packaged README must link to the response metadata guide" + ); symlinkSync(path.join(buildPackage, "node_modules", "typescript"), path.join(consumerModules, "typescript"), directoryLinkType); const installedTypeScript = JSON.parse(readFileSync(path.join(consumerModules, "typescript", "package.json"), "utf8")); assert.match(installedTypeScript.version, /^7\./);