From 4c558f4c8e482eed1a31040e9cd70e5a25ff2ff0 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 17:15:45 -0800 Subject: [PATCH 1/9] feat(core): add opt-in TypeScript attachment runtime Provide deny-wins admission, bounded Bytes/Blob transfers, explicit correlation and provider lifecycle integration. Keep unqualified retrieval and streaming paths closed. Co-Authored-By: Claude Code --- .../workflows/fi-core-attachments-tests.yml | 60 ++++ typescript/packages/fi-core/package.json | 13 +- .../packages/fi-core/scripts/postbuild.cjs | 10 + .../fi-core/src/attachments/correlation.ts | 65 ++++ .../packages/fi-core/src/attachments/index.ts | 7 + .../fi-core/src/attachments/manager.ts | 291 ++++++++++++++++++ .../packages/fi-core/src/attachments/node.ts | 62 ++++ .../fi-core/src/attachments/policy.ts | 59 ++++ .../fi-core/src/attachments/transport-v1.ts | 139 +++++++++ .../packages/fi-core/src/attachments/types.ts | 154 +++++++++ typescript/packages/fi-core/src/index.ts | 2 + typescript/packages/fi-core/src/otel.ts | 25 +- .../fi-core/test/attachments-browser.cjs | 56 ++++ .../fi-core/test/attachments-contract.cjs | 66 ++++ .../fi-core/test/attachments-fixtures.cjs | 23 ++ .../fi-core/test/attachments-tls.test.cjs | 59 ++++ .../fi-core/test/attachments.test.cjs | 235 ++++++++++++++ 17 files changed, 1324 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/fi-core-attachments-tests.yml create mode 100644 typescript/packages/fi-core/src/attachments/correlation.ts create mode 100644 typescript/packages/fi-core/src/attachments/index.ts create mode 100644 typescript/packages/fi-core/src/attachments/manager.ts create mode 100644 typescript/packages/fi-core/src/attachments/node.ts create mode 100644 typescript/packages/fi-core/src/attachments/policy.ts create mode 100644 typescript/packages/fi-core/src/attachments/transport-v1.ts create mode 100644 typescript/packages/fi-core/src/attachments/types.ts create mode 100644 typescript/packages/fi-core/test/attachments-browser.cjs create mode 100644 typescript/packages/fi-core/test/attachments-contract.cjs create mode 100644 typescript/packages/fi-core/test/attachments-fixtures.cjs create mode 100644 typescript/packages/fi-core/test/attachments-tls.test.cjs create mode 100644 typescript/packages/fi-core/test/attachments.test.cjs diff --git a/.github/workflows/fi-core-attachments-tests.yml b/.github/workflows/fi-core-attachments-tests.yml new file mode 100644 index 000000000..b64954d48 --- /dev/null +++ b/.github/workflows/fi-core-attachments-tests.yml @@ -0,0 +1,60 @@ +name: TypeScript attachment core +on: + pull_request: + paths: + - 'typescript/packages/fi-core/**' + - 'typescript/packages/fi-semantic-conventions/**' + - '.github/workflows/fi-core-attachments-tests.yml' + workflow_dispatch: +permissions: + contents: read +jobs: + runtime: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + node: ['18', '24'] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - uses: pnpm/action-setup@v4 + with: + version: 10.30.1 + - name: Build core and run synthetic attachment boundaries + working-directory: typescript + run: | + pnpm --filter @traceai/fi-core... install --frozen-lockfile --ignore-scripts + pnpm --filter @traceai/fi-semantic-conventions run build + pnpm --filter @traceai/fi-core run build + pnpm --filter @traceai/fi-core run test:attachments + - name: Test installed OTel floor and latest qualified release + if: matrix.node == '24' + working-directory: typescript + run: | + mkdir -p "$RUNNER_TEMP/attachment-packs" + pnpm --dir packages/fi-core pack --pack-destination "$RUNNER_TEMP/attachment-packs" + pnpm --dir packages/fi-semantic-conventions pack --pack-destination "$RUNNER_TEMP/attachment-packs" + for version in 2.0.1 2.12.1; do + node packages/fi-core/test/install-sdk-fixture.cjs "$version" "$RUNNER_TEMP/attachment-sdk-$version" "$RUNNER_TEMP"/attachment-packs/traceai-fi-core-*.tgz "$RUNNER_TEMP"/attachment-packs/traceai-fi-semantic-conventions-*.tgz + FI_CORE_PACKAGE_JSON="$RUNNER_TEMP/attachment-sdk-$version/node_modules/@traceai/fi-core/package.json" node --test packages/fi-core/test/attachments.test.cjs packages/fi-core/test/attachments-tls.test.cjs packages/fi-core/test/provider-lifecycle.test.cjs + done + - name: Fetch pinned feature-off contract fixtures + uses: actions/checkout@v4 + with: + ref: aeed09f27d9d1f3649282dd94c885850af3ca27e + path: attachment-contract + persist-credentials: false + - name: Replay privacy and identity fixture safety + run: node typescript/packages/fi-core/test/attachments-contract.cjs attachment-contract/spec/attachments/v1 + - name: Qualify browser-safe leaf + if: matrix.node == '24' + working-directory: typescript/packages/fi-core + run: | + npm install --prefix "$RUNNER_TEMP/attachment-tools" --no-audit --no-fund esbuild@0.25.12 playwright@1.56.1 + "$RUNNER_TEMP/attachment-tools/node_modules/.bin/playwright" install --with-deps chromium + "$RUNNER_TEMP/attachment-tools/node_modules/.bin/esbuild" dist/esm/attachments/index.js --bundle --platform=browser --format=iife --global-name=Attachments --outfile="$RUNNER_TEMP/attachments-browser.js" + node test/attachments-browser.cjs "$RUNNER_TEMP/attachment-tools/node_modules/playwright" "$RUNNER_TEMP/attachments-browser.js" diff --git a/typescript/packages/fi-core/package.json b/typescript/packages/fi-core/package.json index a1dcb3102..0f152d433 100644 --- a/typescript/packages/fi-core/package.json +++ b/typescript/packages/fi-core/package.json @@ -12,19 +12,30 @@ "import": "./dist/esm/index.js", "require": "./dist/src/index.js" }, + "./attachments": { + "types": "./dist/src/attachments/index.d.ts", + "import": "./dist/esm/attachments/index.js", + "require": "./dist/src/attachments/index.js" + }, + "./attachments/node": { + "types": "./dist/src/attachments/node.d.ts", + "import": "./dist/esm/attachments/node.js", + "require": "./dist/src/attachments/node.js" + }, "./privacy": { "types": "./dist/src/privacy-runtime.d.ts", "import": "./dist/esm/privacy.js", "require": "./dist/src/privacy-runtime.js" } }, - "typesVersions": { "*": { "privacy": ["dist/src/privacy-runtime.d.ts"] } }, + "typesVersions": { "*": { "privacy": ["dist/src/privacy-runtime.d.ts"], "attachments": ["dist/src/attachments/index.d.ts"], "attachments/node": ["dist/src/attachments/node.d.ts"] } }, "scripts": { "prebuild": "rimraf dist", "build": "tsc --build tsconfig.json tsconfig.esm.json tsconfig.esnext.json && tsc-alias -p tsconfig.esm.json && pnpm run postbuild", "postbuild": "node scripts/postbuild.cjs", "test": "jest", "test:provider": "node --test test/provider-lifecycle.test.cjs", + "test:attachments": "node --test test/attachments.test.cjs test/attachments-tls.test.cjs", "test:watch": "jest --watch", "test:coverage": "jest --coverage", "lint": "eslint . --ext .ts", diff --git a/typescript/packages/fi-core/scripts/postbuild.cjs b/typescript/packages/fi-core/scripts/postbuild.cjs index 146c4e624..8b419740d 100644 --- a/typescript/packages/fi-core/scripts/postbuild.cjs +++ b/typescript/packages/fi-core/scripts/postbuild.cjs @@ -16,3 +16,13 @@ for (const dir of ['dist/esm', 'dist/esnext']) { fs.writeFileSync(`${dir}/${name}.js`, `export { ${runtimeExports.join(', ')} } from "../src/privacy-runtime.js";\n`); } } + +// A provider and an adapter may import different module formats. Keep correlation +// class identity (and its private span references) identical across those imports. +for (const name of ['index', 'manager', 'correlation', 'policy', 'transport-v1', 'node']) { + const exports = Object.keys(require(`../dist/src/attachments/${name}.js`)); + if (!exports.every(key => /^[A-Za-z_$][\w$]*$/.test(key))) throw new Error('postbuild: unexpected attachment export'); + for (const dir of ['dist/esm', 'dist/esnext']) { + fs.writeFileSync(`${dir}/attachments/${name}.js`, `export { ${exports.join(', ')} } from "../../src/attachments/${name}.js";\n`); + } +} diff --git a/typescript/packages/fi-core/src/attachments/correlation.ts b/typescript/packages/fi-core/src/attachments/correlation.ts new file mode 100644 index 000000000..1cd8567ad --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/correlation.ts @@ -0,0 +1,65 @@ +import type { Span } from "@opentelemetry/api"; +import type { AttachmentReference, AttachmentSpan } from "./types"; + +export function canonicalAttachmentTraceId(value: string): string | undefined { + if (typeof value !== "string" || (value.length !== 32 && value.length !== 36) || !/^(?:[0-9a-f]{32}|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/.test(value)) return undefined; + const hex = value.replace(/-/g, ""); + return /^0+$/.test(hex) ? undefined : hex; +} +export function attachmentTraceUuid(hex: string): string { + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} +export function validAttachmentUuid(value: unknown): value is string { + return typeof value === "string" && value.length === 36 && canonicalAttachmentTraceId(value) !== undefined; +} +/** Snapshot while recording, or bind to the ACTUAL emitted context. Never consult ambient context later. */ +export class AttachmentCorrelation { + private identity?: Readonly<{ traceId: string; spanId: string; sampled: boolean }>; + private span?: Span; + private refs: AttachmentReference[] = []; + private prepared = false; + constructor(spanOrContext?: AttachmentSpan) { if (spanOrContext) this.bind(spanOrContext); } + bind(spanOrContext: AttachmentSpan): boolean { + if (this.identity) return false; + try { + const span = "spanContext" in spanOrContext ? spanOrContext : undefined; + const ctx = span ? span.spanContext() : spanOrContext as Exclude; + const traceId = canonicalAttachmentTraceId(ctx.traceId); + if (!traceId || ctx.spanId.length !== 16 || !/^[0-9a-f]{16}$/.test(ctx.spanId) || /^0+$/.test(ctx.spanId)) return false; + this.identity = Object.freeze({ traceId, spanId: ctx.spanId, sampled: (ctx.traceFlags & 1) === 1 && (!span || span.isRecording()) }); + this.span = span; + return true; + } catch { return false; } + } + /** Stamp only the non-secret writer carrier while the actual span is still recording. */ + prepare(sessionId: string, nonce: string): void { + try { + if (this.prepared || !this.span?.isRecording()) return; + this.span.setAttribute("fi.attachment.schema_version", 1); + this.span.setAttribute("fi.attachment.capture_session_id", sessionId); + this.span.setAttribute("fi.attachment.correlation_nonce", nonce); + this.span.setAttribute("fi.attachment.refs", "[]"); + this.prepared = true; + } catch { /* A foreign span must not break application code. */ } + } + snapshot(): Readonly<{ traceId: string; spanId: string; sampled: boolean }> | undefined { return this.identity; } + /** Called synchronously at admission only. Ended spans are NEVER mutated by transfer completion. */ + project(reference: AttachmentReference, sessionId: string, nonce: string): void { + try { + if (!this.span?.isRecording()) { this.span = undefined; return; } + const next = [...this.refs, Object.freeze({ ...reference })]; + if (next.length > 64) return; + const json = JSON.stringify(next); + if (new TextEncoder().encode(json).byteLength > 16384) return; + this.span.setAttribute("fi.attachment.schema_version", 1); + this.span.setAttribute("fi.attachment.capture_session_id", sessionId); + this.span.setAttribute("fi.attachment.correlation_nonce", nonce); + this.span.setAttribute("fi.attachment.refs", json); + this.refs = next; + } catch { /* Host instrumentation must never fail because of an attachment. */ } + } + references(): readonly AttachmentReference[] { return this.refs.map(ref => Object.freeze({ ...ref })); } +} +export function attachmentMessageUri(reference: AttachmentReference): Readonly<{ type: "uri"; modality: AttachmentReference["modality"]; mime_type: string; uri: string }> { + return Object.freeze({ type: "uri", modality: reference.modality, mime_type: reference.mime_type, uri: `fi-attachment:${reference.occurrence_id}` }); +} diff --git a/typescript/packages/fi-core/src/attachments/index.ts b/typescript/packages/fi-core/src/attachments/index.ts new file mode 100644 index 000000000..df9af31f5 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/index.ts @@ -0,0 +1,7 @@ +/** Browser-safe leaf: no Node filesystem/network imports or ambient credential discovery. */ +export { AttachmentManager } from "./manager"; +export { AttachmentCorrelation, canonicalAttachmentTraceId, attachmentTraceUuid, attachmentMessageUri } from "./correlation"; +export { evaluateAttachmentPolicy, attachmentModality } from "./policy"; +export { createAttachmentHttpTransport, attachmentFetchClient, AttachmentTransportError } from "./transport-v1"; +export type { AttachmentHttpClient, AttachmentHttpRequest, AttachmentHttpResponse, AttachmentHttpTransportOptions } from "./transport-v1"; +export type * from "./types"; diff --git a/typescript/packages/fi-core/src/attachments/manager.ts b/typescript/packages/fi-core/src/attachments/manager.ts new file mode 100644 index 000000000..fce6f7cf2 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/manager.ts @@ -0,0 +1,291 @@ +import { v4 as uuid } from "uuid"; +import { AttachmentCorrelation, attachmentTraceUuid, validAttachmentUuid } from "./correlation"; +import { attachmentEnabled, attachmentModality, evaluateAttachmentPolicy } from "./policy"; +import { AttachmentTransportError, validAttachmentSession, validAttachmentUpload } from "./transport-v1"; +import type { AttachmentAdmission, AttachmentCompletion, AttachmentFlushResult, AttachmentHandle, AttachmentManagerOptions, AttachmentOptions, AttachmentReference, AttachmentSession, AttachmentSource, AttachmentSourceKind, AttachmentSpan, AttachmentUpload, AttachmentUploadRequest } from "./types"; + +// Reserve bounded transfer/read/response scratch for both workers inside the 32 MiB envelope. +const MAX_MEMORY = 32 * 1024 * 1024 - 2 * (256 * 1024 + 2 * 65536); +function bounded(value: number | undefined, fallback: number, max: number): number { + return Number.isInteger(value) && value! > 0 ? Math.min(value!, max) : fallback; +} +function unref(timer: ReturnType): void { (timer as unknown as { unref?: () => void }).unref?.(); } +function handle(admission: AttachmentAdmission): { value: AttachmentHandle; finish: (status: AttachmentCompletion["status"]) => void } { + let resolve!: (result: AttachmentCompletion) => void; + const completion = new Promise(done => { resolve = done; }); + const value = Object.freeze({ attachmentId: uuid(), occurrenceId: uuid(), admission, completion }); + const finish = (status: AttachmentCompletion["status"]) => resolve(Object.freeze({ status })); + if (admission !== "queued") finish(admission); + return { value, finish }; +} +interface Job { + handle: ReturnType; + data?: Uint8Array | Blob; + size: number; + kind: AttachmentSourceKind; + options: AttachmentOptions; + request: AttachmentUploadRequest; + controller: AbortController; + dropped?: boolean; +} +/** One per provider/runtime; explicit attachment APIs and adapters use this same instance. */ +export class AttachmentManager { + private readonly config: AttachmentManagerOptions; + private readonly sessionId = uuid(); + private readonly nonce = uuid(); + private session?: AttachmentSession; + private bootStatus: AttachmentAdmission = "bootstrap_pending"; + private readonly bootController = new AbortController(); + private readonly boot: Promise; + private readonly queue: Job[] = []; + private readonly active = new Set(); + private readonly correlations = new WeakMap(); + private readonly listeners = new Set<() => void>(); + private bytes = 0; + private dropped = 0; + private closed = false; + private closedReason: AttachmentAdmission = "incomplete"; + private shutdownPromise?: Promise; + private readonly maxBytes: number; + private readonly maxDescriptors: number; + private readonly workers: number; + private readonly requestTimeout: number; + private readonly shutdownTimeout: number; + + constructor(options: AttachmentManagerOptions = {}) { + this.config = { ...options, traceConfig: { ...options.traceConfig } }; + this.maxBytes = bounded(options.maxOwnedBytes, MAX_MEMORY, MAX_MEMORY); + this.maxDescriptors = bounded(options.maxDescriptors, 128, 128); + this.workers = bounded(options.workers, 2, 2); + this.requestTimeout = bounded(options.requestTimeoutMs, 2000, 30000); + this.shutdownTimeout = bounded(options.shutdownTimeoutMs, 5000, 30000); + this.boot = this.bootstrap(); + } + private env(): Readonly> { + try { return this.config.environment?.() ?? {}; } + catch { return { FI_ATTACHMENTS_ENABLED: "false" }; } + } + private async bootstrap(): Promise { + try { + if (!attachmentEnabled(this.config, this.env())) { this.bootStatus = "disabled"; return false; } + if (!this.config.transport) { this.bootStatus = "endpoint_unconfigured"; return false; } + if (typeof this.config.projectName !== "string" || !this.config.projectName || this.config.projectName.length > 255 || + typeof this.config.projectType !== "string" || !this.config.projectType || this.config.projectType.length > 64 || + (this.config.workspaceId !== undefined && !validAttachmentUuid(this.config.workspaceId))) { this.bootStatus = "scope_unresolved"; return false; } + const request = { schema_versions: [1] as [1], project_name: this.config.projectName, project_type: this.config.projectType, + capture_session_id: this.sessionId, correlation_nonce: this.nonce, + ...(this.config.workspaceId ? { workspace_id: this.config.workspaceId } : {}) }; + const session = await this.attempt(signal => this.config.transport!.bootstrap(request, signal), this.bootController.signal); + if (!validAttachmentSession(session, this.sessionId)) throw new AttachmentTransportError("unsupported_version"); + if (this.config.workspaceId && session.scope.workspace_id !== this.config.workspaceId) throw new AttachmentTransportError("scope_unresolved"); + if (this.closed) return false; + // Detached bounded wire object: a caller/transport cannot later loosen policy through mutation. + this.session = JSON.parse(JSON.stringify(session)); + this.bootStatus = "queued"; + return true; + } catch (error) { this.bootStatus = error instanceof AttachmentTransportError && error.code !== "failed" ? error.code : "source_unavailable"; return false; } + } + /** Explicit startup wait; setup and inference do not await it. */ + async ready(timeoutMs = 5000): Promise { + return this.deadline(this.boot, bounded(timeoutMs, 5000, 30000), false); + } + correlation(spanOrContext?: AttachmentSpan): AttachmentCorrelation { + if (!spanOrContext) return new AttachmentCorrelation(); + let result = this.correlations.get(spanOrContext); + if (!result) { result = new AttachmentCorrelation(spanOrContext); this.correlations.set(spanOrContext, result); } + if (attachmentEnabled(this.config, this.env())) result.prepare(this.sessionId, this.nonce); + return result; + } + private denial(kind: AttachmentSourceKind, options: AttachmentOptions): AttachmentAdmission | undefined { + if (this.closed) return this.closedReason; + const policy = evaluateAttachmentPolicy(this.config, this.config.traceConfig ?? {}, this.session?.policy, kind, options, this.env()); + if (policy === "bootstrap_pending" && this.bootStatus !== "queued") return this.bootStatus; + if (policy) return policy; + if (this.session && Date.parse(this.session.expires_at) <= Date.now()) return "expired"; + return undefined; + } + attach(spanOrContext: AttachmentSpan | AttachmentCorrelation, source: AttachmentSource, options: AttachmentOptions): AttachmentHandle { + try { + if (this.shutdownPromise) return handle("incomplete").value; + // No access to source.data/path/url until the complete privacy gate passes. + const kind = source.kind; + const denied = this.denial(kind, options); + if (denied) return handle(denied).value; + const correlation = spanOrContext instanceof AttachmentCorrelation ? spanOrContext : this.correlation(spanOrContext); + const identity = correlation.snapshot(); + if (!identity) return handle("correlation_unavailable").value; + if (!identity.sampled) return handle("unsampled").value; + if (!this.validOptions(options)) return handle("policy_denied").value; + if (kind !== "Bytes" && kind !== "Blob") return handle("source_unavailable").value; + if (this.queue.length + this.active.size >= this.maxDescriptors) return handle("queue_full").value; + const data = source.data; + const bytes = kind === "Bytes"; + if (bytes ? !(data instanceof Uint8Array) : typeof Blob === "undefined" || !(data instanceof Blob)) return handle("source_unavailable").value; + const size = bytes ? (data as Uint8Array).byteLength : (data as Blob).size; + const modality = attachmentModality(options.mimeType)!; + const cap = modality === "image" ? 20 * 1024 * 1024 : modality === "audio" || modality === "video" ? 8 * 1024 * 1024 : 50 * 1024 * 1024; + if (!size || size > Math.min(cap, this.session!.limits.max_upload_bytes, this.maxBytes)) return handle("too_large").value; + if (this.queue.length + this.active.size >= this.maxDescriptors || this.bytes + size > this.maxBytes) return handle("queue_full").value; + // Mutable inputs are detached synchronously, after all admission and budget checks. + const owned = bytes ? new Uint8Array(data as Uint8Array) : data as Blob; + const result = handle("queued"); + const snapshot: AttachmentOptions = { mimeType: options.mimeType, direction: options.direction, purpose: options.purpose, + messageIndex: options.messageIndex, partIndex: options.partIndex, role: options.role, channel: options.channel, + conversationId: options.conversationId, turnId: options.turnId, toolCallId: options.toolCallId, + capture: { captureContent: options.capture?.captureContent, recordInputs: options.capture?.recordInputs, recordOutputs: options.capture?.recordOutputs, + muted: options.capture?.muted, temporaryChat: options.capture?.temporaryChat, requested: options.capture?.requested, + modelInput: options.capture?.modelInput, containsInput: options.capture?.containsInput, containsOutput: options.capture?.containsOutput, + errorDetail: options.capture?.errorDetail, newRecording: options.capture?.newRecording } }; + const request: AttachmentUploadRequest = { schema_version: 1, attachment_id: result.value.attachmentId, occurrence_id: result.value.occurrenceId, + association: { capture_session_id: this.sessionId, correlation_nonce: this.nonce, parent: { trace_id: attachmentTraceUuid(identity.traceId), span_id: identity.spanId } }, + policy_version: this.session!.policy.version, deletion_generation: 0, mime_type: snapshot.mimeType, + modality, direction: snapshot.direction, source_kind: kind, declared_bytes: size, + ...(snapshot.purpose ? { purpose: snapshot.purpose } : {}), + ...(snapshot.messageIndex === undefined ? {} : { message_index: snapshot.messageIndex }), + ...(snapshot.partIndex === undefined ? {} : { part_index: snapshot.partIndex }), + ...(snapshot.conversationId ? { conversation_id: snapshot.conversationId } : {}), + ...(snapshot.turnId ? { turn_id: snapshot.turnId } : {}), + ...(snapshot.toolCallId ? { tool_call_id: snapshot.toolCallId } : {}) }; + const job: Job = { handle: result, data: owned, size, kind, options: snapshot, request, controller: new AbortController() }; + this.bytes += size; + this.queue.push(job); + const reference: AttachmentReference = { attachment_id: result.value.attachmentId, occurrence_id: result.value.occurrenceId, + direction: snapshot.direction, modality, mime_type: snapshot.mimeType, ...(snapshot.purpose ? { purpose: snapshot.purpose } : {}), + ...(snapshot.messageIndex === undefined ? {} : { message_index: snapshot.messageIndex }), + ...(snapshot.partIndex === undefined ? {} : { part_index: snapshot.partIndex }), + ...(snapshot.role ? { role: snapshot.role } : {}), ...(snapshot.channel === undefined ? {} : { channel: snapshot.channel }), + ...(snapshot.conversationId ? { conversation_id: snapshot.conversationId } : {}), + ...(snapshot.turnId ? { turn_id: snapshot.turnId } : {}), + ...(snapshot.toolCallId ? { tool_call_id: snapshot.toolCallId } : {}) }; + correlation.project(reference, this.sessionId, this.nonce); + // Avoid even synchronous user transport execution in the inference callback. + void Promise.resolve().then(() => this.pump()); + return result.value; + } catch { return handle("source_unavailable").value; } + } + private validOptions(options: AttachmentOptions): boolean { + const purposes = ["conversation_recording", "turn_recording", "model_input", "model_output", "tool_result", "computer_screenshot", "diagnostic", "document"]; + if (options.purpose !== undefined && !purposes.includes(options.purpose)) return false; + if (options.role !== undefined && !["user", "assistant", "tool", "system", "unknown"].includes(options.role)) return false; + for (const [value, max] of [[options.messageIndex, 65535], [options.partIndex, 65535], [options.channel, 31]] as const) { + if (value !== undefined && (!Number.isInteger(value) || value < 0 || value > max)) return false; + } + return [options.conversationId, options.turnId, options.toolCallId].every(value => + value === undefined || typeof value === "string" && value.trim() === value && /^[a-zA-Z0-9_-]{1,128}$/.test(value)); + } + private pump(): void { + while (this.active.size < this.workers && this.queue.length) { + const job = this.queue.shift()!; + this.active.add(job); + void this.transfer(job).finally(() => { + job.data = undefined; this.bytes -= job.size; this.active.delete(job); + this.notify(); this.pump(); + }); + } + } + private check(job: Job): void { + const denied = job.controller.signal.aborted ? "incomplete" : this.denial(job.kind, job.options); + if (denied) throw new AttachmentTransportError(denied); + if (job.request.policy_version !== this.session!.policy.version) throw new AttachmentTransportError("policy_denied"); + } + private async transfer(job: Job): Promise { + try { + const transport = this.config.transport!; + const guard = () => this.check(job); + guard(); + const upload = await this.attempt(signal => transport.initialize(this.session!, job.request, signal), job.controller.signal, guard); + if (!validAttachmentUpload(upload, job.request)) throw new AttachmentTransportError("failed"); + const checkUpload = () => { guard(); if (Date.parse(upload.expires_at) <= Date.now()) throw new AttachmentTransportError("expired"); }; + // v1 cannot safely replay a partially consumed PUT. Only idempotent control operations retry. + await this.attempt(signal => transport.upload(this.session!, upload, job.data!, signal), job.controller.signal, checkUpload, 1); + await this.attempt(signal => transport.complete(this.session!, upload, job.request, signal), job.controller.signal, checkUpload); + checkUpload(); job.handle.finish("validating"); + } catch (error) { + this.drop(job, error instanceof AttachmentTransportError ? error.code : "failed"); + } + } + private async attempt(operation: (signal: AbortSignal) => Promise, parent: AbortSignal, guard: () => void = () => undefined, attempts = 3): Promise { + for (let attempt = 0; ; attempt++) { + if (parent.aborted) throw new AttachmentTransportError("incomplete"); + guard(); + const controller = new AbortController(); + const abort = () => controller.abort(); + parent.addEventListener("abort", abort, { once: true }); + const timer = setTimeout(abort, this.requestTimeout); unref(timer); + try { + const result = await operation(controller.signal); + if (controller.signal.aborted) throw new AttachmentTransportError("incomplete"); + return result; + } catch (error) { + if (!(error instanceof AttachmentTransportError) || !error.retryable || attempt + 1 >= attempts || parent.aborted || controller.signal.aborted) throw error; + await this.pause(Math.max(error.retryAfterMs, 25 * 2 ** attempt + Math.random() * 25), parent); + } finally { clearTimeout(timer); parent.removeEventListener("abort", abort); } + } + } + private pause(ms: number, signal: AbortSignal): Promise { + return new Promise(resolve => { + const done = () => { clearTimeout(timer); signal.removeEventListener("abort", done); resolve(); }; + const timer = setTimeout(done, ms); unref(timer); + signal.addEventListener("abort", done, { once: true }); + if (signal.aborted) done(); + }); + } + private drop(job: Job, reason: AttachmentCompletion["status"]): void { + if (!job.dropped) { job.dropped = true; this.dropped++; job.handle.finish(reason); } + } + /** Tightening discards all queued data and cancels transfers; re-enable never resurrects it. */ + updatePolicy(update: Pick): void { + if (update.enabled !== undefined) this.config.enabled = update.enabled; + if (update.traceConfig) this.config.traceConfig = { ...this.config.traceConfig, ...update.traceConfig }; + this.cancelDenied(); + } + /** A revoked capability/session/generation never resumes or retains capture data. */ + invalidate(reason: "deleted" | "expired" | "policy_denied" = "deleted"): void { + this.closedReason = reason; this.closed = true; this.bootController.abort(); this.cancelDenied(); + } + private cancelDenied(): void { + for (let index = this.queue.length - 1; index >= 0; index--) { + const job = this.queue[index]; const reason = this.denial(job.kind, job.options); + if (reason) { this.queue.splice(index, 1); job.data = undefined; this.bytes -= job.size; this.drop(job, reason); } + } + for (const job of this.active) { + const reason = this.denial(job.kind, job.options); + if (reason) { job.controller.abort(); this.drop(job, reason); } + } + this.notify(); + } + private notify(): void { for (const notify of this.listeners) notify(); } + private deadline(promise: Promise, timeout: number, fallback: T): Promise { + return new Promise(resolve => { + const timer = setTimeout(() => resolve(fallback), timeout); unref(timer); + void promise.then(value => { clearTimeout(timer); resolve(value); }, () => { clearTimeout(timer); resolve(fallback); }); + }); + } + async flush(timeoutMs = this.shutdownTimeout): Promise { + this.cancelDenied(); + if (this.queue.length || this.active.size) { + await new Promise(resolve => { + const done = () => { clearTimeout(timer); this.listeners.delete(changed); resolve(); }; + const changed = () => { if (!this.queue.length && !this.active.size) done(); }; + const timer = setTimeout(done, bounded(timeoutMs, this.shutdownTimeout, 30000)); unref(timer); + this.listeners.add(changed); changed(); + }); + } + return Object.freeze({ unfinished: this.queue.length + this.active.size, dropped: this.dropped }); + } + shutdown(timeoutMs = this.shutdownTimeout): Promise { + if (!this.shutdownPromise) { + // Stop new admissions, but give accepted jobs a bounded finalization window. + this.shutdownPromise = (async () => { + const result = await this.flush(timeoutMs); + this.closed = true; this.bootController.abort(); this.cancelDenied(); + return Object.freeze({ unfinished: result.unfinished, dropped: this.dropped }); + })(); + } + return this.shutdownPromise; + } + /** Aggregate diagnostics only; no per-source metadata, names, hashes or exception text. */ + status(): Readonly<{ state: AttachmentAdmission; queued: number; active: number; ownedBytes: number; dropped: number }> { + return Object.freeze({ state: this.closed ? this.closedReason : this.bootStatus, queued: this.queue.length, active: this.active.size, ownedBytes: this.bytes, dropped: this.dropped }); + } +} diff --git a/typescript/packages/fi-core/src/attachments/node.ts b/typescript/packages/fi-core/src/attachments/node.ts new file mode 100644 index 000000000..6d71d0402 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/node.ts @@ -0,0 +1,62 @@ +import { request } from "node:https"; +import { AttachmentManager } from "./manager"; +import { AttachmentTransportError, createAttachmentHttpTransport } from "./transport-v1"; +import type { AttachmentHttpClient } from "./transport-v1"; +import type { AttachmentManagerOptions } from "./types"; + +/** Node-only bounded transport. No keep-alive pool, redirect following, cookies or environment proxy. */ +export const attachmentNodeHttpClient: AttachmentHttpClient = input => new Promise((resolve, reject) => { + let settled = false; + const fail = () => { if (!settled) { settled = true; reject(new AttachmentTransportError("failed", !input.signal.aborted)); } }; + try { + const size = typeof input.body === "string" ? Buffer.byteLength(input.body) : input.body instanceof Uint8Array ? input.body.byteLength : input.body.size; + const req = request(input.url, { method: input.method, headers: { ...input.headers, "content-length": String(size) }, + signal: input.signal, agent: false }, response => { + if (!response.statusCode || response.statusCode < 200 || response.statusCode >= 300) { + settled = true; + resolve({ status: response.statusCode ?? 0, body: new Uint8Array(), retryAfter: response.headers["retry-after"] }); + response.destroy(); req.destroy(); return; + } + const chunks: Buffer[] = []; let bytes = 0; + response.on("data", (chunk: Buffer) => { + bytes += chunk.byteLength; + if (bytes > 65536) { settled = true; reject(new AttachmentTransportError("failed")); response.destroy(); req.destroy(); return; } + chunks.push(chunk); + }); + response.on("error", fail); + response.on("aborted", fail); + response.on("end", () => { + if (!settled) { settled = true; resolve({ status: response.statusCode ?? 0, body: Buffer.concat(chunks, bytes), retryAfter: response.headers["retry-after"] }); } + }); + }); + req.on("socket", socket => socket.unref()); + req.on("error", fail); + // A Blob is immutable and read independently, one bounded slice at a time. + void (async () => { + if (typeof input.body === "string") { req.end(input.body); return; } + for (let offset = 0; offset < size; offset += 256 * 1024) { + if (input.signal.aborted || req.destroyed) return; + const chunk = input.body instanceof Uint8Array ? input.body.subarray(offset, offset + 256 * 1024) : + new Uint8Array(await input.body.slice(offset, offset + 256 * 1024).arrayBuffer()); + if (input.signal.aborted || req.destroyed) return; + if (!req.write(chunk)) await new Promise(done => { + const finish = () => { req.removeListener("drain", finish); req.removeListener("close", finish); done(); }; + req.once("drain", finish); req.once("close", finish); + }); + } + req.end(); + })().catch(() => { req.destroy(); fail(); }); + } catch { fail(); } +}); +export interface NodeAttachmentManagerOptions extends AttachmentManagerOptions { headers?: Readonly> } +/** Provider setup passes only its resolved FI credentials, never a provider/model client's key. */ +export function createNodeAttachmentManager(options: NodeAttachmentManagerOptions = {}): AttachmentManager { + const endpoint = options.endpoint ?? process.env.FI_ATTACHMENTS_ENDPOINT; + let transport = options.transport; + if (!transport && endpoint) { + try { transport = createAttachmentHttpTransport({ endpoint, headers: options.headers ?? {}, client: attachmentNodeHttpClient }); } + catch { /* Unconfigured/untrusted origin remains fail closed without affecting tracer setup. */ } + } + return new AttachmentManager({ ...options, endpoint, environment: options.environment ?? (() => process.env), transport }); +} +export * from "./index"; diff --git a/typescript/packages/fi-core/src/attachments/policy.ts b/typescript/packages/fi-core/src/attachments/policy.ts new file mode 100644 index 000000000..2bd2a0fc0 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/policy.ts @@ -0,0 +1,59 @@ +import type { TraceConfigOptions } from "../trace/trace-config/types"; +import type { AttachmentAdmission, AttachmentConfig, AttachmentModality, AttachmentOptions, AttachmentServerPolicy, AttachmentSourceKind } from "./types"; + +const MIME_MODALITIES: Readonly> = Object.freeze({ + "audio/wav": "audio", "audio/ogg": "audio", "audio/mpeg": "audio", "audio/mp4": "audio", "audio/webm": "audio", "audio/flac": "audio", "audio/L16": "audio", + "image/png": "image", "image/jpeg": "image", "image/webp": "image", "video/mp4": "video", "video/webm": "video", + "application/pdf": "document", "application/octet-stream": "file", +}); +export function attachmentModality(mime: string): AttachmentModality | undefined { + return Object.prototype.hasOwnProperty.call(MIME_MODALITIES, mime) ? MIME_MODALITIES[mime] : undefined; +} +const enabledValue = (value: string | undefined) => /^(true|1|yes)$/i.test(value?.trim() ?? ""); +const restrictiveValue = (value: string | undefined) => value !== undefined && !/^(false|0|no)$/i.test(value.trim()); +export function attachmentEnabled(config: AttachmentConfig, env: Readonly>): boolean { + return config.enabled !== false && (config.enabled === true || enabledValue(env.FI_ATTACHMENTS_ENABLED)) && + (env.FI_ATTACHMENTS_ENABLED === undefined || enabledValue(env.FI_ATTACHMENTS_ENABLED)); +} +/** Pure admission. This function never accesses, copies, decodes or hashes a source. */ +export function evaluateAttachmentPolicy(config: AttachmentConfig, traceConfig: TraceConfigOptions, + server: AttachmentServerPolicy | undefined, sourceKind: AttachmentSourceKind, options: AttachmentOptions, + env: Readonly> = {}): AttachmentAdmission | undefined { + if (!attachmentEnabled(config, env)) return "disabled"; + if (!server) return "bootstrap_pending"; + if (!server.enabled) return "policy_denied"; + const capture = options.capture ?? {}; + for (const key of ["captureContent", "recordInputs", "recordOutputs", "muted", "temporaryChat", "requested", "modelInput", "containsInput", "containsOutput", "errorDetail", "newRecording"] as const) { + if (capture[key] !== undefined && typeof capture[key] !== "boolean") return "policy_denied"; + } + if (capture.captureContent === false || capture.requested === false || capture.muted || capture.temporaryChat) return "policy_denied"; + if (!["input", "output", "mixed"].includes(options.direction)) return "policy_denied"; + const input = options.direction !== "output" || capture.containsInput || capture.modelInput || options.purpose === "model_input" || options.purpose === "computer_screenshot"; + const output = options.direction !== "input" || capture.containsOutput; + if ((input && capture.recordInputs === false) || (output && capture.recordOutputs === false)) return "policy_denied"; + const hidden = (key: keyof TraceConfigOptions, name: string) => (traceConfig[key] !== undefined && traceConfig[key] !== false) || restrictiveValue(env[name]); + const inputHidden = hidden("hideInputs", "FI_HIDE_INPUTS") || hidden("hideInputMessages", "FI_HIDE_INPUT_MESSAGES") || hidden("hideInputText", "FI_HIDE_INPUT_TEXT"); + const outputHidden = hidden("hideOutputs", "FI_HIDE_OUTPUTS") || hidden("hideOutputMessages", "FI_HIDE_OUTPUT_MESSAGES") || hidden("hideOutputText", "FI_HIDE_OUTPUT_TEXT"); + if ((input && inputHidden) || (output && outputHidden)) return "policy_denied"; + const modality = attachmentModality(options.mimeType); + if (!modality) return "source_unavailable"; + if (input && modality === "image" && hidden("hideInputImages", "FI_HIDE_INPUT_IMAGES")) return "policy_denied"; + const errorHidden = hidden("hideErrorDetails", "FI_HIDE_ERROR_DETAILS") || + (traceConfig.hideErrorDetails === undefined && env.FI_HIDE_ERROR_DETAILS === undefined && (inputHidden || outputHidden)); + if ((capture.errorDetail || options.purpose === "diagnostic") && errorHidden) return "policy_denied"; + // No local qualified media sanitizer ships with the core. Never trust a caller's "sanitized" flag. + if (server.require_media_redaction || server.require_metadata_removal) return "redaction_unavailable"; + const unredacted = config.allowUnredactedMediaWithTextPii === true || + (config.allowUnredactedMediaWithTextPii === undefined && enabledValue(env.FI_ATTACHMENTS_ALLOW_UNREDACTED_MEDIA_WITH_TEXT_PII)); + if (hidden("piiRedaction", "FI_PII_REDACTION") && !(unredacted && server.allow_unredacted_media_with_text_pii)) return "redaction_unavailable"; + if (!server.allowed_modalities.includes(modality) || !server.allowed_sources.includes(sourceKind)) return "policy_denied"; + if (capture.newRecording) { + const key = modality === "audio" ? "recordAudio" : modality === "video" ? "recordVideo" : undefined; + if (!key) return "recording_disabled"; + const name = key === "recordAudio" ? "FI_ATTACHMENTS_RECORD_AUDIO" : "FI_ATTACHMENTS_RECORD_VIDEO"; + const permitted = key === "recordAudio" ? server.record_audio : server.record_video; + if (!permitted || config[key] === false || !(config[key] === true || enabledValue(env[name])) || + (env[name] !== undefined && !enabledValue(env[name]))) return "recording_disabled"; + } + return undefined; +} diff --git a/typescript/packages/fi-core/src/attachments/transport-v1.ts b/typescript/packages/fi-core/src/attachments/transport-v1.ts new file mode 100644 index 000000000..e11333c7f --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/transport-v1.ts @@ -0,0 +1,139 @@ +import { context } from "@opentelemetry/api"; +import { suppressTracing } from "@opentelemetry/core"; +import { validAttachmentUuid } from "./correlation"; +import type { AttachmentAdmission, AttachmentBootstrap, AttachmentSession, AttachmentTransport, AttachmentUpload, AttachmentUploadRequest } from "./types"; + +/** Safe errors have only fixed codes, never response bodies, credentials or URLs. */ +export class AttachmentTransportError extends Error { + constructor(readonly code: AttachmentAdmission | "failed", readonly retryable = false, readonly retryAfterMs = 0) { + super(code); this.name = "AttachmentTransportError"; + } +} +export interface AttachmentHttpRequest { + url: string; method: "POST" | "PUT"; headers: Record; + body: string | Uint8Array | Blob; signal: AbortSignal; +} +export interface AttachmentHttpResponse { status: number; body: Uint8Array; retryAfter?: string } +export type AttachmentHttpClient = (request: AttachmentHttpRequest) => Promise; + +/** No cookies, redirects, arbitrary destination paths or unbounded JSON reads. */ +export const attachmentFetchClient: AttachmentHttpClient = async request => { + try { + const response = await fetch(request.url, { + method: request.method, headers: request.headers, body: request.body, + signal: request.signal, redirect: "error", credentials: "omit", + }); + if (response.status < 200 || response.status >= 300) { + await response.body?.cancel().catch(() => undefined); + return { status: response.status, body: new Uint8Array(), retryAfter: response.headers.get("retry-after") ?? undefined }; + } + const reader = response.body?.getReader(); + const chunks: Uint8Array[] = []; let size = 0; + if (reader) { + try { + for (;;) { + const { value, done } = await reader.read(); if (done) break; + size += value.byteLength; + if (size > 65536) throw new AttachmentTransportError("failed"); + chunks.push(value); + } + } finally { await reader.cancel().catch(() => undefined); } + } + const body = new Uint8Array(size); let offset = 0; + for (const chunk of chunks) { body.set(chunk, offset); offset += chunk.length; } + return { status: response.status, body, retryAfter: response.headers.get("retry-after") ?? undefined }; + } catch (error) { + if (error instanceof AttachmentTransportError) throw error; + throw new AttachmentTransportError("failed", !request.signal.aborted); + } +}; +function object(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } +export function validAttachmentSession(value: unknown, sessionId: string): value is AttachmentSession { + if (!object(value) || value.schema_version !== 1 || value.capture_session_id !== sessionId || !object(value.scope) || + ![value.scope.organization_id, value.scope.workspace_id, value.scope.project_id].every(validAttachmentUuid) || + !Number.isFinite(Date.parse(value.expires_at)) || Date.parse(value.expires_at) <= Date.now() || + !Array.isArray(value.transfer_modes) || value.transfer_modes.length !== 1 || value.transfer_modes[0] !== "authenticated_stream") return false; + const p = value.policy; + if (!object(p) || typeof p.version !== "string" || p.version.trim() !== p.version || !/^[a-zA-Z0-9_-]{1,128}$/.test(p.version) || + !["enabled", "require_media_redaction", "require_metadata_removal", "allow_unredacted_media_with_text_pii", "record_audio", "record_video"].every(key => typeof p[key] === "boolean") || + !Array.isArray(p.allowed_sources) || p.allowed_sources.length > 6 || !p.allowed_sources.every((s: unknown) => ["Bytes", "Blob", "RemoteUrl", "LocalFile", "OwnedRecordingFile", "ProviderFile"].includes(s as string)) || + !Array.isArray(p.allowed_modalities) || p.allowed_modalities.length > 5 || !p.allowed_modalities.every((m: unknown) => ["audio", "image", "video", "document", "file"].includes(m as string))) return false; + const limits: Record = { max_upload_bytes: 52428800, segment_bytes: 8388608, segment_duration_ms: 30000, + recording_bytes: 1073741824, recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, + reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, + upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, + grant_ttl_seconds: 300, revocation_check_seconds: 5 }; + return object(value.limits) && Object.entries(limits).every(([key, max]) => Number.isInteger(value.limits[key]) && value.limits[key] > 0 && value.limits[key] <= max); +} +export function validAttachmentUpload(value: unknown, request: AttachmentUploadRequest): value is AttachmentUpload { + return object(value) && validAttachmentUuid(value.upload_id) && value.attachment_id === request.attachment_id && + value.occurrence_id === request.occurrence_id && value.state === "pending_upload" && value.reserved_bytes === request.declared_bytes && + Number.isFinite(Date.parse(value.expires_at)) && Date.parse(value.expires_at) > Date.now() && value.transfer_mode === "authenticated_stream" && typeof value.write_capability === "string" && value.write_capability.trim() === value.write_capability && /^[a-zA-Z0-9_.-]{16,4096}$/.test(value.write_capability); +} +export interface AttachmentHttpTransportOptions { + endpoint: string; + /** Explicitly supplied credential/capability. Never read environment in browser code. */ + headers: Readonly>; + client?: AttachmentHttpClient; +} +export function createAttachmentHttpTransport(options: AttachmentHttpTransportOptions): AttachmentTransport { + const url = new URL(options.endpoint); + if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash || url.pathname !== "/") { + throw new AttachmentTransportError("endpoint_unconfigured"); + } + const origin = url.origin; + const headers: Record = {}; + const allowed = new Set(["authorization", "x-api-key", "x-secret-key", "x-workspace-id"]); + for (const [name, value] of Object.entries(options.headers)) { + if (allowed.has(name.toLowerCase()) && typeof value === "string" && value.length <= 8192 && !/[\r\n]/.test(value)) headers[name.toLowerCase()] = value; + } + const client = options.client ?? attachmentFetchClient; + async function send(path: string, body: string | Uint8Array | Blob, signal: AbortSignal, expected: number, key?: string, capability?: string): Promise { + if (signal.aborted) throw new AttachmentTransportError("incomplete"); + const binary = typeof body !== "string"; + let response: AttachmentHttpResponse; + try { + response = await context.with(suppressTracing(context.active()), () => client({ + url: origin + path, method: binary ? "PUT" : "POST", signal, body, + headers: { ...(capability ? { authorization: `Bearer ${capability}` } : headers), "content-type": binary ? "application/octet-stream" : "application/json", "cache-control": "no-store", ...(key ? { "idempotency-key": key } : {}) }, + })); + } catch (error) { throw error instanceof AttachmentTransportError ? error : new AttachmentTransportError("failed", !signal.aborted); } + if (response.status !== expected) { + const code = response.status === 401 || response.status === 403 ? "credential_scope_unsupported" : response.status === 413 ? "too_large" : response.status === 409 ? "policy_denied" : response.status === 410 ? "expired" : "failed"; + const retryable = response.status === 429 || response.status >= 500 && response.status <= 599; + const after = Number(response.retryAfter); + throw new AttachmentTransportError(code, retryable, Number.isFinite(after) && after >= 0 ? Math.min(after * 1000, 2000) : 0); + } + if (response.body.byteLength > 65536) throw new AttachmentTransportError("failed"); + try { return JSON.parse(new TextDecoder().decode(response.body)); } + catch { throw new AttachmentTransportError("failed"); } + } + const base = (session: AttachmentSession) => { + if (!validAttachmentUuid(session.scope.project_id)) throw new AttachmentTransportError("scope_unresolved"); + return `/tracer/v1/projects/${session.scope.project_id}/attachments/uploads`; + }; + const uploadPath = (session: AttachmentSession, upload: AttachmentUpload) => { + if (!validAttachmentUuid(upload.upload_id)) throw new AttachmentTransportError("failed"); + return `${base(session)}/${upload.upload_id}`; + }; + return { + async bootstrap(request, signal) { + const result = await send("/tracer/v1/attachment-sessions", JSON.stringify(request), signal, 200, request.capture_session_id); + if (!validAttachmentSession(result, request.capture_session_id)) throw new AttachmentTransportError("unsupported_version"); + return result; + }, + async initialize(session, request, signal) { + const result = await send(base(session), JSON.stringify(request), signal, 201, request.occurrence_id); + if (!validAttachmentUpload(result, request)) throw new AttachmentTransportError("failed"); + return result; + }, + async upload(session, upload, data, signal) { + const result = await send(`${uploadPath(session, upload)}/content`, data, signal, 200, undefined, upload.write_capability); + if (!object(result) || result.upload_id !== upload.upload_id || result.state !== "pending_upload") throw new AttachmentTransportError("failed"); + }, + async complete(session, upload, request, signal) { + const result = await send(`${uploadPath(session, upload)}/complete`, JSON.stringify({ policy_version: request.policy_version, deletion_generation: request.deletion_generation }), signal, 202, request.occurrence_id, upload.write_capability); + if (!object(result) || result.upload_id !== upload.upload_id || result.state !== "validating") throw new AttachmentTransportError("failed"); + }, + }; +} diff --git a/typescript/packages/fi-core/src/attachments/types.ts b/typescript/packages/fi-core/src/attachments/types.ts new file mode 100644 index 000000000..36f719c70 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/types.ts @@ -0,0 +1,154 @@ +import type { Span, SpanContext } from "@opentelemetry/api"; +import type { TraceConfigOptions } from "../trace/trace-config/types"; + +export type AttachmentDirection = "input" | "output" | "mixed"; +export type AttachmentModality = "audio" | "image" | "video" | "document" | "file"; +export type AttachmentPurpose = "conversation_recording" | "turn_recording" | "model_input" | "model_output" | "tool_result" | "computer_screenshot" | "diagnostic" | "document"; +export type AttachmentSourceKind = "Bytes" | "Blob" | "RemoteUrl" | "LocalFile" | "OwnedRecordingFile" | "ProviderFile"; +/** No source is inferred from a string. Unsupported retrieval sources fail closed. */ +export type AttachmentSource = + | { kind: "Bytes"; data: Uint8Array } + | { kind: "Blob"; data: Blob } + | { kind: "RemoteUrl"; url: string } + | { kind: "LocalFile"; path: string } + | { kind: "OwnedRecordingFile"; lease: unknown } + | { kind: "ProviderFile"; provider: string; fileId: string }; +export type AttachmentAdmission = "queued" | "disabled" | "bootstrap_pending" | "policy_denied" | "credential_scope_unsupported" | "source_unavailable" | "unsampled" | "too_large" | "queue_full" | "unsupported_version" | "endpoint_unconfigured" | "scope_unresolved" | "correlation_unavailable" | "redaction_unavailable" | "recording_disabled" | "expired" | "deleted" | "incomplete"; +export type AttachmentCompletion = Readonly<{ status: AttachmentAdmission | "validating" | "failed" | "deleted" }>; +export interface AttachmentHandle { + readonly attachmentId: string; + readonly occurrenceId: string; + readonly admission: AttachmentAdmission; + /** Always resolves; validating means transferred, NOT ready or readable. */ + readonly completion: Promise; +} +/** Semantic ancestry can only add restrictions, never remove a directional restriction. */ +export interface AttachmentCapturePolicy { + captureContent?: boolean; + recordInputs?: boolean; + recordOutputs?: boolean; + muted?: boolean; + temporaryChat?: boolean; + requested?: boolean; + modelInput?: boolean; + containsInput?: boolean; + containsOutput?: boolean; + errorDetail?: boolean; + newRecording?: boolean; +} +export interface AttachmentOptions { + mimeType: string; + direction: AttachmentDirection; + purpose?: AttachmentPurpose; + conversationId?: string; + turnId?: string; + toolCallId?: string; + messageIndex?: number; + partIndex?: number; + role?: "user" | "assistant" | "tool" | "system" | "unknown"; + channel?: number; + capture?: AttachmentCapturePolicy; +} +export interface AttachmentConfig { + enabled?: boolean; + recordAudio?: boolean; + recordVideo?: boolean; + allowUnredactedMediaWithTextPii?: boolean; + /** Trusted control-plane origin, never inferred from the OTLP or source URL. */ + endpoint?: string; + maxDescriptors?: number; + maxOwnedBytes?: number; + workers?: number; + requestTimeoutMs?: number; + shutdownTimeoutMs?: number; +} +export interface AttachmentReference { + attachment_id: string; + occurrence_id: string; + direction: AttachmentDirection; + modality: AttachmentModality; + mime_type: string; + purpose?: AttachmentPurpose; + message_index?: number; + part_index?: number; + role?: AttachmentOptions["role"]; + channel?: number; + conversation_id?: string; + turn_id?: string; + tool_call_id?: string; +} +export interface AttachmentServerPolicy { + version: string; + enabled: boolean; + allowed_sources: AttachmentSourceKind[]; + allowed_modalities: AttachmentModality[]; + require_media_redaction: boolean; + require_metadata_removal: boolean; + allow_unredacted_media_with_text_pii: boolean; + record_audio: boolean; + record_video: boolean; +} +export interface AttachmentSession { + schema_version: 1; + scope: { organization_id: string; workspace_id: string; project_id: string }; + capture_session_id: string; + policy: AttachmentServerPolicy; + limits: { max_upload_bytes: number; [name: string]: number }; + expires_at: string; + transfer_modes: ["authenticated_stream"]; +} +export interface AttachmentBootstrap { + schema_versions: [1]; + project_name: string; + project_type: string; + workspace_id?: string; + capture_session_id: string; + correlation_nonce: string; +} +export interface AttachmentUploadRequest { + schema_version: 1; + attachment_id: string; + occurrence_id: string; + association: { capture_session_id: string; correlation_nonce: string; parent: { trace_id: string; span_id: string } }; + policy_version: string; + deletion_generation: number; + mime_type: string; + modality: AttachmentModality; + direction: AttachmentDirection; + source_kind: AttachmentSourceKind; + declared_bytes: number; + purpose?: AttachmentPurpose; + message_index?: number; + part_index?: number; + conversation_id?: string; + turn_id?: string; + tool_call_id?: string; +} +export interface AttachmentUpload { + upload_id: string; + attachment_id: string; + occurrence_id: string; + state: "pending_upload"; + reserved_bytes: number; + expires_at: string; + transfer_mode: "authenticated_stream"; + write_capability: string; +} +/** Transport implementations must abort promptly, bound responses and suppress instrumentation. */ +export interface AttachmentTransport { + bootstrap(request: AttachmentBootstrap, signal: AbortSignal): Promise; + initialize(session: AttachmentSession, request: AttachmentUploadRequest, signal: AbortSignal): Promise; + upload(session: AttachmentSession, upload: AttachmentUpload, data: Uint8Array | Blob, signal: AbortSignal): Promise; + complete(session: AttachmentSession, upload: AttachmentUpload, request: AttachmentUploadRequest, signal: AbortSignal): Promise; +} +export interface AttachmentManagerOptions extends AttachmentConfig { + projectName?: string; + projectType?: string; + workspaceId?: string; + traceConfig?: TraceConfigOptions; + /** Node uses process.env via the provider; browsers do not inspect any ambient credentials. */ + environment?: () => Readonly>; + transport?: AttachmentTransport; +} +export interface AttachmentFlushResult { readonly unfinished: number; readonly dropped: number } +export type AttachmentSpan = Span | SpanContext; diff --git a/typescript/packages/fi-core/src/index.ts b/typescript/packages/fi-core/src/index.ts index 24e6448d1..c1e0f021f 100644 --- a/typescript/packages/fi-core/src/index.ts +++ b/typescript/packages/fi-core/src/index.ts @@ -14,3 +14,5 @@ export { projectAttributes, tracerAttributeLimit, DROPPED_CONTENT_COUNT } from " export * from "./awsLambda"; export { rememberProviderResponse } from "./privacy-runtime"; + +export * from "./attachments"; diff --git a/typescript/packages/fi-core/src/otel.ts b/typescript/packages/fi-core/src/otel.ts index 4ced29544..a8cf4ca57 100644 --- a/typescript/packages/fi-core/src/otel.ts +++ b/typescript/packages/fi-core/src/otel.ts @@ -32,6 +32,9 @@ import { seedSecrets } from "./secrets"; import { DefaultBatchSpanProcessor, DefaultSimpleSpanProcessor, SpanProcessorRegistry } from "./spanProcessorRegistry"; import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; +import { createNodeAttachmentManager, NodeAttachmentManagerOptions } from "./attachments/node"; +import { AttachmentManager } from "./attachments/manager"; + const privacyDestinations = new WeakSet(); /** @internal True only for constructors in this core copy, never based on user properties. */ export function hasExportPrivacyBoundary(exporter: SpanExporter): boolean { return privacyDestinations.has(exporter); } @@ -456,6 +459,8 @@ export function createSpanExporter(options: ExportPrivacyOptions & { // --- TracerProvider --- export interface FITracerProviderOptions extends ExportPrivacyOptions { + /** Opt-in out-of-band media, separate from the trace collector destination. */ + attachments?: NodeAttachmentManagerOptions; /** Default false selects SimpleSpanProcessor; true selects BatchSpanProcessor. */ batch?: boolean; resource?: Resource; @@ -469,6 +474,8 @@ export interface FITracerProviderOptions extends ExportPrivacyOptions { } class FITracerProvider extends BasicTracerProvider { + /** The single attachment runtime shared by explicit calls and integration adapters. */ + readonly attachments: AttachmentManager; private readonly processorRegistry: SpanProcessorRegistry; private shutdownPromise?: Promise; private defaultProcessorAttached: boolean = false; @@ -521,6 +528,17 @@ class FITracerProvider extends BasicTracerProvider { this.endpoint = endpoint; this.headers = headers; this.transport = transport; + const attributes = config.resource?.attributes ?? {}; + this.attachments = createNodeAttachmentManager({ + ...config.attachments, + headers: config.attachments?.headers ?? headers, + projectName: config.attachments?.projectName ?? String(attributes[PROJECT_NAME] ?? ""), + projectType: config.attachments?.projectType ?? String(attributes[PROJECT_TYPE] ?? ""), + traceConfig: { ...config.attachments?.traceConfig, ...config.traceConfig, + ...Object.fromEntries(Object.entries(config.attachments?.traceConfig ?? {}).filter(([key, value]) => + (key.startsWith("hide") || key === "piiRedaction") && value === true)), + }, + }); // Log to confirm processor and exporter details if (verbose) { @@ -575,12 +593,15 @@ class FITracerProvider extends BasicTracerProvider { detailsMsg += "| Using a default SpanProcessor. `addSpanProcessor` will overwrite this default.\n"; } } + async forceFlush(): Promise { + await Promise.all([super.forceFlush(), this.attachments.flush()]); + } shutdown(): Promise { if (!this.shutdownPromise) { if (this.verbose) { diag.info("Shutting down FI TracerProvider..."); } - this.shutdownPromise = super.shutdown(); + this.shutdownPromise = Promise.all([super.shutdown(), this.attachments.shutdown()]).then(() => undefined); } return this.shutdownPromise; } @@ -611,6 +632,7 @@ class BatchSpanProcessor extends OTelBatchSpanProcessor { } export interface RegisterOptions extends ExportPrivacyOptions { + attachments?: NodeAttachmentManagerOptions; /** Preserve application service identity; FI project routing is separate. */ resource?: Resource; projectName?: string; @@ -738,6 +760,7 @@ if (!projectName) { batch: options.batch ?? false, spanProcessors: options.spanProcessors, traceConfig: options.traceConfig, + attachments: options.attachments, resourceAttributes: options.resourceAttributes, }); diff --git a/typescript/packages/fi-core/test/attachments-browser.cjs b/typescript/packages/fi-core/test/attachments-browser.cjs new file mode 100644 index 000000000..9f6dd2e25 --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-browser.cjs @@ -0,0 +1,56 @@ +// Run against an esbuild browser bundle with Playwright installed in the qualification tools directory. +const assert = require('node:assert/strict'); +const { chromium } = require(process.argv[2]); +const bundle = process.argv[3]; +const id = '10000000-0000-4000-8000-000000000001'; +const expires = new Date(Date.now() + 60000).toISOString(); +const policy = { version: 'v1', enabled: true, allowed_sources: ['Bytes', 'Blob'], allowed_modalities: ['audio', 'image'], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }; +const limits = { max_upload_bytes: 52428800, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, + recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, + initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, + transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }; +(async () => { + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); const calls = []; let content; + await page.route('https://attachments.example/**', async route => { + const req = route.request(); const url = req.url(); calls.push(url); + const data = req.postDataBuffer(); let response; let status = 200; + if (url.endsWith('attachment-sessions')) response = { schema_version: 1, scope: { organization_id: id, workspace_id: id, project_id: id }, + capture_session_id: JSON.parse(data).capture_session_id, policy, limits, expires_at: expires, transfer_modes: ['authenticated_stream'] }; + else if (url.endsWith('/uploads')) { const r = JSON.parse(data); status = 201; response = { upload_id: id, attachment_id: r.attachment_id, + occurrence_id: r.occurrence_id, state: 'pending_upload', reserved_bytes: r.declared_bytes, expires_at: expires, transfer_mode: 'authenticated_stream', write_capability: 'synthetic-write-capability' }; } + else if (url.endsWith('/content')) { content = [...data]; response = { upload_id: id, state: 'pending_upload', reason: 'queued' }; } + else { status = 202; response = { upload_id: id, state: 'validating', reason: 'binding_pending' }; } + await route.fulfill({ status, headers: { 'content-type': 'application/json', 'access-control-allow-origin': '*' }, body: JSON.stringify(response) }); + }); + await page.addScriptTag({ path: bundle }); + const result = await page.evaluate(async () => { + const { AttachmentManager, createAttachmentHttpTransport } = Attachments; + const identity = { traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }; + const transport = createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic-write-capability' } }); + const manager = new AttachmentManager({ enabled: true, projectName: 'browser-fixture', projectType: 'observe', transport }); + if (!await manager.ready()) throw Error('bootstrap failed'); + const result = manager.attach(identity, { kind: 'Blob', data: new Blob([new Uint8Array([1, 2, 3])]) }, { direction: 'input', mimeType: 'image/png' }); + const completion = await result.completion; await manager.shutdown(); return completion; + }); + assert.deepEqual(result, { status: 'validating' }); assert.deepEqual(content, [1, 2, 3]); assert.equal(calls.length, 4); + const denied = await page.evaluate(async () => { + const { AttachmentManager, createAttachmentHttpTransport } = Attachments; + const transport = createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic-write-capability' } }); + const identity = { traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }; + const states = []; + for (const options of [{}, { enabled: true, traceConfig: { piiRedaction: true } }, { enabled: true, traceConfig: { hideInputImages: true } }]) { + const manager = new AttachmentManager({ projectName: 'browser-fixture', projectType: 'observe', transport, ...options }); + await manager.ready(); let touched = false; + const h = manager.attach(identity, { kind: 'Blob', get data() { touched = true; throw Error('denied-source'); } }, { direction: 'input', mimeType: 'image/png' }); + states.push([h.admission, touched]); await manager.shutdown(); + } + return states; + }); + assert.deepEqual(denied, [['disabled', false], ['redaction_unavailable', false], ['policy_denied', false]]); + assert.equal(calls.length, 6); // two bootstrap controls; ZERO denied transfers. + console.log(JSON.stringify({ browser: browser.version(), passed: 4, transferRequests: 3, deniedTransferRequests: 0 })); + } finally { await browser.close(); } +})().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/typescript/packages/fi-core/test/attachments-contract.cjs b/typescript/packages/fi-core/test/attachments-contract.cjs new file mode 100644 index 000000000..3b34b23c9 --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-contract.cjs @@ -0,0 +1,66 @@ +// Runtime safety replay of every F01 privacy row. Unsupported positive paths +// are reported separately: fail-closed rejection is NOT integrated qualification. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { AttachmentManager, AttachmentCorrelation, AttachmentTransportError, canonicalAttachmentTraceId, attachmentTraceUuid } = require('../dist/src/attachments'); +const { identity, policy, fake, source } = require('./attachments-fixtures.cjs'); +const directory = process.argv[2]; +if (!directory) throw Error('Pass the frozen spec/attachments/v1 directory'); +const privacy = JSON.parse(fs.readFileSync(path.join(directory, 'privacy-fixtures.json'))).cases; +const identities = JSON.parse(fs.readFileSync(path.join(directory, 'identity-fixtures.json'))).cases; +const keepalive = setInterval(() => {}, 1000); +(async () => { + const results = []; + for (const fixture of privacy) { + const c = fixture.context; + const traceConfig = Object.fromEntries(['hide_inputs', 'hide_outputs', 'hide_input_messages', 'hide_output_messages', 'hide_input_images', 'hide_input_text', 'hide_output_text', 'hide_error_details', 'pii_redaction'].map(key => [key.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()), c[key]])); + const mimeType = { audio: 'audio/wav', image: 'image/png', video: 'video/mp4', document: 'application/pdf', file: 'application/octet-stream' }[c.modality]; + const options = { mimeType, direction: c.direction, capture: { captureContent: c.integration_capture_content, recordInputs: c.integration_record_inputs, + recordOutputs: c.integration_record_outputs, muted: c.muted, temporaryChat: c.temporary_chat, requested: c.call_requested, + modelInput: c.model_bound_image, containsInput: c.semantic_input, containsOutput: c.semantic_output, errorDetail: c.error_diagnostic, + newRecording: c.operation === 'create_recording' } }; + const f = fake(); const requests = []; + const transport = { ...f.transport, + async bootstrap(request) { + if (c.policy_known !== true) return new Promise(() => {}); + if (!c.credential_allowed) throw new AttachmentTransportError('credential_scope_unsupported'); + const result = await f.transport.bootstrap(request); + result.policy = { ...policy, enabled: c.server_allowed, + allowed_sources: c.source_authorized ? [c.source_kind] : [], allowed_modalities: c.modality_allowed ? [c.modality] : [], + require_media_redaction: c.server_require_media_redaction, require_metadata_removal: c.require_metadata_removal, + allow_unredacted_media_with_text_pii: c.server_allow_unredacted_media_with_text_pii, record_audio: c.server_record_audio, record_video: c.server_record_video }; + return result; + }, + async initialize(session, request, signal) { requests.push(request.occurrence_id); return f.transport.initialize(session, request, signal); }, + }; + const manager = new AttachmentManager({ enabled: c.enabled, projectName: 'fixture', projectType: 'observe', traceConfig, + recordAudio: c.record_audio, recordVideo: c.record_video, allowUnredactedMediaWithTextPii: c.allow_unredacted_media_with_text_pii, + transport: c.endpoint_configured ? transport : undefined, maxOwnedBytes: c.budget_available ? undefined : 1, + maxDescriptors: c.queue_available ? undefined : 1 }); + await manager.ready(1); + if (!c.generation_current) manager.invalidate('deleted'); + if (!c.queue_available) manager.attach(identity, source(), options); + const actualSource = c.browser_remote_fetch ? { kind: 'RemoteUrl', url: 'https://untrusted.example/source' } : + c.source_kind === 'Bytes' ? { kind: 'Bytes', data: c.source_available ? source().data : undefined } : { kind: c.source_kind }; + const context = !c.correlation_available ? new AttachmentCorrelation() : { ...identity, traceFlags: c.sampled ? 1 : 0 }; + const result = manager.attach(context, actualSource, options); + const unsupported = c.qualified_client_sanitizer || c.qualified_client_metadata_removal || ['RemoteUrl', 'OwnedRecordingFile', 'ProviderFile'].includes(c.source_kind); + if (fixture.expected.decision === 'deny' || unsupported) { + assert.notEqual(result.admission, 'queued', fixture.id); + assert.equal(requests.includes(result.occurrenceId), false, fixture.id); + } else assert.equal(result.admission, 'queued', fixture.id); + results.push({ id: fixture.id, expected: fixture.expected.reason, actual: result.admission, + qualification: fixture.expected.decision === 'allow' && unsupported ? 'unsupported_fail_closed' : 'safety_pass' }); + await manager.shutdown(10); + } + for (const fixture of identities) { + const hex = canonicalAttachmentTraceId(fixture.input); + assert.equal(hex !== undefined, fixture.valid, JSON.stringify(fixture.input)); + if (fixture.valid) assert.equal(attachmentTraceUuid(hex), fixture.canonical); + } + console.log(JSON.stringify({ privacyFixtures: results.length, identityFixtures: identities.length, + exactReasons: results.filter(result => result.expected === result.actual).length, + unsupportedPositive: results.filter(result => result.qualification === 'unsupported_fail_closed'), + conservativeReasonDifferences: results.filter(result => result.expected !== result.actual && result.qualification !== 'unsupported_fail_closed') }, null, 2)); +})().catch(error => { console.error(error); process.exitCode = 1; }).finally(() => clearInterval(keepalive)); diff --git a/typescript/packages/fi-core/test/attachments-fixtures.cjs b/typescript/packages/fi-core/test/attachments-fixtures.cjs new file mode 100644 index 000000000..2dfb96b4f --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-fixtures.cjs @@ -0,0 +1,23 @@ +const identity = { traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }; +const ids = ['10000000-0000-4000-8000-000000000001', '10000000-0000-4000-8000-000000000002', '10000000-0000-4000-8000-000000000003']; +const policy = { version: 'v1', enabled: true, allowed_sources: ['Bytes', 'Blob'], allowed_modalities: ['audio', 'image', 'video', 'document', 'file'], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }; +const limits = { max_upload_bytes: 52428800, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, + recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, + initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, + transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }; +const options = { mimeType: 'audio/wav', direction: 'mixed' }; +const source = () => ({ kind: 'Bytes', data: new Uint8Array([82, 73, 70, 70, 1, 2, 3, 4]) }); +const expiry = () => new Date(Date.now() + 60000).toISOString(); +function fake(overrides = {}) { + const calls = []; let body; + const transport = { + async bootstrap(r) { calls.push('bootstrap'); return { schema_version: 1, scope: { organization_id: ids[0], workspace_id: ids[1], project_id: ids[2] }, capture_session_id: r.capture_session_id, policy, limits, expires_at: expiry(), transfer_modes: ['authenticated_stream'] }; }, + async initialize(_s, r) { calls.push('initialize'); return { upload_id: ids[0], attachment_id: r.attachment_id, occurrence_id: r.occurrence_id, state: 'pending_upload', reserved_bytes: r.declared_bytes, expires_at: expiry(), transfer_mode: 'authenticated_stream', write_capability: 'synthetic-write-capability' }; }, + async upload(_s, _u, data) { calls.push('upload'); body = Array.from(data instanceof Uint8Array ? data : new Uint8Array(await data.arrayBuffer())); }, + async complete() { calls.push('complete'); }, ...overrides, + }; + return { transport, calls, body: () => body }; +} + +module.exports = { identity, ids, policy, limits, options, source, expiry, fake }; diff --git a/typescript/packages/fi-core/test/attachments-tls.test.cjs b/typescript/packages/fi-core/test/attachments-tls.test.cjs new file mode 100644 index 000000000..f57caa9ab --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-tls.test.cjs @@ -0,0 +1,59 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const https = require('node:https'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { execFileSync, spawn } = require('node:child_process'); +const { once } = require('node:events'); +const { ids, fake } = require('./attachments-fixtures.cjs'); + +test('Node TLS transfer uses upload capability, bounds readers and does not keep host alive', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'attachment-tls-')); + const key = path.join(dir, 'test.key'); const cert = path.join(dir, 'test.crt'); + execFileSync('openssl', ['req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', key, '-out', cert, '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=IP:127.0.0.1,DNS:localhost'], { stdio: 'ignore' }); + const bodies = []; const headers = []; let stalled = false; + const server = https.createServer({ key: fs.readFileSync(key), cert: fs.readFileSync(cert) }, async (req, res) => { + headers.push(req.headers); const chunks = []; + for await (const chunk of req) chunks.push(chunk); + const body = Buffer.concat(chunks); res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('attachment-sessions')) res.end(JSON.stringify(await fake().transport.bootstrap(JSON.parse(body)))); + else if (req.url.endsWith('/uploads')) { res.statusCode = 201; res.end(JSON.stringify(await fake().transport.initialize(null, JSON.parse(body)))); } + else if (req.url.endsWith('/content')) { + bodies.push(body); assert.equal(Number(req.headers['content-length']), body.length); + if (stalled) return; + res.end(JSON.stringify({ upload_id: ids[0], state: 'pending_upload', reason: 'queued' })); + } else { res.statusCode = 202; res.end(JSON.stringify({ upload_id: ids[0], state: 'validating', reason: 'binding_pending' })); } + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const modulePath = path.resolve(path.dirname(process.env.FI_CORE_PACKAGE_JSON || path.resolve(__dirname, '../package.json')), 'dist/src/attachments/node.js'); + async function child(stall) { + const script = ` + const { createNodeAttachmentManager } = require(${JSON.stringify(modulePath)}); + const manager = createNodeAttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', + endpoint: 'https://127.0.0.1:${server.address().port}', headers: { authorization: 'Bearer synthetic-bootstrap', 'x-api-key': 'synthetic-initial-only' } }); + const live = setTimeout(() => {}, ${stall ? 500 : 10000}); + (async () => { + if (!await manager.ready()) throw Error('ready'); + const handle = manager.attach({ traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }, + { kind: 'Blob', data: new Blob([new Uint8Array(700000).fill(7)]) }, { direction: 'mixed', mimeType: 'audio/wav' }); + if (${stall}) { console.log('TRANSFER_STARTED'); return; } + console.log(JSON.stringify(await handle.completion)); await manager.shutdown(); clearTimeout(live); + })().catch(() => process.exitCode = 1); + `; + const processChild = spawn(process.execPath, ['-e', script], { env: { ...process.env, NODE_EXTRA_CA_CERTS: cert }, stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; let stderr = ''; processChild.stdout.on('data', c => { stdout += c; }); processChild.stderr.on('data', c => { stderr += c; }); + const deadline = setTimeout(() => processChild.kill('SIGKILL'), 5000); + const [code, signal] = await once(processChild, 'exit'); clearTimeout(deadline); + assert.equal(signal, null, 'background transfer kept the process alive'); assert.equal(code, 0, stderr); + return stdout; + } + try { + assert.match(await child(false), /"status":"validating"/); + assert.equal(bodies[0].length, 700000); assert.ok(bodies[0].every(byte => byte === 7)); + assert.equal(headers[2].authorization, 'Bearer synthetic-write-capability'); assert.equal(headers[2]['x-api-key'], undefined); + assert.equal(headers[3].authorization, 'Bearer synthetic-write-capability'); + stalled = true; + assert.match(await child(true), /TRANSFER_STARTED/); assert.equal(bodies.length, 2); + } finally { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); fs.rmSync(dir, { recursive: true, force: true }); } +}); diff --git a/typescript/packages/fi-core/test/attachments.test.cjs b/typescript/packages/fi-core/test/attachments.test.cjs new file mode 100644 index 000000000..20467abc6 --- /dev/null +++ b/typescript/packages/fi-core/test/attachments.test.cjs @@ -0,0 +1,235 @@ +/* Component qualification only: synthetic bytes and a loopback attachment API. */ +const { test, before, after } = require('node:test'); +const assert = require('node:assert/strict'); +const http = require('node:http'); +const { once } = require('node:events'); +const { pathToFileURL } = require('node:url'); +const path = require('node:path'); +const { createRequire } = require('node:module'); +const packageJson = process.env.FI_CORE_PACKAGE_JSON || path.resolve(__dirname, '../package.json'); +const load = createRequire(packageJson); +const { AttachmentManager, AttachmentCorrelation, canonicalAttachmentTraceId, attachmentTraceUuid, + evaluateAttachmentPolicy, createAttachmentHttpTransport, AttachmentTransportError } = load('@traceai/fi-core/attachments'); +const { register, ProjectType } = load('@traceai/fi-core'); +const { identity, ids, policy, limits, options, source, expiry, fake } = require('./attachments-fixtures.cjs'); +async function manager(overrides = {}, transportOverrides = {}) { + const fixture = fake(transportOverrides); + const value = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', transport: fixture.transport, ...overrides }); + assert.equal(await value.ready(), true); + return { value, ...fixture }; +} +const live = setInterval(() => {}, 1000); // Explicit test owner, not an SDK keepalive. +after(() => clearInterval(live)); + +test('OFF by default: no bootstrap, source access or reference metadata', async () => { + const f = fake(); let read = false; + const value = new AttachmentManager({ transport: f.transport }); + const result = value.attach(identity, { kind: 'Bytes', get data() { read = true; throw Error('synthetic-secret'); } }, options); + assert.equal(result.admission, 'disabled'); assert.equal(read, false); assert.deepEqual(f.calls, []); + assert.deepEqual(await result.completion, { status: 'disabled' }); assert.equal(JSON.stringify(result).includes('synthetic-secret'), false); +}); + +test('unknown bootstrap retains no bytes and never reads a denied source', async () => { + const value = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', transport: fake({ bootstrap: () => new Promise(() => {}) }).transport }); + let read = false; + const h = value.attach(identity, { kind: 'Bytes', get data() { read = true; return new Uint8Array(1); } }, options); + assert.equal(h.admission, 'bootstrap_pending'); assert.equal(read, false); assert.equal(value.status().ownedBytes, 0); + await value.shutdown(1); +}); + +for (const [key, direction, mimeType, capture] of [ + ['hideInputs', 'input', 'audio/wav'], ['hideInputMessages', 'input', 'application/pdf'], ['hideInputText', 'input', 'image/png'], + ['hideOutputs', 'output', 'audio/wav'], ['hideOutputMessages', 'output', 'image/png'], ['hideOutputText', 'mixed', 'audio/wav'], + ['hideInputImages', 'output', 'image/png', { modelInput: true }], ['hideErrorDetails', 'output', 'audio/wav', { errorDetail: true }], + ['piiRedaction', 'mixed', 'audio/wav'], ['piiRedaction', 'input', 'image/png'], ['piiRedaction', 'output', 'video/mp4'], + ['piiRedaction', 'input', 'application/pdf'], ['piiRedaction', 'input', 'application/octet-stream'], +]) test(`no source access or upload with ${key}/${mimeType}`, async () => { + const f = await manager({ traceConfig: { [key]: true } }); let read = false; + const h = f.value.attach(identity, { kind: 'Bytes', get data() { read = true; throw Error('synthetic-secret'); } }, { mimeType, direction, capture }); + assert.equal(h.admission, key === 'piiRedaction' ? 'redaction_unavailable' : 'policy_denied'); + await f.value.flush(); assert.equal(read, false); assert.deepEqual(f.calls, ['bootstrap']); await f.value.shutdown(); +}); + +test('environment restrictions win over explicit per-call and constructor allows', async () => { + const f = await manager({ environment: () => ({ FI_HIDE_INPUTS: 'malformed' }), traceConfig: { hideInputs: false } }); + assert.equal(f.value.attach(identity, source(), options).admission, 'policy_denied'); + assert.equal(evaluateAttachmentPolicy({ enabled: true }, {}, policy, 'Bytes', options, { FI_ATTACHMENTS_ENABLED: 'false' }), 'disabled'); + await f.value.shutdown(); +}); + +test('mutable ownership is detached, admission asynchronous, completion not readiness', async () => { + const f = await manager(); const src = source(); + const h = f.value.attach(identity, src, options); src.data.fill(9); + assert.deepEqual(f.calls, ['bootstrap']); assert.equal(h.admission, 'queued'); + assert.deepEqual(await h.completion, { status: 'validating' }); assert.deepEqual(f.body(), [82, 73, 70, 70, 1, 2, 3, 4]); + assert.deepEqual(await f.value.flush(), { unfinished: 0, dropped: 0 }); assert.equal(f.value.status().ownedBytes, 0); await f.value.shutdown(); +}); + +test('Blob read is independent; unknown stream, file and URL never read', async () => { + const f = await manager(); const blob = new Blob([new Uint8Array([1, 2, 3])]); + assert.equal((await f.value.attach(identity, { kind: 'Blob', data: blob }, options).completion).status, 'validating'); + for (const kind of ['RemoteUrl', 'LocalFile', 'OwnedRecordingFile', 'ProviderFile']) { + const item = { kind, get url() { throw Error('must-not-read'); }, get path() { throw Error('must-not-read'); } }; + assert.notEqual(f.value.attach(identity, item, options).admission, 'queued'); + } + let pulled = false; + assert.equal(f.value.attach(identity, { kind: 'Bytes', data: { getReader() { pulled = true; } } }, options).admission, 'source_unavailable'); + assert.equal(pulled, false); await f.value.shutdown(); +}); + +test('known sampled identity required and exact trace bits preserved', async () => { + const f = await manager(); + for (const traceId of ['00000000000000000000000000000000', 'BAD', '0123456789ABCDEF0123456789ABCDEF']) assert.equal(canonicalAttachmentTraceId(traceId), undefined); + const hex = 'ffffffffffffffffffffffffffffffff'; assert.equal(attachmentTraceUuid(canonicalAttachmentTraceId(hex)), 'ffffffff-ffff-ffff-ffff-ffffffffffff'); + assert.equal(f.value.attach({ ...identity, traceFlags: 0 }, source(), options).admission, 'unsampled'); + assert.equal(f.value.attach(new AttachmentCorrelation(), source(), options).admission, 'correlation_unavailable'); + await f.value.shutdown(); +}); + +test('same-span references bounded to 64; ended span untouched by completion', async () => { + const f = await manager(); let recording = true; let writes = 0; const attrs = {}; + const span = { spanContext: () => identity, isRecording: () => recording, setAttribute(k, v) { writes++; attrs[k] = v; } }; + const correlation = f.value.correlation(span); const handles = []; + for (let i = 0; i < 70; i++) handles.push(f.value.attach(span, source(), options)); + assert.equal(JSON.parse(attrs['fi.attachment.refs']).length, 64); assert.ok(Buffer.byteLength(attrs['fi.attachment.refs']) <= 16384); + recording = false; const before = writes; + await Promise.all(handles.map(h => h.completion)); assert.equal(writes, before); + assert.equal(correlation.snapshot().sampled, true); await f.value.shutdown(); +}); + +test('memory/descriptors include active transfer and backpressure drops without copying', async () => { + let release; const wait = new Promise(resolve => { release = resolve; }); + const f = await manager({ maxOwnedBytes: 8, maxDescriptors: 1 }, { upload: async () => { await wait; } }); + const h = f.value.attach(identity, source(), options); let read = false; + assert.equal(f.value.attach(identity, { kind: 'Bytes', get data() { read = true; return new Uint8Array(8); } }, options).admission, 'queue_full'); + assert.equal(read, false); assert.equal(f.value.status().ownedBytes, 8); + release(); await h.completion; await f.value.flush(); assert.equal(f.value.status().ownedBytes, 0); + assert.equal(f.value.attach(identity, { kind: 'Bytes', data: new Uint8Array(9) }, options).admission, 'too_large'); + await f.value.shutdown(); +}); + +test('policy tightening cancels queued bytes and does not resurrect after re-enable', async () => { + const f = await manager(); const h = f.value.attach(identity, source(), options); + f.value.updatePolicy({ traceConfig: { hideInputs: true } }); + assert.equal((await h.completion).status, 'policy_denied'); assert.equal(f.value.status().ownedBytes, 0); + f.value.updatePolicy({ traceConfig: { hideInputs: false } }); await f.value.flush(); assert.deepEqual(f.calls, ['bootstrap']); await f.value.shutdown(); +}); + +test('bounded retries recheck live policy and preserve occurrence IDs', async () => { + let tries = 0; const requests = []; const f = fake(); + const fixture = await manager({}, { initialize: async (s, r, signal) => { requests.push(r.occurrence_id); if (++tries < 3) throw new AttachmentTransportError('failed', true); return f.transport.initialize(s, r, signal); } }); + const h = fixture.value.attach(identity, source(), options); await h.completion; + assert.equal(tries, 3); assert.equal(new Set(requests).size, 1); await fixture.value.shutdown(); +}); + +test('authorization failures never retry and exception bodies are absent', async () => { + let tries = 0; + const f = await manager({}, { initialize: async () => { tries++; throw new AttachmentTransportError('credential_scope_unsupported'); } }); + const h = f.value.attach(identity, source(), options); assert.deepEqual(await h.completion, { status: 'credential_scope_unsupported' }); assert.equal(tries, 1); await f.value.shutdown(); +}); + +test('flush deadline and idempotent shutdown cancel active work', async () => { + let aborted = false; + const f = await manager({}, { upload: (_s, _u, _d, signal) => new Promise((resolve, reject) => signal.addEventListener('abort', () => { aborted = true; reject(Error('synthetic-secret')); }, { once: true })) }); + const h = f.value.attach(identity, source(), options); + await new Promise(resolve => setImmediate(resolve)); + assert.equal((await f.value.flush(5)).unfinished, 1); + const closing = f.value.shutdown(5); assert.equal(f.value.shutdown(5), closing); await closing; + assert.equal(aborted, true); assert.equal((await h.completion).status, 'incomplete'); + assert.equal(f.value.attach(identity, source(), options).admission, 'incomplete'); +}); + +test('HTTP v1 loopback upload: stable identity, bounded response, no source URLs', async () => { + const calls = []; const uploads = new Map(); let policyOverride; + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const chunk of req) chunks.push(chunk); + const data = Buffer.concat(chunks); calls.push({ method: req.method, path: req.url, data, headers: req.headers }); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('attachment-sessions')) { + const body = JSON.parse(data); const fixture = await fake().transport.bootstrap(body); + res.end(JSON.stringify({ ...fixture, policy: policyOverride ?? policy })); + } else if (req.url.endsWith('/uploads')) { + const body = JSON.parse(data); const upload = await fake().transport.initialize(null, body); uploads.set(upload.upload_id, upload); + assert.equal(body.association.parent.trace_id, '01234567-89ab-cdef-0123-456789abcdef'); res.statusCode = 201; res.end(JSON.stringify(upload)); + } else if (req.url.endsWith('/content')) { + assert.deepEqual([...data], [...source().data]); res.end(JSON.stringify({ upload_id: ids[0], state: 'pending_upload', reason: 'queued' })); + } else { res.statusCode = 202; res.end(JSON.stringify({ upload_id: ids[0], state: 'validating', reason: 'binding_pending' })); } + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const client = req => new Promise((resolve, reject) => { + const request = http.request(`http://127.0.0.1:${server.address().port}${new URL(req.url).pathname}`, { method: req.method, headers: req.headers, signal: req.signal }, response => { + const chunks = []; response.on('data', c => chunks.push(c)); response.on('end', () => resolve({ status: response.statusCode, body: Buffer.concat(chunks) })); + }); request.on('error', reject); request.end(req.body); + }); + try { + const transport = createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic', cookie: 'never-forward' }, client }); + const f = await manager({ transport }); const h = f.value.attach(identity, source(), options); + assert.equal((await h.completion).status, 'validating'); assert.equal(calls.length, 4); assert.equal(calls.some(c => c.headers.cookie), false); + await f.value.shutdown(); + for (const traceConfig of [{ hideInputs: true }, { hideOutputs: true }, { piiRedaction: true }]) { + const value = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', transport, traceConfig }); + await value.ready(); const before = calls.length; assert.notEqual(value.attach(identity, source(), options).admission, 'queued'); + await value.flush(); assert.equal(calls.length, before); await value.shutdown(); + } + } finally { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } +}); + +test('transport blocks redirects and invalid origins', async () => { + for (const endpoint of ['http://example.com', 'https://user:secret@example.com', 'https://example.com/path', 'https://example.com/?token=secret']) assert.throws(() => createAttachmentHttpTransport({ endpoint, headers: {} }), /endpoint_unconfigured/); + const transport = createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: {}, client: async () => ({ status: 302, body: Buffer.from('synthetic-secret') }) }); + await assert.rejects(transport.bootstrap({}, new AbortController().signal), error => error.message === 'failed' && !error.retryable); +}); + +test('CJS and ESM leaf export the public manager API', async () => { + const esm = await import(pathToFileURL(path.resolve(path.dirname(packageJson), 'dist/esm/attachments/index.js'))); + assert.equal(esm.AttachmentManager, AttachmentManager); assert.equal(esm.AttachmentCorrelation, AttachmentCorrelation); assert.equal(typeof esm.createAttachmentHttpTransport, 'function'); +}); + +test('provider owns one manager and flush/shutdown drain its lifecycle', async () => { + const f = fake(); + const provider = register({ projectName: 'fixture', projectType: ProjectType.OBSERVE, setGlobalTracerProvider: false, spanProcessors: [], attachments: { enabled: true, transport: f.transport } }); + await provider.attachments.ready(); + const h = provider.attachments.attach(identity, source(), options); await provider.forceFlush(); assert.equal((await h.completion).status, 'validating'); + const first = provider.shutdown(); assert.equal(first, provider.shutdown()); await first; +}); + +test('late completion uses the stamped writer carrier and never mutates an ended span', async () => { + const f = await manager(); const attributes = {}; let recording = true; let writes = 0; + const span = { spanContext: () => identity, isRecording: () => recording, setAttribute(k, v) { attributes[k] = v; writes++; } }; + const correlation = f.value.correlation(span); + assert.match(attributes['fi.attachment.capture_session_id'], /^[0-9a-f-]{36}$/); + recording = false; const before = writes; + const h = f.value.attach(correlation, source(), { ...options, conversationId: 'conversation-1', turnId: 'turn-1', toolCallId: 'call-1' }); + assert.equal((await h.completion).status, 'validating'); assert.equal(writes, before); await f.value.shutdown(); +}); + +test('untrusted metadata keys and malformed IDs are not retained or projected', async () => { + const f = await manager(); + const bad = ['abc\n', 'has spaces', 'secret@example.com', '../path', 'x'.repeat(129)]; + for (const conversationId of bad) assert.equal(f.value.attach(identity, source(), { ...options, conversationId }).admission, 'policy_denied'); + assert.equal(canonicalAttachmentTraceId(identity.traceId + '\n'), undefined); + assert.equal(f.value.attach({ ...identity, spanId: identity.spanId + '\n' }, source(), options).admission, 'correlation_unavailable'); + assert.equal(f.value.attach(identity, source(), { ...options, capture: { recordInputs: 'unknown' } }).admission, 'policy_denied'); + await f.value.shutdown(); +}); + +test('upload response binds exact IDs, reservation and capability', async () => { + for (const field of ['attachment_id', 'occurrence_id', 'reserved_bytes', 'write_capability']) { + const t = fake(); const f = await manager({}, { initialize: async (s, r) => ({ ...await t.transport.initialize(s, r), [field]: field === 'reserved_bytes' ? 999 : 'wrong' }) }); + const h = f.value.attach(identity, source(), options); assert.equal((await h.completion).status, 'failed'); + assert.equal(f.calls.includes('upload'), false); await f.value.shutdown(); + } +}); + +test('successful response reader rejects oversized control data', async () => { + const transport = createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: {}, client: async () => ({ status: 200, body: new Uint8Array(65537) }) }); + await assert.rejects(transport.bootstrap({}, new AbortController().signal), error => error.message === 'failed' && !error.retryable); +}); + +test('provider trace options cannot weaken attachment-level privacy restrictions', async () => { + const f = fake(); + const provider = register({ projectName: 'fixture', projectType: ProjectType.OBSERVE, setGlobalTracerProvider: false, spanProcessors: [], + traceConfig: { hideInputs: false }, attachments: { enabled: true, transport: f.transport, traceConfig: { hideInputs: true } } }); + await provider.attachments.ready(); assert.equal(provider.attachments.attach(identity, source(), options).admission, 'policy_denied'); + await provider.shutdown(); assert.deepEqual(f.calls, ['bootstrap']); +}); From d787ba08ccf7f467708b1096201240782e9b2a3a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 17:41:27 -0800 Subject: [PATCH 2/9] fix(ci): pin supported browser qualification runner Playwright 1.56.1 does not support the Ubuntu 26.04 image now selected by ubuntu-latest; retain the qualified browser on Ubuntu 24.04. Co-Authored-By: Claude Code --- .github/workflows/fi-core-attachments-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/fi-core-attachments-tests.yml b/.github/workflows/fi-core-attachments-tests.yml index b64954d48..6fc84201a 100644 --- a/.github/workflows/fi-core-attachments-tests.yml +++ b/.github/workflows/fi-core-attachments-tests.yml @@ -10,7 +10,7 @@ permissions: contents: read jobs: runtime: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 15 strategy: fail-fast: false From 7145f78396b42e938c1ebff0d5c5b57fd3ebbdc5 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 18:14:48 -0800 Subject: [PATCH 3/9] fix(core): preserve explicit attachment lifecycle grace periods Keep ready and flush deadlines referenced only while explicitly awaited. Background requests and retries remain unreferenced. Reproduce prior early CLI exit with a real TLS child process that has no artificial keepalive. Co-Authored-By: Claude Code --- typescript/packages/fi-core/src/attachments/manager.ts | 5 +++-- typescript/packages/fi-core/src/attachments/types.ts | 2 +- typescript/packages/fi-core/test/attachments-tls.test.cjs | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/typescript/packages/fi-core/src/attachments/manager.ts b/typescript/packages/fi-core/src/attachments/manager.ts index fce6f7cf2..e23390829 100644 --- a/typescript/packages/fi-core/src/attachments/manager.ts +++ b/typescript/packages/fi-core/src/attachments/manager.ts @@ -257,7 +257,8 @@ export class AttachmentManager { private notify(): void { for (const notify of this.listeners) notify(); } private deadline(promise: Promise, timeout: number, fallback: T): Promise { return new Promise(resolve => { - const timer = setTimeout(() => resolve(fallback), timeout); unref(timer); + // An explicit lifecycle wait owns a bounded grace period; background work stays unref'd. + const timer = setTimeout(() => resolve(fallback), timeout); void promise.then(value => { clearTimeout(timer); resolve(value); }, () => { clearTimeout(timer); resolve(fallback); }); }); } @@ -267,7 +268,7 @@ export class AttachmentManager { await new Promise(resolve => { const done = () => { clearTimeout(timer); this.listeners.delete(changed); resolve(); }; const changed = () => { if (!this.queue.length && !this.active.size) done(); }; - const timer = setTimeout(done, bounded(timeoutMs, this.shutdownTimeout, 30000)); unref(timer); + const timer = setTimeout(done, bounded(timeoutMs, this.shutdownTimeout, 30000)); this.listeners.add(changed); changed(); }); } diff --git a/typescript/packages/fi-core/src/attachments/types.ts b/typescript/packages/fi-core/src/attachments/types.ts index 36f719c70..4c1b906d3 100644 --- a/typescript/packages/fi-core/src/attachments/types.ts +++ b/typescript/packages/fi-core/src/attachments/types.ts @@ -19,7 +19,7 @@ export interface AttachmentHandle { readonly attachmentId: string; readonly occurrenceId: string; readonly admission: AttachmentAdmission; - /** Always resolves; validating means transferred, NOT ready or readable. */ + /** Never rejects; validating is transferred, NOT readable. Use flush/shutdown for process-exit grace. */ readonly completion: Promise; } /** Semantic ancestry can only add restrictions, never remove a directional restriction. */ diff --git a/typescript/packages/fi-core/test/attachments-tls.test.cjs b/typescript/packages/fi-core/test/attachments-tls.test.cjs index f57caa9ab..0d6ea7b5b 100644 --- a/typescript/packages/fi-core/test/attachments-tls.test.cjs +++ b/typescript/packages/fi-core/test/attachments-tls.test.cjs @@ -32,13 +32,14 @@ test('Node TLS transfer uses upload capability, bounds readers and does not keep const { createNodeAttachmentManager } = require(${JSON.stringify(modulePath)}); const manager = createNodeAttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', endpoint: 'https://127.0.0.1:${server.address().port}', headers: { authorization: 'Bearer synthetic-bootstrap', 'x-api-key': 'synthetic-initial-only' } }); - const live = setTimeout(() => {}, ${stall ? 500 : 10000}); + const live = ${stall} ? setTimeout(() => {}, 500) : undefined; (async () => { if (!await manager.ready()) throw Error('ready'); const handle = manager.attach({ traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }, { kind: 'Blob', data: new Blob([new Uint8Array(700000).fill(7)]) }, { direction: 'mixed', mimeType: 'audio/wav' }); if (${stall}) { console.log('TRANSFER_STARTED'); return; } - console.log(JSON.stringify(await handle.completion)); await manager.shutdown(); clearTimeout(live); + await manager.flush(); console.log(JSON.stringify(await handle.completion)); await manager.shutdown(); + if (live) clearTimeout(live); })().catch(() => process.exitCode = 1); `; const processChild = spawn(process.execPath, ['-e', script], { env: { ...process.env, NODE_EXTRA_CA_CERTS: cert }, stdio: ['ignore', 'pipe', 'pipe'] }); From 149fef0a80c6c20c118e8b2b4b3dcd635e8c4550 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 21:53:23 -0800 Subject: [PATCH 4/9] feat(core): TypeScript attachment streaming, recordings, imports and lifecycle per the SDK design (WIP) Implements sdk-streaming-design sections 0-9 in fi-core attachments: preflight tickets, PCM recording writer with WAV segmentation and prefix-monotonic manifests, consumed-stream uploads, completed-recording import seam, owned recording file scope and tiers, capability and session renewal by replay, the disabled null object, the one versioned wire adapter (idempotency keys, status table, abort, manifest, import routes) and hand validators for every F01 shape. The legacy four-method transport keeps working through an adapter. Co-Authored-By: Claude Opus 5.5 --- .../workflows/fi-core-attachments-tests.yml | 16 +- typescript/packages/fi-core/package.json | 2 +- .../packages/fi-core/scripts/postbuild.cjs | 2 +- .../fi-core/src/attachments/admission.ts | 50 + .../fi-core/src/attachments/correlation.ts | 5 +- .../fi-core/src/attachments/disabled.ts | 51 + .../fi-core/src/attachments/imports.ts | 42 + .../packages/fi-core/src/attachments/index.ts | 14 +- .../fi-core/src/attachments/internal.ts | 65 + .../fi-core/src/attachments/manager.ts | 1156 ++++++++++++++--- .../fi-core/src/attachments/node-sources.ts | 112 ++ .../packages/fi-core/src/attachments/node.ts | 89 +- .../packages/fi-core/src/attachments/owned.ts | 28 + .../fi-core/src/attachments/recording.ts | 821 ++++++++++++ .../fi-core/src/attachments/stream.ts | 44 + .../fi-core/src/attachments/transport-v1.ts | 383 +++++- .../packages/fi-core/src/attachments/types.ts | 187 ++- .../packages/fi-core/src/attachments/wire.ts | 132 ++ .../fi-core/test/attachments-browser.cjs | 2 +- .../fi-core/test/attachments-design.test.cjs | 729 +++++++++++ .../fi-core/test/attachments-exit-probe.cjs | 49 + .../fi-core/test/attachments-exit.test.cjs | 17 + .../fi-core/test/attachments-files.test.cjs | 138 ++ .../fi-core/test/attachments-fixtures.cjs | 3 +- .../packages/fi-core/test/attachments-v1.cjs | 93 ++ .../fi-core/test/attachments-wire.test.cjs | 32 + .../fi-core/test/attachments.test.cjs | 10 +- 27 files changed, 4008 insertions(+), 264 deletions(-) create mode 100644 typescript/packages/fi-core/src/attachments/admission.ts create mode 100644 typescript/packages/fi-core/src/attachments/disabled.ts create mode 100644 typescript/packages/fi-core/src/attachments/imports.ts create mode 100644 typescript/packages/fi-core/src/attachments/internal.ts create mode 100644 typescript/packages/fi-core/src/attachments/node-sources.ts create mode 100644 typescript/packages/fi-core/src/attachments/owned.ts create mode 100644 typescript/packages/fi-core/src/attachments/recording.ts create mode 100644 typescript/packages/fi-core/src/attachments/stream.ts create mode 100644 typescript/packages/fi-core/src/attachments/wire.ts create mode 100644 typescript/packages/fi-core/test/attachments-design.test.cjs create mode 100644 typescript/packages/fi-core/test/attachments-exit-probe.cjs create mode 100644 typescript/packages/fi-core/test/attachments-exit.test.cjs create mode 100644 typescript/packages/fi-core/test/attachments-files.test.cjs create mode 100644 typescript/packages/fi-core/test/attachments-v1.cjs create mode 100644 typescript/packages/fi-core/test/attachments-wire.test.cjs diff --git a/.github/workflows/fi-core-attachments-tests.yml b/.github/workflows/fi-core-attachments-tests.yml index 6fc84201a..3b5e51409 100644 --- a/.github/workflows/fi-core-attachments-tests.yml +++ b/.github/workflows/fi-core-attachments-tests.yml @@ -24,8 +24,16 @@ jobs: - uses: pnpm/action-setup@v4 with: version: 10.30.1 + - name: Fetch pinned F01 contract fixtures + uses: actions/checkout@v4 + with: + ref: f3b0923c62d43b3a52bf4c16a2226c39c22b690d + path: attachment-contract + persist-credentials: false - name: Build core and run synthetic attachment boundaries working-directory: typescript + env: + FI_ATTACHMENTS_SPEC_DIR: ${{ github.workspace }}/attachment-contract/spec/attachments/v1 run: | pnpm --filter @traceai/fi-core... install --frozen-lockfile --ignore-scripts pnpm --filter @traceai/fi-semantic-conventions run build @@ -40,14 +48,8 @@ jobs: pnpm --dir packages/fi-semantic-conventions pack --pack-destination "$RUNNER_TEMP/attachment-packs" for version in 2.0.1 2.12.1; do node packages/fi-core/test/install-sdk-fixture.cjs "$version" "$RUNNER_TEMP/attachment-sdk-$version" "$RUNNER_TEMP"/attachment-packs/traceai-fi-core-*.tgz "$RUNNER_TEMP"/attachment-packs/traceai-fi-semantic-conventions-*.tgz - FI_CORE_PACKAGE_JSON="$RUNNER_TEMP/attachment-sdk-$version/node_modules/@traceai/fi-core/package.json" node --test packages/fi-core/test/attachments.test.cjs packages/fi-core/test/attachments-tls.test.cjs packages/fi-core/test/provider-lifecycle.test.cjs + FI_CORE_PACKAGE_JSON="$RUNNER_TEMP/attachment-sdk-$version/node_modules/@traceai/fi-core/package.json" FI_ATTACHMENTS_SPEC_DIR="$GITHUB_WORKSPACE/attachment-contract/spec/attachments/v1" node --test packages/fi-core/test/attachments.test.cjs packages/fi-core/test/attachments-tls.test.cjs packages/fi-core/test/attachments-design.test.cjs packages/fi-core/test/attachments-files.test.cjs packages/fi-core/test/attachments-exit.test.cjs packages/fi-core/test/attachments-wire.test.cjs packages/fi-core/test/provider-lifecycle.test.cjs done - - name: Fetch pinned feature-off contract fixtures - uses: actions/checkout@v4 - with: - ref: aeed09f27d9d1f3649282dd94c885850af3ca27e - path: attachment-contract - persist-credentials: false - name: Replay privacy and identity fixture safety run: node typescript/packages/fi-core/test/attachments-contract.cjs attachment-contract/spec/attachments/v1 - name: Qualify browser-safe leaf diff --git a/typescript/packages/fi-core/package.json b/typescript/packages/fi-core/package.json index 0f152d433..c0b9be5fc 100644 --- a/typescript/packages/fi-core/package.json +++ b/typescript/packages/fi-core/package.json @@ -35,7 +35,7 @@ "postbuild": "node scripts/postbuild.cjs", "test": "jest", "test:provider": "node --test test/provider-lifecycle.test.cjs", - "test:attachments": "node --test test/attachments.test.cjs test/attachments-tls.test.cjs", + "test:attachments": "node --test test/attachments.test.cjs test/attachments-tls.test.cjs test/attachments-design.test.cjs test/attachments-files.test.cjs test/attachments-exit.test.cjs test/attachments-wire.test.cjs", "test:watch": "jest --watch", "test:coverage": "jest --coverage", "lint": "eslint . --ext .ts", diff --git a/typescript/packages/fi-core/scripts/postbuild.cjs b/typescript/packages/fi-core/scripts/postbuild.cjs index 8b419740d..09450b0fe 100644 --- a/typescript/packages/fi-core/scripts/postbuild.cjs +++ b/typescript/packages/fi-core/scripts/postbuild.cjs @@ -19,7 +19,7 @@ for (const dir of ['dist/esm', 'dist/esnext']) { // A provider and an adapter may import different module formats. Keep correlation // class identity (and its private span references) identical across those imports. -for (const name of ['index', 'manager', 'correlation', 'policy', 'transport-v1', 'node']) { +for (const name of ['index', 'manager', 'correlation', 'policy', 'transport-v1', 'node', 'disabled', 'admission', 'stream', 'recording', 'imports', 'owned', 'node-sources', 'wire']) { const exports = Object.keys(require(`../dist/src/attachments/${name}.js`)); if (!exports.every(key => /^[A-Za-z_$][\w$]*$/.test(key))) throw new Error('postbuild: unexpected attachment export'); for (const dir of ['dist/esm', 'dist/esnext']) { diff --git a/typescript/packages/fi-core/src/attachments/admission.ts b/typescript/packages/fi-core/src/attachments/admission.ts new file mode 100644 index 000000000..df34a7944 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/admission.ts @@ -0,0 +1,50 @@ +import type { AttachmentCorrelation } from "./correlation"; +import type { AdmissionFacts } from "./internal"; +import type { + AttachmentAdmission, AttachmentAdmissionTicket, AttachmentFields, AttachmentHandle, AttachmentSource, AttachmentStream, AttachmentStreamOptions, +} from "./types"; + +/** @internal */ +export interface TicketHost { + commitTicket(ticket: AttachmentTicket, source: Extract, fields: AttachmentFields): AttachmentHandle; + commitStream(ticket: AttachmentTicket, options: AttachmentStreamOptions): AttachmentStream; + releaseReservation(id: number): void; + deniedHandle(reason: AttachmentAdmission): AttachmentHandle; + deniedStream(reason: AttachmentAdmission): AttachmentStream; +} +/** + * Single-use preflight ticket (design 1.5). It holds a reservation id only, never the source; the manager tracks the + * reservation by id so an abandoned ticket can be collected and its budget returned by a finalizer. + */ +export class AttachmentTicket implements AttachmentAdmissionTicket { + /** @internal */ used = false; + constructor( + private readonly host: TicketHost, + readonly reason: AttachmentAdmission, + readonly maxBytes: number, + readonly correlation: AttachmentCorrelation | undefined, + /** @internal */ readonly facts: AdmissionFacts | undefined, + /** @internal */ readonly reservation: number, + /** @internal */ readonly expiresAt: number, + ) {} + get admitted(): boolean { return this.reason === "queued"; } + attach(source: Extract, fields: AttachmentFields = {}): AttachmentHandle { + try { + if (!this.admitted) return this.host.deniedHandle(this.reason); + if (this.used) return this.host.deniedHandle("conflict" as AttachmentAdmission); + return this.host.commitTicket(this, source, fields ?? {}); + } catch { return this.host.deniedHandle("source_unavailable"); } + } + startStream(options: AttachmentStreamOptions): AttachmentStream { + try { + if (!this.admitted) return this.host.deniedStream(this.reason); + if (this.used) return this.host.deniedStream("conflict" as AttachmentAdmission); + return this.host.commitStream(this, options); + } catch { return this.host.deniedStream("source_unavailable"); } + } + release(): void { + if (this.used || !this.admitted) return; + this.used = true; + this.host.releaseReservation(this.reservation); + } +} diff --git a/typescript/packages/fi-core/src/attachments/correlation.ts b/typescript/packages/fi-core/src/attachments/correlation.ts index 1cd8567ad..7174a7761 100644 --- a/typescript/packages/fi-core/src/attachments/correlation.ts +++ b/typescript/packages/fi-core/src/attachments/correlation.ts @@ -31,14 +31,15 @@ export class AttachmentCorrelation { return true; } catch { return false; } } - /** Stamp only the non-secret writer carrier while the actual span is still recording. */ + /** Stamp the four F01 carrier attributes while the actual span is still recording. Called only for an ADMITTED capture. */ prepare(sessionId: string, nonce: string): void { try { if (this.prepared || !this.span?.isRecording()) return; this.span.setAttribute("fi.attachment.schema_version", 1); this.span.setAttribute("fi.attachment.capture_session_id", sessionId); this.span.setAttribute("fi.attachment.correlation_nonce", nonce); - this.span.setAttribute("fi.attachment.refs", "[]"); + // The carrier always carries the span's CURRENT projection (design 0.8), never a reset to "[]". + this.span.setAttribute("fi.attachment.refs", JSON.stringify(this.refs)); this.prepared = true; } catch { /* A foreign span must not break application code. */ } } diff --git a/typescript/packages/fi-core/src/attachments/disabled.ts b/typescript/packages/fi-core/src/attachments/disabled.ts new file mode 100644 index 000000000..d49a804e3 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/disabled.ts @@ -0,0 +1,51 @@ +import { AttachmentTicket } from "./admission"; +import { AttachmentCorrelation } from "./correlation"; +import { AttachmentManager } from "./manager"; +import type { + AttachmentAdmissionTicket, AttachmentCompletedRecordingRequest, AttachmentCompletion, AttachmentFlushResult, AttachmentHandle, AttachmentImportHandle, + AttachmentImportOptions, AttachmentManagerOptions, AttachmentOptions, AttachmentOwnedFileRef, AttachmentOwnedRecordingHandle, AttachmentOwnedRecordingOptions, + AttachmentPreflightOptions, AttachmentRecordingOptions, AttachmentRecordingWriter, AttachmentSource, AttachmentSpan, AttachmentStatus, AttachmentStream, + AttachmentStreamOptions, AttachmentTarget, +} from "./types"; + +const DISABLED = Object.freeze({ status: "disabled" as const }) as AttachmentCompletion; +const DONE = Promise.resolve(DISABLED); +const EMPTY: AttachmentFlushResult = Object.freeze({ unfinished: 0, dropped: 0 }); +const STATUS: AttachmentStatus = Object.freeze({ state: "disabled", queued: 0, active: 0, openTickets: 0, openStreams: 0, openRecordings: 0, ownedBytes: 0, dropped: 0 }); +const HANDLE: AttachmentHandle = Object.freeze({ attachmentId: "00000000-0000-0000-0000-000000000000", occurrenceId: "00000000-0000-0000-0000-000000000000", + admission: "disabled", completion: DONE }); +const STREAM: AttachmentStream = Object.freeze({ reason: "disabled" as const, tryWrite: () => false, finish: () => HANDLE, abort: () => undefined }); +const OWNED: AttachmentOwnedRecordingHandle = Object.freeze({ tier: "denied" as const, reason: "disabled" as const, completion: DONE, wait: () => DONE, cancel: () => undefined }); +const IMPORT: AttachmentImportHandle = Object.freeze({ reason: "disabled" as const, completion: DONE, status: async () => ({ state: "failed", reason: "disabled" }) }); +const RESULT = Object.freeze({ recordingId: "00000000-0000-0000-0000-000000000000", state: "denied" as const, reason: "disabled", segmentsPublished: 0, gapMs: 0, + captureComplete: false, manifestRevision: 0 }); +const WRITER: AttachmentRecordingWriter = Object.freeze({ recordingId: RESULT.recordingId, reason: "disabled" as const, tryWrite: () => false, markGap: () => undefined, + markPlayout: () => undefined, finish: async () => RESULT, abort: () => undefined, + status: () => Object.freeze({ state: "terminal" as const, reason: "disabled" as const, sealed: 0, uploaded: 0, gapMs: 0, manifestRevision: 0, ownedBytes: 0 }) }); + +/** + * Attachments OFF (the default): the full public surface, every call returning immediately. It starts no timer, + * socket, process listener, finalizer or bootstrap, writes no span attribute and never touches a source (design 8.1). + */ +export class DisabledAttachmentManager extends AttachmentManager { + constructor(_options: AttachmentManagerOptions = {}) { super({ enabled: false, environment: () => ({}) }); } + async ready(): Promise { return false; } + correlation(spanOrContext?: AttachmentSpan): AttachmentCorrelation { return new AttachmentCorrelation(spanOrContext); } + preflight(_target: AttachmentTarget, _options: AttachmentPreflightOptions): AttachmentAdmissionTicket { + return new AttachmentTicket({ commitTicket: () => HANDLE, commitStream: () => STREAM, releaseReservation: () => undefined, deniedHandle: () => HANDLE, deniedStream: () => STREAM }, + "disabled", 0, undefined, undefined, -1, 0); + } + attach(_target: AttachmentTarget, _source: AttachmentSource, _options: AttachmentOptions): AttachmentHandle { return HANDLE; } + startAttachmentStream(_target: AttachmentTarget, _options: AttachmentStreamOptions): AttachmentStream { return STREAM; } + startRecording(_target: AttachmentTarget, _options: AttachmentRecordingOptions): AttachmentRecordingWriter { return WRITER; } + attachOwnedRecording(_target: AttachmentTarget, _source: { kind: "OwnedRecordingFile"; lease: AttachmentOwnedFileRef }, _options: AttachmentOwnedRecordingOptions): AttachmentOwnedRecordingHandle { return OWNED; } + /** The resolver is NOT called. */ + async importCompletedRecording(_target: AttachmentTarget, _request: AttachmentCompletedRecordingRequest, _options: AttachmentImportOptions): Promise { return IMPORT; } + updatePolicy(): void { /* can never turn a disabled manager on */ } + invalidate(): void { /* nothing to revoke */ } + async flush(): Promise { return EMPTY; } + shutdown(): Promise { return Promise.resolve(EMPTY); } + status(): AttachmentStatus { return STATUS; } +} +/** Same public surface whether attachments are on or off. */ +export type AttachmentManagerLike = AttachmentManager; diff --git a/typescript/packages/fi-core/src/attachments/imports.ts b/typescript/packages/fi-core/src/attachments/imports.ts new file mode 100644 index 000000000..c3b534127 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/imports.ts @@ -0,0 +1,42 @@ +import { validAttachmentUuid } from "./correlation"; +import type { + AttachmentAdmission, AttachmentCompletedRecordingRequest, AttachmentCompletedRecordingResolver, AttachmentDirection, AttachmentResolvedCompletedRecording, +} from "./types"; + +const AUDIO = ["audio/wav", "audio/ogg", "audio/mpeg", "audio/mp4", "audio/webm", "audio/flac", "audio/L16"]; +const TOKEN = /^[a-zA-Z0-9_-]{1,128}$/; +const RESOLVER_REASONS: readonly AttachmentAdmission[] = ["source_unavailable", "policy_denied", "credential_scope_unsupported", "too_large"]; + +/** Core default: the core never talks to providers (design 5.2). */ +export class AttachmentUnavailableResolver implements AttachmentCompletedRecordingResolver { + async resolve(): Promise { return "source_unavailable"; } +} +export function validAttachmentImportRequest(request: AttachmentCompletedRecordingRequest): boolean { + return !!request && typeof request.provider === "string" && TOKEN.test(request.provider) && typeof request.providerRef === "string" && + TOKEN.test(request.providerRef) && (request.direction === undefined || ["input", "output", "mixed"].includes(request.direction)); +} +/** Resolved direction must be at least as inclusive as the request; `mixed` wins (trusted ancestry). */ +export function attachmentImportDirection(requested: AttachmentDirection, resolved: AttachmentDirection): AttachmentDirection | undefined { + if (resolved === requested) return resolved; + if (resolved === "mixed") return "mixed"; + return undefined; +} +/** Validates a resolver result (design 5.3 step 3). Returns a reason, or undefined when valid. */ +export function attachmentResolvedRecordingDenial(value: unknown, limits: { recording_bytes?: number; recording_duration_ms?: number } | undefined): AttachmentAdmission | "unsupported_version" | undefined { + if (typeof value === "string") return RESOLVER_REASONS.includes(value as AttachmentAdmission) ? value as AttachmentAdmission : "source_unavailable"; + const r = value as AttachmentResolvedCompletedRecording; + if (!r || typeof r !== "object" || !r.source || typeof r.source !== "object") return "unsupported_version"; + const s = r.source as Record; + if (s.kind === "ProviderFile") { + if (Object.keys(s).length !== 3 || !validAttachmentUuid(s.connectionId) || !validAttachmentUuid(s.registeredOutputId)) return "unsupported_version"; + } else if (s.kind === "RemoteUrlEnvelope") { + if (Object.keys(s).length !== 2 || !validAttachmentUuid(s.envelopeId)) return "unsupported_version"; + } else return "unsupported_version"; + if (!AUDIO.includes(r.mimeType) || !["input", "output", "mixed"].includes(r.direction)) return "unsupported_version"; + if (!Number.isInteger(r.declaredBytes) || !Number.isInteger(r.declaredDurationMs)) return "unsupported_version"; + const maxBytes = Math.min(limits?.recording_bytes ?? 1073741824, 1073741824); + const maxMs = Math.min(limits?.recording_duration_ms ?? 7200000, 7200000); + // Never truncated: an out-of-range completed source is too_large (F01 import rule 2). + if (r.declaredBytes < 1 || r.declaredBytes > maxBytes || r.declaredDurationMs < 1 || r.declaredDurationMs > maxMs) return "too_large"; + return undefined; +} diff --git a/typescript/packages/fi-core/src/attachments/index.ts b/typescript/packages/fi-core/src/attachments/index.ts index df9af31f5..6fc134ada 100644 --- a/typescript/packages/fi-core/src/attachments/index.ts +++ b/typescript/packages/fi-core/src/attachments/index.ts @@ -1,7 +1,17 @@ /** Browser-safe leaf: no Node filesystem/network imports or ambient credential discovery. */ export { AttachmentManager } from "./manager"; +export { DisabledAttachmentManager } from "./disabled"; +export type { AttachmentManagerLike } from "./disabled"; +export { AttachmentTicket } from "./admission"; +export { AttachmentConsumedStream } from "./stream"; +export { AttachmentRecording, attachmentBoundaryStep, attachmentSegmentFrames, attachmentFramesToMs, attachmentWavHeader, validAttachmentManifest } from "./recording"; +export { AttachmentUnavailableResolver } from "./imports"; +export { validAttachmentWire, ATTACHMENT_WIRE_SHAPES } from "./wire"; export { AttachmentCorrelation, canonicalAttachmentTraceId, attachmentTraceUuid, attachmentMessageUri } from "./correlation"; export { evaluateAttachmentPolicy, attachmentModality } from "./policy"; -export { createAttachmentHttpTransport, attachmentFetchClient, AttachmentTransportError } from "./transport-v1"; -export type { AttachmentHttpClient, AttachmentHttpRequest, AttachmentHttpResponse, AttachmentHttpTransportOptions } from "./transport-v1"; +export { + createAttachmentHttpTransport, createAttachmentHttpTransportV1, attachmentFetchClient, AttachmentTransportError, attachmentHttpError, + attachmentIdempotencyKey, attachmentRetryAfterMs, attachmentStrictJson, attachmentLegacyTransport, ATTACHMENT_TRANSPORT_V1, +} from "./transport-v1"; +export type { AttachmentHttpClient, AttachmentHttpRequest, AttachmentHttpResponse, AttachmentHttpTransportOptions, AttachmentErrorCode } from "./transport-v1"; export type * from "./types"; diff --git a/typescript/packages/fi-core/src/attachments/internal.ts b/typescript/packages/fi-core/src/attachments/internal.ts new file mode 100644 index 000000000..5a152da51 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/internal.ts @@ -0,0 +1,65 @@ +/** @internal Contracts between the manager and its ticket, stream, recording and import helpers. Not public API. */ +import type { + AttachmentAdmission, AttachmentCapturePolicy, AttachmentClock, AttachmentCompletion, AttachmentContentSource, AttachmentDirection, + AttachmentOperation, AttachmentPurpose, AttachmentRuntimeState, AttachmentSession, AttachmentSourceKind, AttachmentUpload, +} from "./types"; + +export interface AdmissionFacts { + mimeType: string; + direction: AttachmentDirection; + purpose?: AttachmentPurpose; + capture?: AttachmentCapturePolicy; + operation: AttachmentOperation; + sourceKind: AttachmentSourceKind; + /** Only the completed-import path may name ProviderFile / RemoteUrl (design 1.4 step 8). */ + importSource?: boolean; +} +export interface CoreJob { + readonly lane: string; + /** Owned bytes released by the manager when this job ends. */ + bytes: number; + readonly controller: AbortController; + run(signal: AbortSignal): Promise; + /** Re-evaluates admission against the current policy epoch; a reason drops the job. */ + denied(): AttachmentAdmission | undefined; + fail(reason: AttachmentCompletion["status"]): void; + settled?: boolean; +} +export interface UploadSpec { + /** Serialized once at admission; replays are byte-identical. */ + body: string; + occurrenceId: string; + declaredBytes: number; + source: AttachmentContentSource; + recheck(): AttachmentAdmission | undefined; + /** Recording segments under BR-7 fallback PUT the manifest between content and complete. */ + beforeComplete?(outcome: UploadOutcome, signal: AbortSignal): Promise; +} +export interface UploadOutcome { + status: AttachmentCompletion["status"]; + upload?: AttachmentUpload; + body: string; + renewals: number; +} +export interface AttachmentHost { + readonly clock: AttachmentClock; + readonly sessionId: string; + readonly nonce: string; + readonly chunkBytes: number; + readonly manifestTimeoutMs: number; + readonly finalizationTimeoutMs: number; + currentSession(): AttachmentSession | undefined; + runtimeState(): AttachmentRuntimeState; + policyEpoch(): number; + stopping(): boolean; + recheck(facts: AdmissionFacts, options?: { ignoreSessionExpiry?: boolean }): AttachmentAdmission | undefined; + reserveBytes(bytes: number): boolean; + releaseBytes(bytes: number): void; + enqueue(job: CoreJob): void; + runUpload(spec: UploadSpec, signal: AbortSignal): Promise; + renewUpload(outcome: UploadOutcome, signal: AbortSignal): Promise; + putManifest(capability: string, recordingId: string, body: string, key: string, signal: AbortSignal, replay: boolean): Promise; + recordingClosed(id: string): void; + countDrop(): void; + notify(): void; +} diff --git a/typescript/packages/fi-core/src/attachments/manager.ts b/typescript/packages/fi-core/src/attachments/manager.ts index e23390829..eb24dc9ad 100644 --- a/typescript/packages/fi-core/src/attachments/manager.ts +++ b/typescript/packages/fi-core/src/attachments/manager.ts @@ -1,64 +1,165 @@ +import { context, trace } from "@opentelemetry/api"; +import { isTracingSuppressed } from "@opentelemetry/core"; import { v4 as uuid } from "uuid"; +import { AttachmentTicket, type TicketHost } from "./admission"; import { AttachmentCorrelation, attachmentTraceUuid, validAttachmentUuid } from "./correlation"; +import { attachmentImportDirection, attachmentResolvedRecordingDenial, validAttachmentImportRequest } from "./imports"; +import type { AdmissionFacts, AttachmentHost, CoreJob, UploadOutcome, UploadSpec } from "./internal"; +import { validAttachmentWire } from "./wire"; +import { attachmentOwnedFile, type AttachmentOwnedFileRecord, type AttachmentOwnedLease } from "./owned"; import { attachmentEnabled, attachmentModality, evaluateAttachmentPolicy } from "./policy"; -import { AttachmentTransportError, validAttachmentSession, validAttachmentUpload } from "./transport-v1"; -import type { AttachmentAdmission, AttachmentCompletion, AttachmentFlushResult, AttachmentHandle, AttachmentManagerOptions, AttachmentOptions, AttachmentReference, AttachmentSession, AttachmentSource, AttachmentSourceKind, AttachmentSpan, AttachmentUpload, AttachmentUploadRequest } from "./types"; +import { + ATTACHMENT_WAV_HEADER_BYTES, AttachmentRecording, attachmentChunkedSource, attachmentRecordingShape, attachmentSegmentFrames, attachmentWavHeader, + type RecordingFileSegment, +} from "./recording"; +import { AttachmentConsumedStream, type StreamHost, type StreamState } from "./stream"; +import { AttachmentTransportError, attachmentIdempotencyKey, attachmentWireTransport, validAttachmentSession, validAttachmentUpload } from "./transport-v1"; +import type { + AttachmentAdmission, AttachmentAdmissionTicket, AttachmentClock, AttachmentCompletedRecordingRequest, AttachmentCompletion, AttachmentContentSource, + AttachmentFields, AttachmentFlushResult, AttachmentHandle, AttachmentImportHandle, AttachmentImportOptions, AttachmentManagerOptions, AttachmentOptions, + AttachmentOwnedFileRef, AttachmentOwnedRecordingHandle, AttachmentOwnedRecordingOptions, AttachmentPacer, AttachmentPreflightOptions, + AttachmentRecordingOptions, AttachmentRecordingWriter, AttachmentReference, AttachmentRuntimeState, AttachmentSession, AttachmentSource, + AttachmentSourceKind, AttachmentSpan, AttachmentStatus, AttachmentStream, AttachmentStreamOptions, AttachmentTarget, AttachmentTransportV1, + AttachmentUploadRequest, +} from "./types"; +const MiB = 1024 * 1024; +const CHUNK = 256 * 1024; // Reserve bounded transfer/read/response scratch for both workers inside the 32 MiB envelope. -const MAX_MEMORY = 32 * 1024 * 1024 - 2 * (256 * 1024 + 2 * 65536); +const MAX_MEMORY = 32 * MiB - 2 * (CHUNK + 2 * 65536); +const SKEW = 5000; +const QUALIFIED = new Set(["Bytes", "Blob", "OwnedRecordingFile"]); +const PURPOSES = ["conversation_recording", "turn_recording", "model_input", "model_output", "tool_result", "computer_screenshot", "diagnostic", "document"]; +const ROLES = ["user", "assistant", "tool", "system", "unknown"]; +const TOKEN = /^[a-zA-Z0-9_-]{1,128}$/; + function bounded(value: number | undefined, fallback: number, max: number): number { return Number.isInteger(value) && value! > 0 ? Math.min(value!, max) : fallback; } function unref(timer: ReturnType): void { (timer as unknown as { unref?: () => void }).unref?.(); } -function handle(admission: AttachmentAdmission): { value: AttachmentHandle; finish: (status: AttachmentCompletion["status"]) => void } { +const defaultClock: AttachmentClock = { + now: () => (typeof performance !== "undefined" && typeof performance.now === "function" ? performance.now() : Date.now()), + wall: () => Date.now(), +}; +type HandleControl = { value: AttachmentHandle; finish: (status: AttachmentCompletion["status"]) => void }; +function handle(admission: AttachmentAdmission): HandleControl { let resolve!: (result: AttachmentCompletion) => void; const completion = new Promise(done => { resolve = done; }); const value = Object.freeze({ attachmentId: uuid(), occurrenceId: uuid(), admission, completion }); - const finish = (status: AttachmentCompletion["status"]) => resolve(Object.freeze({ status })); + let settled = false; + const finish = (status: AttachmentCompletion["status"]) => { if (!settled) { settled = true; resolve(Object.freeze({ status })); } }; if (admission !== "queued") finish(admission); return { value, finish }; } -interface Job { - handle: ReturnType; - data?: Uint8Array | Blob; - size: number; - kind: AttachmentSourceKind; - options: AttachmentOptions; - request: AttachmentUploadRequest; - controller: AbortController; - dropped?: boolean; +function codeOf(error: unknown): Exclude { + if (error instanceof AttachmentTransportError) return error.code === "failed" ? "incomplete" : error.code; + return "incomplete"; +} +function raceAbort(promise: Promise, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(new AttachmentTransportError("incomplete")); + return new Promise((resolve, reject) => { + const abort = () => reject(new AttachmentTransportError("incomplete", false)); + signal.addEventListener("abort", abort, { once: true }); + promise.then(value => { signal.removeEventListener("abort", abort); resolve(value); }, error => { signal.removeEventListener("abort", abort); reject(error); }); + }); } +function pause(ms: number, signal: AbortSignal): Promise { + return new Promise(resolve => { + const done = () => { clearTimeout(timer); signal.removeEventListener("abort", done); resolve(); }; + const timer = setTimeout(done, Math.max(0, ms)); unref(timer); + signal.addEventListener("abort", done, { once: true }); + if (signal.aborted) done(); + }); +} +/** One token bucket per manager at the negotiated upload_bytes_per_second. */ +class TokenBucket implements AttachmentPacer { + private tokens = 0; private last = 0; private started = false; + constructor(private readonly rate: () => number, private readonly clock: AttachmentClock) {} + async take(bytes: number, signal: AbortSignal): Promise { + const rate = this.rate(); + if (!(rate > 0) || rate >= 1073741824) return; + const now = this.clock.now(); + if (!this.started) { this.started = true; this.tokens = rate; this.last = now; } + this.tokens = Math.min(rate, this.tokens + (now - this.last) * rate / 1000); this.last = now; + this.tokens -= bytes; + if (this.tokens < 0) await pause(-this.tokens / rate * 1000, signal); + } +} +class Lane { readonly jobs: CoreJob[] = []; busy = false; constructor(readonly id: string) {} } +interface Reservation { bytes: number; expiresAt: number } +interface StreamExtra { facts: AdmissionFacts; fields: AttachmentFields; correlation: AttachmentCorrelation; purpose?: AttachmentOptions["purpose"] } + /** One per provider/runtime; explicit attachment APIs and adapters use this same instance. */ -export class AttachmentManager { +export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost { + /** @internal */ readonly clock: AttachmentClock; + /** @internal */ readonly sessionId = uuid(); + /** @internal */ readonly nonce = uuid(); + /** @internal */ readonly chunkBytes = CHUNK; + /** @internal */ readonly manifestTimeoutMs: number; + /** @internal */ readonly finalizationTimeoutMs: number; private readonly config: AttachmentManagerOptions; - private readonly sessionId = uuid(); - private readonly nonce = uuid(); + private readonly enabled: boolean; private session?: AttachmentSession; - private bootStatus: AttachmentAdmission = "bootstrap_pending"; - private readonly bootController = new AbortController(); + private sessionLifetime = 0; + private bootStatus: AttachmentRuntimeState = "bootstrap_pending"; + private readonly bootController?: AbortController; private readonly boot: Promise; - private readonly queue: Job[] = []; - private readonly active = new Set(); + private readonly wire?: AttachmentTransportV1; + private readonly lanes = new Map(); + private order: Lane[] = []; + private cursor = 0; + private queued = 0; + private readonly active = new Set(); private readonly correlations = new WeakMap(); private readonly listeners = new Set<() => void>(); + private readonly reservations = new Map(); + private nextReservation = 0; + private readonly streams = new Set(); + private readonly recordings = new Map(); + private recordingSlots = 0; private bytes = 0; private dropped = 0; + private epoch = 0; private closed = false; - private closedReason: AttachmentAdmission = "incomplete"; + private stoppingFlag = false; + private closedReason: AttachmentRuntimeState = "incomplete"; private shutdownPromise?: Promise; + private renewal = 0; + private renewing = false; + private nextRenewalAt = 0; + private renewalFailures = 0; + private registry?: FinalizationRegistry<() => void>; + private readonly pacer: TokenBucket; private readonly maxBytes: number; private readonly maxDescriptors: number; private readonly workers: number; private readonly requestTimeout: number; private readonly shutdownTimeout: number; + private readonly contentTimeout: number; + private readonly syncCopyBytes: number; + private readonly ticketTtl: number; + private readonly streamIdleTtl: number; constructor(options: AttachmentManagerOptions = {}) { this.config = { ...options, traceConfig: { ...options.traceConfig } }; + this.clock = options.clock ?? defaultClock; this.maxBytes = bounded(options.maxOwnedBytes, MAX_MEMORY, MAX_MEMORY); this.maxDescriptors = bounded(options.maxDescriptors, 128, 128); this.workers = bounded(options.workers, 2, 2); this.requestTimeout = bounded(options.requestTimeoutMs, 2000, 30000); this.shutdownTimeout = bounded(options.shutdownTimeoutMs, 5000, 30000); + this.contentTimeout = bounded(options.contentTimeoutMs, 120000, 600000); + this.manifestTimeoutMs = bounded(options.manifestTimeoutMs, 10000, 30000); + this.syncCopyBytes = bounded(options.syncCopyBytes, 8 * MiB, 32 * MiB); + this.ticketTtl = bounded(options.ticketTtlMs, 10000, 30000); + this.streamIdleTtl = bounded(options.streamIdleTtlMs, 60000, 300000); + this.finalizationTimeoutMs = bounded(options.finalizationTimeoutMs, 30000, 30000); + this.pacer = new TokenBucket(() => this.session?.limits.upload_bytes_per_second ?? 0, this.clock); + // Resolved ONCE (design 8.1): enabling later means building a new provider. + this.enabled = attachmentEnabled(this.config, this.env()); + if (!this.enabled) { this.bootStatus = "disabled"; this.boot = Promise.resolve(false); return; } + this.bootController = new AbortController(); + this.wire = attachmentWireTransport(options.transport); this.boot = this.bootstrap(); } private env(): Readonly> { @@ -67,194 +168,926 @@ export class AttachmentManager { } private async bootstrap(): Promise { try { - if (!attachmentEnabled(this.config, this.env())) { this.bootStatus = "disabled"; return false; } - if (!this.config.transport) { this.bootStatus = "endpoint_unconfigured"; return false; } + if (!this.wire) { this.bootStatus = "endpoint_unconfigured"; return false; } if (typeof this.config.projectName !== "string" || !this.config.projectName || this.config.projectName.length > 255 || typeof this.config.projectType !== "string" || !this.config.projectType || this.config.projectType.length > 64 || (this.config.workspaceId !== undefined && !validAttachmentUuid(this.config.workspaceId))) { this.bootStatus = "scope_unresolved"; return false; } const request = { schema_versions: [1] as [1], project_name: this.config.projectName, project_type: this.config.projectType, capture_session_id: this.sessionId, correlation_nonce: this.nonce, ...(this.config.workspaceId ? { workspace_id: this.config.workspaceId } : {}) }; - const session = await this.attempt(signal => this.config.transport!.bootstrap(request, signal), this.bootController.signal); - if (!validAttachmentSession(session, this.sessionId)) throw new AttachmentTransportError("unsupported_version"); + const key = attachmentIdempotencyKey("bootstrap", this.sessionId, 0); + const session = await this.attempt(signal => this.wire!.bootstrap(request, key, signal), this.bootController!.signal); + if (!validAttachmentSession(session, this.sessionId, this.clock.wall())) throw new AttachmentTransportError("unsupported_version"); if (this.config.workspaceId && session.scope.workspace_id !== this.config.workspaceId) throw new AttachmentTransportError("scope_unresolved"); if (this.closed) return false; // Detached bounded wire object: a caller/transport cannot later loosen policy through mutation. this.session = JSON.parse(JSON.stringify(session)); - this.bootStatus = "queued"; + this.sessionLifetime = Math.max(0, Date.parse(session.expires_at) - this.clock.wall()); + this.bootStatus = "ready"; return true; - } catch (error) { this.bootStatus = error instanceof AttachmentTransportError && error.code !== "failed" ? error.code : "source_unavailable"; return false; } + } catch (error) { + const code = error instanceof AttachmentTransportError ? error.code : "source_unavailable"; + this.bootStatus = code === "credential_scope_unsupported" || code === "policy_denied" ? "credential_scope_unsupported" : + code === "scope_unresolved" || code === "not_found" ? "scope_unresolved" : + code === "unsupported_version" || code === "endpoint_unconfigured" ? code : "source_unavailable"; + return false; + } } /** Explicit startup wait; setup and inference do not await it. */ async ready(timeoutMs = 5000): Promise { + if (!this.enabled) return false; return this.deadline(this.boot, bounded(timeoutMs, 5000, 30000), false); } + /** Snapshot (cached per span) for later or late capture. Stamping happens only at an ADMITTED capture (design 0.8). */ correlation(spanOrContext?: AttachmentSpan): AttachmentCorrelation { if (!spanOrContext) return new AttachmentCorrelation(); let result = this.correlations.get(spanOrContext); if (!result) { result = new AttachmentCorrelation(spanOrContext); this.correlations.set(spanOrContext, result); } - if (attachmentEnabled(this.config, this.env())) result.prepare(this.sessionId, this.nonce); return result; } - private denial(kind: AttachmentSourceKind, options: AttachmentOptions): AttachmentAdmission | undefined { + + // --- state and admission ------------------------------------------------------------------------ + /** @internal */ + currentSession(): AttachmentSession | undefined { return this.session; } + /** @internal */ + policyEpoch(): number { return this.epoch; } + /** @internal */ + stopping(): boolean { return this.stoppingFlag || this.closed; } + /** @internal */ + runtimeState(): AttachmentRuntimeState { if (this.closed) return this.closedReason; - const policy = evaluateAttachmentPolicy(this.config, this.config.traceConfig ?? {}, this.session?.policy, kind, options, this.env()); - if (policy === "bootstrap_pending" && this.bootStatus !== "queued") return this.bootStatus; - if (policy) return policy; - if (this.session && Date.parse(this.session.expires_at) <= Date.now()) return "expired"; + if (this.stoppingFlag) return "incomplete"; + if (this.bootStatus === "ready" && this.session && this.clock.wall() >= Date.parse(this.session.expires_at) - SKEW) return "expired"; + return this.bootStatus; + } + /** @internal Design 1.4 steps 1-8 against the CURRENT policy epoch. Never touches a source. */ + recheck(facts: AdmissionFacts, options: { ignoreSessionExpiry?: boolean; admission?: boolean } = {}): AttachmentAdmission | undefined { + const env = this.env(); + if (!this.enabled || !attachmentEnabled(this.config, env)) return "disabled"; + if (this.closed) return this.closedReason as AttachmentAdmission; + if (options.admission && (this.stoppingFlag || this.shutdownPromise)) return "incomplete"; + const state = this.runtimeState(); + if (state === "expired") { if (!options.ignoreSessionExpiry) return "expired"; } + else if (state !== "ready" && !(state === "incomplete" && !options.admission)) return state as AttachmentAdmission; + if (!this.session) return "bootstrap_pending"; + if (options.admission) { try { if (isTracingSuppressed(context.active())) return "policy_denied"; } catch { return "policy_denied"; } } + if (facts.purpose !== undefined && !PURPOSES.includes(facts.purpose)) return "policy_denied"; + const capture = { ...facts.capture, newRecording: facts.operation === "create_recording" || facts.capture?.newRecording === true }; + for (const key of Object.keys(facts.capture ?? {})) { + const value = (facts.capture as Record)[key]; + if (value !== undefined && typeof value !== "boolean") return "policy_denied"; + } + const denied = evaluateAttachmentPolicy(this.config, this.config.traceConfig ?? {}, this.session.policy, facts.sourceKind, + { mimeType: facts.mimeType, direction: facts.direction, purpose: facts.purpose, capture }, env); + if (denied) return denied; + if (!QUALIFIED.has(facts.sourceKind) && !(facts.importSource && (facts.sourceKind === "ProviderFile" || facts.sourceKind === "RemoteUrl"))) return "source_unavailable"; return undefined; } - attach(spanOrContext: AttachmentSpan | AttachmentCorrelation, source: AttachmentSource, options: AttachmentOptions): AttachmentHandle { + private resolveCorrelation(target: AttachmentTarget): AttachmentCorrelation | undefined { + try { + if (target instanceof AttachmentCorrelation) return target; + if (target === undefined) { + // The current span is read ONCE at call time. + const span = trace.getSpan(context.active()); + return span ? this.correlation(span) : undefined; + } + return this.correlation(target); + } catch { return undefined; } + } + private descriptors(): number { return this.queued + this.active.size + this.reservations.size + this.streams.size + this.recordings.size; } + private uploadCap(mimeType: string, sourceKind: AttachmentSourceKind): number { + const limit = Math.min(this.session?.limits.max_upload_bytes ?? 0, 50 * MiB); + const kind = attachmentModality(mimeType) === "image" ? Math.min(limit, 20 * MiB) : limit; + return sourceKind === "OwnedRecordingFile" ? kind : Math.min(kind, this.maxBytes); + } + /** @internal */ + reserveBytes(bytes: number): boolean { + if (!(bytes >= 0) || this.bytes + bytes > this.maxBytes) return false; + this.bytes += bytes; return true; + } + /** @internal */ + releaseBytes(bytes: number): void { if (bytes > 0) this.bytes = Math.max(0, this.bytes - bytes); } + /** @internal */ + countDrop(): void { this.dropped++; } + /** @internal */ + notify(): void { for (const notify of this.listeners) notify(); } + /** @internal */ + recordingClosed(id: string): void { this.recordings.delete(id); this.notify(); } + private validFields(fields: AttachmentFields): boolean { + if (fields.role !== undefined && !ROLES.includes(fields.role)) return false; + for (const [value, max] of [[fields.messageIndex, 65535], [fields.partIndex, 65535], [fields.channel, 31]] as const) { + if (value !== undefined && (!Number.isInteger(value) || value < 0 || value > max)) return false; + } + return [fields.conversationId, fields.turnId, fields.toolCallId].every(value => + value === undefined || typeof value === "string" && value.trim() === value && TOKEN.test(value)); + } + private validOptions(options: AttachmentOptions): boolean { + if (options.purpose !== undefined && !PURPOSES.includes(options.purpose)) return false; + return this.validFields(options); + } + private finalizer(): FinalizationRegistry<() => void> | undefined { + if (!this.registry && typeof FinalizationRegistry !== "undefined") this.registry = new FinalizationRegistry(release => { try { release(); } catch { /* bounded */ } }); + return this.registry; + } + /** Lazy sweep at every entry point: no timers are armed for tickets or streams (design 1.5, 6.3). */ + private sweep(): void { + const now = this.clock.now(); + for (const [id, reservation] of this.reservations) if (reservation.expiresAt <= now) this.releaseReservation(id); + for (const state of this.streams) if (state.open && now - state.lastWrite > state.idleTtl) this.streamAbort(state, "incomplete"); + } + private maybeRenew(): void { + const session = this.session; + if (!session || this.renewing || this.closed || this.stoppingFlag || this.bootStatus !== "ready" || !this.wire) return; + const now = this.clock.wall(); const expires = Date.parse(session.expires_at); + if (expires - now > Math.max(60000, 0.2 * this.sessionLifetime)) return; + if (this.clock.now() < this.nextRenewalAt) return; + this.renewing = true; + const sequence = this.renewal + 1; + void (async () => { + try { + const next = await this.attempt(signal => this.wire!.renewSession(session, sequence, signal), this.bootController!.signal, () => undefined, 3); + if (!validAttachmentSession(next, this.sessionId, this.clock.wall())) throw new AttachmentTransportError("unsupported_version"); + const scope = (s: AttachmentSession) => `${s.scope.organization_id}/${s.scope.workspace_id}/${s.scope.project_id}`; + if (scope(next) !== scope(session)) { this.invalidate("scope_unresolved" as "deleted"); return; } + if (this.session !== session || this.closed) return; + const changed = JSON.stringify(next.policy) !== JSON.stringify(session.policy); + this.renewal = sequence; + this.session = JSON.parse(JSON.stringify(next)); + this.sessionLifetime = Math.max(0, Date.parse(next.expires_at) - this.clock.wall()); + this.renewalFailures = 0; this.nextRenewalAt = this.clock.now() + 30000; + if (changed) { this.epoch++; this.cancelDenied(); } + } catch { + this.renewalFailures++; + this.nextRenewalAt = this.clock.now() + Math.min(30000, 1000 * 2 ** Math.min(this.renewalFailures, 5)); + } finally { this.renewing = false; } + })(); + } + private enter(): void { this.sweep(); this.maybeRenew(); } + private preflightFacts(options: AttachmentPreflightOptions, importSource = false): AdmissionFacts { + return { mimeType: options.mimeType, direction: options.direction, purpose: options.purpose, + capture: options.capture ? { ...options.capture } : undefined, operation: options.operation ?? "observe_consumed", + sourceKind: options.sourceKind ?? "Bytes", importSource }; + } + private admitTarget(target: AttachmentTarget, facts: AdmissionFacts): { reason?: AttachmentAdmission; correlation?: AttachmentCorrelation; identity?: { traceId: string; spanId: string } } { + if (typeof facts.mimeType !== "string" || !["input", "output", "mixed"].includes(facts.direction) || + !["observe_consumed", "import_existing", "create_recording"].includes(facts.operation)) { + const first = this.recheck(facts, { admission: true }); + return { reason: first ?? "policy_denied" }; + } + const reason = this.recheck(facts, { admission: true }); + if (reason) return { reason }; + const correlation = this.resolveCorrelation(target); + const identity = correlation?.snapshot(); + if (!correlation || !identity) return { reason: "correlation_unavailable" }; + if (!identity.sampled) return { reason: "unsampled" }; + return { correlation, identity }; + } + /** @internal */ + deniedHandle(reason: AttachmentAdmission): AttachmentHandle { return handle(reason).value; } + /** @internal */ + deniedStream(reason: AttachmentAdmission): AttachmentStream { + const denied = handle(reason).value; + return new AttachmentConsumedStream(undefined, { reason, chunks: [], length: 0, maxBytes: 0, reserved: 0, lastWrite: 0, idleTtl: 0, epoch: 0, open: false, handle: denied }); + } + + /** + * Full deny-wins check BEFORE any decode, read, copy, file open, provider call or stream observer (design section 1). + * Returns a single-use ticket holding a descriptor and a byte budget. + */ + preflight(target: AttachmentTarget, options: AttachmentPreflightOptions): AttachmentAdmissionTicket { + const denied = (reason: AttachmentAdmission) => new AttachmentTicket(this, reason, 0, undefined, undefined, -1, 0); + try { + this.enter(); + const facts = this.preflightFacts(options ?? ({} as AttachmentPreflightOptions)); + const { reason, correlation } = this.admitTarget(target, facts); + if (reason) return denied(reason); + const maxBytes = options.maxBytes; + if (!Number.isInteger(maxBytes) || maxBytes <= 0 || maxBytes > this.uploadCap(facts.mimeType, facts.sourceKind)) return denied("too_large"); + const reserve = facts.sourceKind === "OwnedRecordingFile" ? CHUNK : maxBytes; + if (this.descriptors() >= this.maxDescriptors || this.bytes + reserve > this.maxBytes) return denied("queue_full"); + const ttl = bounded(options.ttlMs, this.ticketTtl, 30000); + const id = ++this.nextReservation; const expiresAt = this.clock.now() + ttl; + this.bytes += reserve; this.reservations.set(id, { bytes: reserve, expiresAt }); + const ticket = new AttachmentTicket(this, "queued", maxBytes, correlation, facts, id, expiresAt); + this.finalizer()?.register(ticket, () => this.releaseReservation(id), ticket); + return ticket; + } catch { return denied("source_unavailable"); } + } + /** @internal */ + releaseReservation(id: number): void { + const reservation = this.reservations.get(id); + if (!reservation) return; + this.reservations.delete(id); this.releaseBytes(reservation.bytes); this.notify(); + } + private takeTicket(ticket: AttachmentTicket): { reason?: AttachmentAdmission; reservation?: Reservation } { + ticket.used = true; + try { this.registry?.unregister(ticket); } catch { /* bounded */ } + this.sweep(); + const reservation = this.reservations.get(ticket.reservation); + if (!reservation || this.clock.now() >= ticket.expiresAt) { this.releaseReservation(ticket.reservation); return { reason: "expired" }; } + // Commit re-check against the CURRENT policy epoch; tightening denies without reading the source. + const reason = this.recheck(ticket.facts!, { admission: true }); + if (reason) { this.releaseReservation(ticket.reservation); return { reason }; } + this.reservations.delete(ticket.reservation); + return { reservation }; + } + /** @internal */ + commitTicket(ticket: AttachmentTicket, source: Extract, fields: AttachmentFields): AttachmentHandle { + const taken = this.takeTicket(ticket); + if (taken.reason) return handle(taken.reason).value; + const reserved = taken.reservation!.bytes; + const fail = (reason: AttachmentAdmission) => { this.releaseBytes(reserved); this.notify(); return handle(reason).value; }; + const facts = ticket.facts!; + if (!source || source.kind !== facts.sourceKind) return fail("conflict" as AttachmentAdmission); + if (!this.validFields(fields)) return fail("policy_denied"); + const read = this.readSource(source, ticket.maxBytes); + if ("reason" in read) return fail(read.reason); + const keep = facts.sourceKind === "OwnedRecordingFile" ? reserved : read.size; + this.releaseBytes(reserved - keep); // the reservation shrinks to the actual length + return this.enqueueUpload(facts, fields, ticket.correlation!, read.size, read.content, keep, read.owned); + } + private readSource(source: AttachmentSource, maxBytes: number): { reason: AttachmentAdmission } | { size: number; content?: AttachmentContentSource; owned?: AttachmentOwnedFileRecord } { + if (source.kind === "Bytes") { + const data = source.data; + if (!(data instanceof Uint8Array)) return { reason: "source_unavailable" }; + const size = data.byteLength; + if (!size || size > maxBytes || size > this.syncCopyBytes) return { reason: "too_large" }; + // Mutable inputs are detached synchronously, after all admission and budget checks. + return { size, content: new Uint8Array(data) }; + } + if (source.kind === "Blob") { + const data = source.data; + if (typeof Blob === "undefined" || !(data instanceof Blob)) return { reason: "source_unavailable" }; + if (!data.size || data.size > maxBytes) return { reason: "too_large" }; + return { size: data.size, content: data }; + } + if (source.kind === "OwnedRecordingFile") { + const record = attachmentOwnedFile(source.lease); + if (!record || record.closed()) return { reason: "source_unavailable" }; + const size = record.identity.size; + if (!size || size > maxBytes) return { reason: "too_large" }; + return { size, owned: record }; + } + return { reason: "source_unavailable" }; + } + private uploadRequest(facts: AdmissionFacts, fields: AttachmentFields, identity: { traceId: string; spanId: string }, ids: { attachmentId: string; occurrenceId: string }, size: number): AttachmentUploadRequest { + return { schema_version: 1, attachment_id: ids.attachmentId, occurrence_id: ids.occurrenceId, + association: { capture_session_id: this.sessionId, correlation_nonce: this.nonce, parent: { trace_id: attachmentTraceUuid(identity.traceId), span_id: identity.spanId } }, + policy_version: this.session!.policy.version, deletion_generation: 0, mime_type: facts.mimeType, + modality: attachmentModality(facts.mimeType)!, direction: facts.direction, source_kind: facts.sourceKind, declared_bytes: size, + ...(facts.purpose ? { purpose: facts.purpose } : {}), + ...(fields.messageIndex === undefined ? {} : { message_index: fields.messageIndex }), + ...(fields.partIndex === undefined ? {} : { part_index: fields.partIndex }), + ...(fields.conversationId ? { conversation_id: fields.conversationId } : {}), + ...(fields.turnId ? { turn_id: fields.turnId } : {}), + ...(fields.toolCallId ? { tool_call_id: fields.toolCallId } : {}) }; + } + private reference(request: AttachmentUploadRequest, fields: AttachmentFields): AttachmentReference { + // role and channel are display hints on the live span only; UploadRequest never carries them (design 7). + return { attachment_id: request.attachment_id, occurrence_id: request.occurrence_id, direction: request.direction, modality: request.modality, + mime_type: request.mime_type, ...(request.purpose ? { purpose: request.purpose } : {}), + ...(fields.messageIndex === undefined ? {} : { message_index: fields.messageIndex }), + ...(fields.partIndex === undefined ? {} : { part_index: fields.partIndex }), + ...(fields.role ? { role: fields.role } : {}), ...(fields.channel === undefined ? {} : { channel: fields.channel }), + ...(fields.conversationId ? { conversation_id: fields.conversationId } : {}), + ...(fields.turnId ? { turn_id: fields.turnId } : {}), + ...(fields.toolCallId ? { tool_call_id: fields.toolCallId } : {}) }; + } + private enqueueUpload(facts: AdmissionFacts, fields: AttachmentFields, correlation: AttachmentCorrelation, size: number, + content: AttachmentContentSource | undefined, bytes: number, owned?: AttachmentOwnedFileRecord, project = true): AttachmentHandle { + const result = handle("queued"); + const identity = correlation.snapshot()!; + const request = this.uploadRequest(facts, fields, identity, result.value, size); + const body = JSON.stringify(request); + const controller = new AbortController(); + let data = content; + const job: CoreJob = { + lane: request.occurrence_id, bytes, controller, + denied: () => this.recheck(facts), + fail: reason => { data = undefined; result.finish(reason); }, + run: async signal => { + let lease: AttachmentOwnedLease | undefined; + try { + let source = data; + if (owned) { + lease = await raceAbort(owned.open(), signal); + const opened = lease; + source = { size: opened.size, read: (offset, length) => opened.read(offset, length), check: () => opened.check() }; + } + if (!source) { result.finish("incomplete"); return; } + const outcome = await this.runUpload({ body, occurrenceId: request.occurrence_id, declaredBytes: size, source, recheck: () => this.recheck(facts, { ignoreSessionExpiry: true }) }, signal); + result.finish(outcome.status); + if (outcome.status !== "validating") this.dropped++; + } catch (error) { result.finish(codeOf(error)); this.dropped++; } + finally { data = undefined; if (lease) await lease.close().catch(() => undefined); } + }, + }; + this.enqueue(job); + if (project) correlation.project(this.reference(request, fields), this.sessionId, this.nonce); + return result.value; + } + attach(spanOrContext: AttachmentTarget, source: AttachmentSource, options: AttachmentOptions): AttachmentHandle { try { if (this.shutdownPromise) return handle("incomplete").value; + this.enter(); // No access to source.data/path/url until the complete privacy gate passes. const kind = source.kind; - const denied = this.denial(kind, options); - if (denied) return handle(denied).value; - const correlation = spanOrContext instanceof AttachmentCorrelation ? spanOrContext : this.correlation(spanOrContext); - const identity = correlation.snapshot(); - if (!identity) return handle("correlation_unavailable").value; - if (!identity.sampled) return handle("unsampled").value; + const facts: AdmissionFacts = { mimeType: options.mimeType, direction: options.direction, purpose: options.purpose, + capture: options.capture ? { ...options.capture } : undefined, operation: options.capture?.newRecording ? "create_recording" : "observe_consumed", sourceKind: kind }; + const { reason, correlation } = this.admitTarget(spanOrContext, facts); + if (reason) return handle(reason).value; if (!this.validOptions(options)) return handle("policy_denied").value; - if (kind !== "Bytes" && kind !== "Blob") return handle("source_unavailable").value; - if (this.queue.length + this.active.size >= this.maxDescriptors) return handle("queue_full").value; - const data = source.data; - const bytes = kind === "Bytes"; - if (bytes ? !(data instanceof Uint8Array) : typeof Blob === "undefined" || !(data instanceof Blob)) return handle("source_unavailable").value; - const size = bytes ? (data as Uint8Array).byteLength : (data as Blob).size; - const modality = attachmentModality(options.mimeType)!; - const cap = modality === "image" ? 20 * 1024 * 1024 : modality === "audio" || modality === "video" ? 8 * 1024 * 1024 : 50 * 1024 * 1024; - if (!size || size > Math.min(cap, this.session!.limits.max_upload_bytes, this.maxBytes)) return handle("too_large").value; - if (this.queue.length + this.active.size >= this.maxDescriptors || this.bytes + size > this.maxBytes) return handle("queue_full").value; - // Mutable inputs are detached synchronously, after all admission and budget checks. - const owned = bytes ? new Uint8Array(data as Uint8Array) : data as Blob; - const result = handle("queued"); - const snapshot: AttachmentOptions = { mimeType: options.mimeType, direction: options.direction, purpose: options.purpose, - messageIndex: options.messageIndex, partIndex: options.partIndex, role: options.role, channel: options.channel, - conversationId: options.conversationId, turnId: options.turnId, toolCallId: options.toolCallId, - capture: { captureContent: options.capture?.captureContent, recordInputs: options.capture?.recordInputs, recordOutputs: options.capture?.recordOutputs, - muted: options.capture?.muted, temporaryChat: options.capture?.temporaryChat, requested: options.capture?.requested, - modelInput: options.capture?.modelInput, containsInput: options.capture?.containsInput, containsOutput: options.capture?.containsOutput, - errorDetail: options.capture?.errorDetail, newRecording: options.capture?.newRecording } }; - const request: AttachmentUploadRequest = { schema_version: 1, attachment_id: result.value.attachmentId, occurrence_id: result.value.occurrenceId, - association: { capture_session_id: this.sessionId, correlation_nonce: this.nonce, parent: { trace_id: attachmentTraceUuid(identity.traceId), span_id: identity.spanId } }, - policy_version: this.session!.policy.version, deletion_generation: 0, mime_type: snapshot.mimeType, - modality, direction: snapshot.direction, source_kind: kind, declared_bytes: size, - ...(snapshot.purpose ? { purpose: snapshot.purpose } : {}), - ...(snapshot.messageIndex === undefined ? {} : { message_index: snapshot.messageIndex }), - ...(snapshot.partIndex === undefined ? {} : { part_index: snapshot.partIndex }), - ...(snapshot.conversationId ? { conversation_id: snapshot.conversationId } : {}), - ...(snapshot.turnId ? { turn_id: snapshot.turnId } : {}), - ...(snapshot.toolCallId ? { tool_call_id: snapshot.toolCallId } : {}) }; - const job: Job = { handle: result, data: owned, size, kind, options: snapshot, request, controller: new AbortController() }; - this.bytes += size; - this.queue.push(job); - const reference: AttachmentReference = { attachment_id: result.value.attachmentId, occurrence_id: result.value.occurrenceId, - direction: snapshot.direction, modality, mime_type: snapshot.mimeType, ...(snapshot.purpose ? { purpose: snapshot.purpose } : {}), - ...(snapshot.messageIndex === undefined ? {} : { message_index: snapshot.messageIndex }), - ...(snapshot.partIndex === undefined ? {} : { part_index: snapshot.partIndex }), - ...(snapshot.role ? { role: snapshot.role } : {}), ...(snapshot.channel === undefined ? {} : { channel: snapshot.channel }), - ...(snapshot.conversationId ? { conversation_id: snapshot.conversationId } : {}), - ...(snapshot.turnId ? { turn_id: snapshot.turnId } : {}), - ...(snapshot.toolCallId ? { tool_call_id: snapshot.toolCallId } : {}) }; - correlation.project(reference, this.sessionId, this.nonce); - // Avoid even synchronous user transport execution in the inference callback. - void Promise.resolve().then(() => this.pump()); - return result.value; + if (this.descriptors() >= this.maxDescriptors) return handle("queue_full").value; + const read = this.readSource(source, this.uploadCap(options.mimeType, kind)); + if ("reason" in read) return handle(read.reason).value; + const reserve = kind === "OwnedRecordingFile" ? CHUNK : read.size; + if (this.descriptors() >= this.maxDescriptors || this.bytes + reserve > this.maxBytes) return handle("queue_full").value; + this.bytes += reserve; + const fields: AttachmentFields = { messageIndex: options.messageIndex, partIndex: options.partIndex, role: options.role, channel: options.channel, + conversationId: options.conversationId, turnId: options.turnId, toolCallId: options.toolCallId }; + return this.enqueueUpload(facts, fields, correlation!, read.size, read.content, reserve, read.owned); } catch { return handle("source_unavailable").value; } } - private validOptions(options: AttachmentOptions): boolean { - const purposes = ["conversation_recording", "turn_recording", "model_input", "model_output", "tool_result", "computer_screenshot", "diagnostic", "document"]; - if (options.purpose !== undefined && !purposes.includes(options.purpose)) return false; - if (options.role !== undefined && !["user", "assistant", "tool", "system", "unknown"].includes(options.role)) return false; - for (const [value, max] of [[options.messageIndex, 65535], [options.partIndex, 65535], [options.channel, 31]] as const) { - if (value !== undefined && (!Number.isInteger(value) || value < 0 || value > max)) return false; + + // --- consumed streams ------------------------------------------------------------------------------ + startAttachmentStream(target: AttachmentTarget, options: AttachmentStreamOptions): AttachmentStream { + try { + const ticket = this.preflight(target, { mimeType: options.mimeType, direction: options.direction, maxBytes: options.maxBytes, + purpose: options.purpose, capture: options.capture, operation: "observe_consumed", sourceKind: "Bytes" }); + return ticket.startStream(options); + } catch { return this.deniedStream("source_unavailable"); } + } + /** @internal */ + commitStream(ticket: AttachmentTicket, options: AttachmentStreamOptions): AttachmentStream { + const taken = this.takeTicket(ticket); + if (taken.reason) return this.deniedStream(taken.reason); + const reserved = taken.reservation!.bytes; + const facts = ticket.facts!; + const fail = (reason: AttachmentAdmission) => { this.releaseBytes(reserved); this.notify(); return this.deniedStream(reason); }; + // Narrowing only: a stream cannot change MIME, direction, purpose or source kind, nor exceed the ticket budget. + if (!options || options.mimeType !== facts.mimeType || options.direction !== facts.direction || options.purpose !== facts.purpose || facts.sourceKind !== "Bytes") return fail("conflict" as AttachmentAdmission); + if (!Number.isInteger(options.maxBytes) || options.maxBytes <= 0 || options.maxBytes > ticket.maxBytes) return fail("too_large"); + const fields: AttachmentFields = { messageIndex: options.messageIndex, partIndex: options.partIndex, role: options.role, channel: options.channel, + conversationId: options.conversationId, turnId: options.turnId, toolCallId: options.toolCallId }; + if (!this.validFields(fields)) return fail("policy_denied"); + const keep = options.maxBytes; this.releaseBytes(reserved - keep); + const state: StreamState & StreamExtra = { reason: "queued", attachmentId: uuid(), occurrenceId: uuid(), chunks: [], length: 0, maxBytes: options.maxBytes, + reserved: keep, lastWrite: this.clock.now(), idleTtl: bounded(options.idleTtlMs, this.streamIdleTtl, 300000), epoch: this.epoch, open: true, + facts, fields, correlation: ticket.correlation!, purpose: options.purpose }; + this.streams.add(state); + const stream = new AttachmentConsumedStream(this, state); + this.finalizer()?.register(stream, () => { if (state.open) this.streamAbort(state, "incomplete"); }, state); + return stream; + } + /** @internal */ + streamWrite(state: StreamState, chunk: Uint8Array): boolean { + const s = state as StreamState & StreamExtra; + if (!s.open || !(chunk instanceof Uint8Array)) return false; + if (s.epoch !== this.epoch || this.closed) { + const reason = this.recheck(s.facts); + if (reason) { this.streamAbort(s, "policy_denied"); return false; } } - return [options.conversationId, options.turnId, options.toolCallId].every(value => - value === undefined || typeof value === "string" && value.trim() === value && /^[a-zA-Z0-9_-]{1,128}$/.test(value)); + if (s.length + chunk.byteLength > s.maxBytes) { this.streamAbort(s, "too_large"); return false; } + // Copy in bounded slices; the caller may reuse a BYOB buffer immediately. + for (let offset = 0; offset < chunk.byteLength; offset += CHUNK) s.chunks.push(chunk.slice(offset, Math.min(chunk.byteLength, offset + CHUNK))); + s.length += chunk.byteLength; s.lastWrite = this.clock.now(); + return true; } - private pump(): void { - while (this.active.size < this.workers && this.queue.length) { - const job = this.queue.shift()!; - this.active.add(job); - void this.transfer(job).finally(() => { - job.data = undefined; this.bytes -= job.size; this.active.delete(job); - this.notify(); this.pump(); - }); + /** @internal */ + streamFinish(state: StreamState): AttachmentHandle { + const s = state as StreamState & StreamExtra; + if (!s.open) return s.handle ?? handle(s.reason === "queued" ? "incomplete" : s.reason).value; + s.open = false; this.streams.delete(s); + try { this.registry?.unregister(s); } catch { /* bounded */ } + const chunks = s.chunks; s.chunks = []; + const release = (reason: AttachmentAdmission) => { this.releaseBytes(s.reserved); s.reserved = 0; s.reason = reason; s.handle = handle(reason).value; this.notify(); return s.handle; }; + if (!s.length) return release("incomplete"); + const reason = this.recheck(s.facts); + if (reason) return release(reason); + this.releaseBytes(s.reserved - s.length); const bytes = s.length; s.reserved = 0; + const content = attachmentChunkedSource(new Uint8Array(0), chunks); + s.handle = this.enqueueUploadWithIds(s, content, bytes); + return s.handle; + } + private enqueueUploadWithIds(s: StreamState & StreamExtra, content: AttachmentContentSource, bytes: number): AttachmentHandle { + const result = handle("queued"); + const ids = { attachmentId: s.attachmentId!, occurrenceId: s.occurrenceId! }; + const identity = s.correlation.snapshot()!; + const request = this.uploadRequest(s.facts, s.fields, identity, ids, bytes); + const body = JSON.stringify(request); + let data: AttachmentContentSource | undefined = content; + const value = Object.freeze({ attachmentId: ids.attachmentId, occurrenceId: ids.occurrenceId, admission: "queued" as AttachmentAdmission, completion: result.value.completion }); + this.enqueue({ lane: ids.occurrenceId, bytes, controller: new AbortController(), denied: () => this.recheck(s.facts), + fail: reason => { data = undefined; result.finish(reason); }, + run: async signal => { + try { + if (!data) { result.finish("incomplete"); return; } + const outcome = await this.runUpload({ body, occurrenceId: ids.occurrenceId, declaredBytes: bytes, source: data, recheck: () => this.recheck(s.facts, { ignoreSessionExpiry: true }) }, signal); + result.finish(outcome.status); if (outcome.status !== "validating") this.dropped++; + } finally { data = undefined; } + } }); + // Projected only at finish, only within the cap, only while the span still records. + s.correlation.project(this.reference(request, s.fields), this.sessionId, this.nonce); + return value; + } + /** @internal */ + streamAbort(state: StreamState, reason: "incomplete" | "policy_denied" | "too_large"): void { + if (!state.open) return; + state.open = false; this.streams.delete(state as StreamState & StreamExtra); + try { this.registry?.unregister(state); } catch { /* bounded */ } + state.chunks = []; state.length = 0; + this.releaseBytes(state.reserved); state.reserved = 0; + state.reason = reason; state.handle = handle(reason).value; + this.notify(); + } + + // --- recordings ------------------------------------------------------------------------------------ + private recordingFacts(options: AttachmentRecordingOptions, direction: "input" | "output" | "mixed"): AdmissionFacts { + return { mimeType: "audio/wav", direction, purpose: options.purpose ?? "conversation_recording", + capture: options.capture ? { ...options.capture } : undefined, operation: options.operation, sourceKind: "Bytes" }; + } + private trackDenials(options: AttachmentRecordingOptions, admission = true): { reason?: AttachmentAdmission; tracks?: (AttachmentAdmission | undefined)[] } { + if (options.layout !== "separate_tracks") { + const reason = this.recheck(this.recordingFacts(options, "mixed"), { admission }); + return reason ? { reason } : { tracks: [] }; } + const tracks = options.tracks.map(track => this.recheck(this.recordingFacts(options, track.direction), { admission })); + if (tracks.every(Boolean)) return { reason: tracks[0] }; + return { tracks }; } - private check(job: Job): void { - const denied = job.controller.signal.aborted ? "incomplete" : this.denial(job.kind, job.options); - if (denied) throw new AttachmentTransportError(denied); - if (job.request.policy_version !== this.session!.policy.version) throw new AttachmentTransportError("policy_denied"); + private recordingLimit(): number { return Math.min(this.session?.limits.active_recordings ?? 4, 4); } + startRecording(target: AttachmentTarget, options: AttachmentRecordingOptions): AttachmentRecordingWriter { + const inert = (reason: AttachmentAdmission) => new AttachmentRecording({ host: this, options, admission: reason, readmit: () => ({ reason }), stamp: () => undefined, close: () => undefined }); + try { + this.enter(); + if (!attachmentRecordingShape(options)) { + const first = this.recheck(this.recordingFacts(options ?? ({} as AttachmentRecordingOptions), "mixed"), { admission: true }); + return inert(first ?? "policy_denied"); + } + const pending = this.enabled && attachmentEnabled(this.config, this.env()) && !this.closed && !this.stoppingFlag && this.runtimeState() === "bootstrap_pending"; + let trackDenials: (AttachmentAdmission | undefined)[] | undefined; + if (!pending) { + const evaluated = this.trackDenials(options); + if (evaluated.reason) return inert(evaluated.reason); + trackDenials = evaluated.tracks; + } + const correlation = this.resolveCorrelation(target); + const identity = correlation?.snapshot(); + if (!correlation || !identity) return inert("correlation_unavailable"); + if (!identity.sampled) return inert("unsampled"); + if (this.descriptors() >= this.maxDescriptors) return inert("queue_full"); + if (!pending && this.recordingSlots >= this.recordingLimit()) return inert("queue_full"); + let slot = false; + const take = () => { if (!slot) { slot = true; this.recordingSlots++; } }; + if (!pending) take(); + const writer: AttachmentRecording = new AttachmentRecording({ host: this, options, identity, admission: pending ? "bootstrap_pending" : "queued", trackDenials, + readmit: () => { + const again = this.trackDenials(options, false); + if (again.reason) return { reason: again.reason }; + if (!slot && this.recordingSlots >= this.recordingLimit()) return { reason: "queue_full" }; + take(); + return { reason: "queued", trackDenials: again.tracks }; + }, + stamp: () => correlation.prepare(this.sessionId, this.nonce), + close: () => { if (slot) { slot = false; this.recordingSlots--; } } }); + if (writer.reason === "queued" || writer.reason === "bootstrap_pending") this.recordings.set(writer.recordingId, writer); + return writer; + } catch { return inert("source_unavailable"); } } - private async transfer(job: Job): Promise { + + // --- owned recording files --------------------------------------------------------------------------- + attachOwnedRecording(target: AttachmentTarget, source: { kind: "OwnedRecordingFile"; lease: AttachmentOwnedFileRef }, options: AttachmentOwnedRecordingOptions): AttachmentOwnedRecordingHandle { + const denied = (reason: AttachmentAdmission): AttachmentOwnedRecordingHandle => { + const completion = Promise.resolve(Object.freeze({ status: reason }) as AttachmentCompletion); + return Object.freeze({ tier: "denied" as const, reason, completion, wait: () => completion, cancel: () => undefined }); + }; try { - const transport = this.config.transport!; - const guard = () => this.check(job); - guard(); - const upload = await this.attempt(signal => transport.initialize(this.session!, job.request, signal), job.controller.signal, guard); - if (!validAttachmentUpload(upload, job.request)) throw new AttachmentTransportError("failed"); - const checkUpload = () => { guard(); if (Date.parse(upload.expires_at) <= Date.now()) throw new AttachmentTransportError("expired"); }; - // v1 cannot safely replay a partially consumed PUT. Only idempotent control operations retry. - await this.attempt(signal => transport.upload(this.session!, upload, job.data!, signal), job.controller.signal, checkUpload, 1); - await this.attempt(signal => transport.complete(this.session!, upload, job.request, signal), job.controller.signal, checkUpload); - checkUpload(); job.handle.finish("validating"); - } catch (error) { - this.drop(job, error instanceof AttachmentTransportError ? error.code : "failed"); + this.enter(); + const record = source && source.kind === "OwnedRecordingFile" ? attachmentOwnedFile(source.lease) : undefined; + const purpose = options?.purpose ?? "conversation_recording"; + const facts: AdmissionFacts = { mimeType: options?.mimeType, direction: options?.direction, purpose, capture: options?.capture ? { ...options.capture } : undefined, + operation: "import_existing", sourceKind: "OwnedRecordingFile" }; + const { reason, correlation, identity } = this.admitTarget(target, facts); + if (reason) return denied(reason); + if (!record || record.closed()) return denied("source_unavailable"); + if (purpose !== "conversation_recording" && purpose !== "turn_recording") return denied("policy_denied"); + if (options.conversationId !== undefined && !TOKEN.test(options.conversationId)) return denied("policy_denied"); + const size = record.identity.size; + const singleCap = Math.min(this.session!.limits.max_upload_bytes, 50 * MiB); + const wav = options.mimeType === "audio/wav" && (size > singleCap || !!options.wavTracks); + if (!Number.isInteger(size) || size <= 0 || (!wav && size > singleCap)) return denied("too_large"); + if (this.descriptors() >= this.maxDescriptors || this.bytes + CHUNK > this.maxBytes) return denied("queue_full"); + if (wav && options.wavTracks && !this.validWavTracks(options.wavTracks)) return denied("policy_denied"); + this.bytes += CHUNK; + return this.ownedTransfer(record, facts, correlation!, identity!, options, wav, singleCap); + } catch { return denied("source_unavailable"); } + } + private validWavTracks(tracks: readonly { channels: readonly number[]; direction: string; role: string }[]): boolean { + return Array.isArray(tracks) && tracks.length >= 1 && tracks.length <= 32 && tracks.every(t => Array.isArray(t.channels) && t.channels.length >= 1 && + ["input", "output", "mixed"].includes(t.direction) && ["user", "assistant", "mixed", "unknown"].includes(t.role)); + } + private ownedTransfer(record: AttachmentOwnedFileRecord, facts: AdmissionFacts, correlation: AttachmentCorrelation, identity: { traceId: string; spanId: string }, + options: AttachmentOwnedRecordingOptions, wav: boolean, singleCap: number): AttachmentOwnedRecordingHandle { + let tier: "single" | "wav_recording" | "denied" = wav ? "wav_recording" : "single"; + let recordingId: string | undefined; + let resolve!: (value: AttachmentCompletion) => void; + const completion = new Promise(done => { resolve = done; }); + let settled = false; + const finish = (status: AttachmentCompletion["status"]) => { if (!settled) { settled = true; resolve(Object.freeze({ status })); } }; + const controller = new AbortController(); + let lease: AttachmentOwnedLease | undefined; + let reservation = CHUNK; + const releaseSlice = () => { this.releaseBytes(reservation); reservation = 0; }; + const closeLease = async () => { const open = lease; lease = undefined; if (open) await open.close().catch(() => undefined); }; + correlation.prepare(this.sessionId, this.nonce); + const fields: AttachmentFields = { conversationId: options.conversationId }; + const job: CoreJob = { + lane: uuid(), bytes: 0, controller, + denied: () => this.recheck(facts), + fail: reason => { finish(reason); releaseSlice(); void closeLease(); }, + run: async signal => { + try { + lease = await raceAbort(record.open(), signal); + if (tier === "wav_recording") { + const layout = await this.wavLayout(lease, options, singleCap); + if (typeof layout === "string") { + if (layout === "single") tier = "single"; + else { tier = "denied"; finish(layout); releaseSlice(); await closeLease(); return; } + } else { + // Segmented recording from lease ranges; the outer job returns so it never holds a worker. + const writer = this.ownedWavRecording(identity, correlation, options, layout, lease); + if (typeof writer === "string") { tier = "denied"; finish(writer); releaseSlice(); await closeLease(); return; } + recordingId = writer.recordingId; + const openLease = lease; lease = undefined; + void writer.completeFile(controller.signal).then(status => finish(status), () => finish("incomplete")) + .finally(() => { releaseSlice(); void openLease.close().catch(() => undefined); }); + return; + } + } + const opened = lease; + const request = this.uploadRequest({ ...facts }, fields, identity, { attachmentId: uuid(), occurrenceId: uuid() }, opened.size); + const outcome = await this.runUpload({ body: JSON.stringify(request), occurrenceId: request.occurrence_id, declaredBytes: opened.size, + source: { size: opened.size, read: (o, n) => opened.read(o, n), check: () => opened.check() }, recheck: () => this.recheck(facts, { ignoreSessionExpiry: true }) }, signal); + finish(outcome.status); + releaseSlice(); await closeLease(); + } catch (error) { finish(codeOf(error)); releaseSlice(); await closeLease(); } + }, + }; + this.enqueue(job); + const cancel = () => { controller.abort(); finish("incomplete"); }; + const value: AttachmentOwnedRecordingHandle = { + get tier() { return tier; }, + reason: "queued", + get recordingId() { return recordingId; }, + completion, + wait: (timeoutMs?: number) => new Promise(done => { + const timer = setTimeout(() => { cancel(); done(Object.freeze({ status: "incomplete" as const })); }, bounded(timeoutMs, this.finalizationTimeoutMs, 30000)); + void completion.then(result => { clearTimeout(timer); done(result); }); + }), + cancel, + }; + return value; + } + /** Parses at most 4 KiB of a WAV header through the lease, after admission (design 4.6). */ + private async wavLayout(lease: AttachmentOwnedLease, options: AttachmentOwnedRecordingOptions, singleCap: number): + Promise<{ rate: number; channels: number; dataOffset: number; dataBytes: number; layout: AttachmentRecordingOptions["layout"]; tracks: AttachmentRecordingOptions["tracks"] } | "single" | AttachmentAdmission> { + const fallback = lease.size <= singleCap ? "single" as const : "too_large" as const; + const head = await lease.read(0, Math.min(4096, lease.size)); + const view = new DataView(head.buffer, head.byteOffset, head.byteLength); + const tag = (offset: number) => String.fromCharCode(...head.subarray(offset, offset + 4)); + if (head.byteLength < 44 || tag(0) !== "RIFF" || tag(8) !== "WAVE") return fallback; + let offset = 12; let fmt: { format: number; channels: number; rate: number; bits: number; sub?: string } | undefined; + while (offset + 8 <= head.byteLength) { + const id = tag(offset); const size = view.getUint32(offset + 4, true); const body = offset + 8; + if (id === "fmt ") { + if (size < 16 || body + size > head.byteLength) return fallback; + const format = view.getUint16(body, true); + fmt = { format, channels: view.getUint16(body + 2, true), rate: view.getUint32(body + 4, true), bits: view.getUint16(body + 14, true) }; + if (format === 0xfffe) { + if (size < 40) return fallback; + fmt.sub = Array.from(head.subarray(body + 24, body + 40)).map(b => b.toString(16).padStart(2, "0")).join(""); + } + } else if (id === "data") { + if (!fmt) return fallback; + const pcm = fmt.format === 1 || (fmt.format === 0xfffe && fmt.sub === "0100000000001000800000aa00389b71"); + if (!pcm || fmt.bits !== 16 || fmt.channels < 1 || fmt.channels > 32 || fmt.rate < 8000 || fmt.rate > 192000) return fallback; + if (body + size !== lease.size || size % (fmt.channels * 2) !== 0 || size === 0) return fallback; + const frames = size / (fmt.channels * 2); + if (lease.size > Math.min(this.session!.limits.recording_bytes, 1073741824) || + Math.ceil(frames * 1000 / fmt.rate) > Math.min(this.session!.limits.recording_duration_ms, 7200000)) return "too_large"; + const all = Array.from({ length: fmt.channels }, (_, i) => i); + const given = options.wavTracks; + let layout: AttachmentRecordingOptions["layout"] = "mixed"; + let tracks: AttachmentRecordingOptions["tracks"] = [{ channels: all, direction: "mixed", role: fmt.channels === 1 ? (given?.[0]?.role ?? "unknown") : "unknown" }]; + if (fmt.channels === 2 && given && given.length === 2 && given[0].channels.length === 1 && given[0].channels[0] === 0 && + given[1].channels.length === 1 && given[1].channels[0] === 1) { layout = "stereo"; tracks = given.map(t => ({ channels: [...t.channels], direction: t.direction, role: t.role })); } + return { rate: fmt.rate, channels: fmt.channels, dataOffset: body, dataBytes: size, layout, tracks }; + } + offset = body + size + (size % 2); + } + return fallback; + } + private ownedWavRecording(identity: { traceId: string; spanId: string }, correlation: AttachmentCorrelation, options: AttachmentOwnedRecordingOptions, + layout: { rate: number; channels: number; dataOffset: number; dataBytes: number; layout: AttachmentRecordingOptions["layout"]; tracks: AttachmentRecordingOptions["tracks"] }, + lease: AttachmentOwnedLease): AttachmentRecording | AttachmentAdmission { + if (this.recordingSlots >= this.recordingLimit()) return "queue_full"; + this.recordingSlots++; + let slot = true; + const recordingOptions: AttachmentRecordingOptions = { format: { sampleRateHz: layout.rate }, layout: layout.layout, tracks: layout.tracks, + operation: "observe_consumed", purpose: options.purpose ?? "conversation_recording", capture: options.capture, conversationId: options.conversationId, windowSegments: 1 }; + const writer = new AttachmentRecording({ host: this, options: recordingOptions, identity, admission: "queued", sourceKind: "OwnedRecordingFile", + readmit: () => ({ reason: "queued" }), stamp: () => correlation.prepare(this.sessionId, this.nonce), + close: () => { if (slot) { slot = false; this.recordingSlots--; } } }); + if (writer.reason !== "queued") return writer.reason; + const frameBytes = layout.channels * 2; + const limits = this.session!.limits; + const F = attachmentSegmentFrames(layout.rate, frameBytes, limits.segment_duration_ms, Math.min(limits.segment_bytes, limits.max_upload_bytes)); + const total = layout.dataBytes / frameBytes; + const segments: RecordingFileSegment[] = []; + for (let start = 0; start < total; start += F) { + const end = Math.min(total, start + F); const bytes = (end - start) * frameBytes; + const header = attachmentWavHeader(layout.channels, layout.rate, bytes); const base = layout.dataOffset + start * frameBytes; + segments.push({ track: 0, startFrame: start, endFrame: end, source: { + size: ATTACHMENT_WAV_HEADER_BYTES + bytes, + read: async (offset, length) => { + // WavSegmentSource: a synthesized header then a pread of the frame range; never materialized. + const out = new Uint8Array(Math.min(length, ATTACHMENT_WAV_HEADER_BYTES + bytes - offset)); let written = 0; + if (offset < ATTACHMENT_WAV_HEADER_BYTES) { const part = header.subarray(offset, Math.min(ATTACHMENT_WAV_HEADER_BYTES, offset + out.byteLength)); out.set(part); written = part.byteLength; } + if (written < out.byteLength) { + const from = Math.max(0, offset - ATTACHMENT_WAV_HEADER_BYTES); + const part = await lease.read(base + from, out.byteLength - written); + if (part.byteLength !== out.byteLength - written) throw new AttachmentTransportError("source_unavailable"); + out.set(part, written); + } + return out; + }, + check: () => lease.check(), + } }); + } + this.recordings.set(writer.recordingId, writer); + writer.loadFileSegments(segments, total); + return writer; + } + + // --- completed-recording import --------------------------------------------------------------------- + async importCompletedRecording(target: AttachmentTarget, request: AttachmentCompletedRecordingRequest, options: AttachmentImportOptions): Promise { + const finished = (reason: AttachmentAdmission | AttachmentCompletion["status"]): AttachmentImportHandle => { + const completion = Promise.resolve(Object.freeze({ status: reason }) as AttachmentCompletion); + return Object.freeze({ reason, completion, status: async () => ({ state: "failed", reason: String(reason) }) }) as AttachmentImportHandle; + }; + try { + this.enter(); + const direction = request?.direction ?? "mixed"; + const facts: AdmissionFacts = { mimeType: "audio/ogg", direction, purpose: "conversation_recording", capture: options?.capture ? { ...options.capture } : undefined, + operation: "import_existing", sourceKind: "ProviderFile", importSource: true }; + const { reason, correlation, identity } = this.admitTarget(target, facts); + // Denied preflight never calls the resolver. + if (reason) return finished(reason); + if (!validAttachmentImportRequest(request) || !options?.resolver || typeof options.resolver.resolve !== "function") return finished("policy_denied"); + if (this.descriptors() >= this.maxDescriptors || this.bytes + 1 > this.maxBytes) return finished("queue_full"); + const id = ++this.nextReservation; this.bytes += 1; this.reservations.set(id, { bytes: 1, expiresAt: Infinity }); + const release = () => this.releaseReservation(id); + correlation!.prepare(this.sessionId, this.nonce); + const timeout = bounded(options.timeoutMs, 30000, 30000); + const controller = new AbortController(); + let resolved: unknown; + try { + resolved = await new Promise(resolve => { + const timer = setTimeout(() => { controller.abort(); resolve("source_unavailable"); }, timeout); + Promise.resolve().then(() => options.resolver.resolve({ provider: request.provider, providerRef: request.providerRef, direction }, controller.signal)) + .then(value => { clearTimeout(timer); resolve(value); }, () => { clearTimeout(timer); resolve("source_unavailable"); }); + }); + } catch { resolved = "source_unavailable"; } + const invalid = attachmentResolvedRecordingDenial(resolved, this.session?.limits as { recording_bytes?: number; recording_duration_ms?: number } | undefined); + if (invalid) { release(); return finished(invalid); } + const result = resolved as import("./types").AttachmentResolvedCompletedRecording; + const finalDirection = attachmentImportDirection(direction, result.direction); + if (!finalDirection) { release(); return finished("policy_denied"); } + const sourceKind: AttachmentSourceKind = result.source.kind === "ProviderFile" ? "ProviderFile" : "RemoteUrl"; + const finalFacts: AdmissionFacts = { ...facts, mimeType: result.mimeType, direction: finalDirection, sourceKind }; + const again = this.recheck(finalFacts, { admission: true }); + if (again) { release(); return finished(again); } + const recordingId = uuid(); + const body = JSON.stringify({ mode: "completed_recording", recording: { schema_version: 1, + association: { capture_session_id: this.sessionId, correlation_nonce: this.nonce, parent: { trace_id: attachmentTraceUuid(identity!.traceId), span_id: identity!.spanId } }, + policy_version: this.session!.policy.version, deletion_generation: 0, mime_type: result.mimeType, direction: finalDirection, source_kind: sourceKind, + recording_id: recordingId, modality: "audio", declared_bytes: result.declaredBytes, declared_duration_ms: result.declaredDurationMs }, + source: result.source.kind === "ProviderFile" ? { kind: "ProviderFile", connection_id: result.source.connectionId, registered_output_id: result.source.registeredOutputId } : + { kind: "RemoteUrlEnvelope", envelope_id: result.source.envelopeId } }); + if (!validAttachmentWire("RecordingImportRequest", JSON.parse(body))) { release(); return finished("unsupported_version"); } + let state: AttachmentAdmission | Exclude = "queued"; + let jobId: string | undefined; + const done = handle("queued"); + this.reservations.delete(id); // the descriptor becomes the job + this.enqueue({ lane: recordingId, bytes: 1, controller: new AbortController(), denied: () => this.recheck(finalFacts), + fail: reason => { state = reason === "failed" ? "incomplete" : reason; done.finish(state); }, + run: async signal => { + try { + const job = await this.attempt(s => this.wire!.submitImport(this.session!, body, attachmentIdempotencyKey("import", recordingId), s), signal, + () => { const r = this.recheck(finalFacts); if (r) throw new AttachmentTransportError(r); }); + if (!job || job.recordingId !== recordingId || !validAttachmentUuid(job.jobId) || job.state !== "importing") throw new AttachmentTransportError("unsupported_version"); + jobId = job.jobId; state = "queued"; done.finish("importing"); + } catch (error) { state = codeOf(error); done.finish(state); this.dropped++; } + } }); + const manager = this; + return { + get reason() { return state; }, + recordingId, + get jobId() { return jobId; }, + completion: done.value.completion, + async status(timeoutMs?: number) { + if (!jobId || !manager.session || !manager.wire) return { state: "failed", reason: String(state === "queued" ? "incomplete" : state) }; + const c = new AbortController(); + const timer = setTimeout(() => c.abort(), bounded(timeoutMs, 2000, 30000)); unref(timer); + try { return await raceAbort(manager.wire.importStatus(manager.session, jobId, c.signal), c.signal); } + catch (error) { return { state: "failed", reason: codeOf(error) }; } + finally { clearTimeout(timer); } + }, + }; + } catch { return finished("source_unavailable"); } + } + + // --- queue, workers and transfers ---------------------------------------------------------------------- + /** @internal */ + enqueue(job: CoreJob): void { + let lane = this.lanes.get(job.lane); + if (!lane) { lane = new Lane(job.lane); this.lanes.set(job.lane, lane); this.order.push(lane); } + lane.jobs.push(job); this.queued++; + // Avoid even synchronous user transport execution in the caller's callback. + void Promise.resolve().then(() => this.pump()); + } + private nextLane(): Lane | undefined { + const count = this.order.length; + for (let i = 0; i < count; i++) { + const index = (this.cursor + i) % count; const lane = this.order[index]; + if (!lane.busy && lane.jobs.length) { this.cursor = index + 1; return lane; } } + return undefined; + } + private dropLane(lane: Lane): void { + if (lane.busy || lane.jobs.length) return; + this.lanes.delete(lane.id); this.order = this.order.filter(item => item !== lane); + if (this.cursor > this.order.length) this.cursor = 0; + } + private endJob(job: CoreJob): void { + if (job.settled) return; + job.settled = true; this.releaseBytes(job.bytes); job.bytes = 0; } - private async attempt(operation: (signal: AbortSignal) => Promise, parent: AbortSignal, guard: () => void = () => undefined, attempts = 3): Promise { + private pump(): void { + while (this.active.size < this.workers) { + const lane = this.nextLane(); + if (!lane) break; + const job = lane.jobs.shift()!; this.queued--; + if (job.controller.signal.aborted) { job.fail("incomplete"); this.endJob(job); this.dropLane(lane); this.notify(); continue; } + lane.busy = true; this.active.add(job); + void (async () => { try { await job.run(job.controller.signal); } catch { job.fail("incomplete"); } })().finally(() => { + this.endJob(job); this.active.delete(job); lane.busy = false; this.dropLane(lane); + this.notify(); this.pump(); + }); + } + } + private async attempt(operation: (signal: AbortSignal) => Promise, parent: AbortSignal, guard: () => void = () => undefined, attempts = 3, + timeoutMs = this.requestTimeout): Promise { for (let attempt = 0; ; attempt++) { if (parent.aborted) throw new AttachmentTransportError("incomplete"); guard(); const controller = new AbortController(); const abort = () => controller.abort(); parent.addEventListener("abort", abort, { once: true }); - const timer = setTimeout(abort, this.requestTimeout); unref(timer); + const timer = setTimeout(abort, timeoutMs); unref(timer); try { - const result = await operation(controller.signal); - if (controller.signal.aborted) throw new AttachmentTransportError("incomplete"); + // Raced against the abort signal: a transport that ignores its signal can never hold shutdown. + const result = await raceAbort(Promise.resolve().then(() => operation(controller.signal)), controller.signal); + if (controller.signal.aborted) throw new AttachmentTransportError("incomplete", !parent.aborted); return result; - } catch (error) { - if (!(error instanceof AttachmentTransportError) || !error.retryable || attempt + 1 >= attempts || parent.aborted || controller.signal.aborted) throw error; - await this.pause(Math.max(error.retryAfterMs, 25 * 2 ** attempt + Math.random() * 25), parent); + } catch (raw) { + const timedOut = controller.signal.aborted && !parent.aborted; + const error = raw instanceof AttachmentTransportError ? (raw.code === "failed" ? new AttachmentTransportError("incomplete", raw.retryable, raw.retryAfterMs, raw.httpStatus) : raw) + : new AttachmentTransportError("incomplete", timedOut); + const retryable = error.retryable || (timedOut && error.code === "incomplete"); + if (!retryable || attempt + 1 >= attempts || parent.aborted) throw error; + const backoff = (0.5 + Math.random() * 0.5) * 100 * 2 ** (attempt + 1); + await pause(Math.max(Math.min(error.retryAfterMs, 5000), backoff), parent); } finally { clearTimeout(timer); parent.removeEventListener("abort", abort); } } } - private pause(ms: number, signal: AbortSignal): Promise { - return new Promise(resolve => { - const done = () => { clearTimeout(timer); signal.removeEventListener("abort", done); resolve(); }; - const timer = setTimeout(done, ms); unref(timer); - signal.addEventListener("abort", done, { once: true }); - if (signal.aborted) done(); - }); + /** @internal Initialize -> content -> complete, with capability renewal and a best-effort abort (design 2.7, 3.2). */ + async runUpload(spec: UploadSpec, signal: AbortSignal): Promise { + const wire = this.wire!; + const outcome: UploadOutcome = { status: "incomplete", body: spec.body, renewals: 0 }; + let request: AttachmentUploadRequest; + try { request = JSON.parse(spec.body); } catch { outcome.status = "unsupported_version"; return outcome; } + // Closed-schema check before any request: never a role, channel, URL, path or credential (T-CH-01). + if (!validAttachmentWire("UploadRequest", request)) { outcome.status = "unsupported_version"; return outcome; } + const guard = () => { + if (signal.aborted) throw new AttachmentTransportError("incomplete"); + const reason = spec.recheck(); if (reason) throw new AttachmentTransportError(reason); + if (!this.session || request.policy_version !== this.session.policy.version) throw new AttachmentTransportError("policy_denied"); + }; + try { + const live = () => { guard(); if (this.runtimeState() === "expired") throw new AttachmentTransportError("expired"); }; + const upload = await this.attempt(s => wire.initialize(this.session!, spec.body, attachmentIdempotencyKey("upload", spec.occurrenceId), s), signal, live); + if (!validAttachmentUpload(upload, request, this.clock.wall())) throw new AttachmentTransportError("unsupported_version"); + outcome.upload = upload; + } catch (error) { outcome.status = codeOf(error); return outcome; } + const fresh = async () => { + if (this.clock.wall() >= Date.parse(outcome.upload!.expires_at) - SKEW && !(await this.renewUpload(outcome, signal))) throw new AttachmentTransportError("expired"); + }; + const withCapability = async (op: (s: AbortSignal) => Promise, timeout: number): Promise => { + for (let renewed = false; ;) { + try { return await this.attempt(async s => { await fresh(); return op(s); }, signal, guard, 3, timeout); } + catch (error) { + // One renewal after a 401 on a capability route, then one more try. + if (error instanceof AttachmentTransportError && error.httpStatus === 401 && !renewed) { + renewed = true; + if (await this.renewUpload(outcome, signal)) continue; + throw new AttachmentTransportError("expired"); + } + throw error; + } + } + }; + try { + const transferTimeout = Math.min(this.contentTimeout, (this.session?.limits.transfer_timeout_seconds ?? 600) * 1000); + await withCapability(s => wire.putContent(this.session!, outcome.upload!, spec.source, s, this.pacer), transferTimeout); + if (spec.beforeComplete) await spec.beforeComplete(outcome, signal); + await withCapability(s => wire.complete(this.session!, outcome.upload!, attachmentIdempotencyKey("complete", outcome.upload!.upload_id), s), this.requestTimeout); + guard(); + outcome.status = "validating"; + } catch (error) { + outcome.status = codeOf(error); + // Exactly one best-effort abort after a terminal failure that followed a successful initialize. + const upload = outcome.upload; + if (upload && this.session) { + const c = new AbortController(); const timer = setTimeout(() => c.abort(), this.requestTimeout); unref(timer); + try { await raceAbort(wire.abort(this.session, upload, attachmentIdempotencyKey("abort", upload.upload_id), c.signal), c.signal); } + catch { /* best effort */ } finally { clearTimeout(timer); } + } + outcome.upload = undefined; + } + return outcome; } - private drop(job: Job, reason: AttachmentCompletion["status"]): void { - if (!job.dropped) { job.dropped = true; this.dropped++; job.handle.finish(reason); } + /** @internal Replay initialize with the original identity for a fresh capability (BR-1). */ + async renewUpload(outcome: UploadOutcome, signal: AbortSignal): Promise { + const upload = outcome.upload; + if (!upload || !this.session || !this.wire || outcome.renewals >= 3 || signal.aborted) return false; + outcome.renewals++; + try { + const next = await this.attempt(s => this.wire!.renewUpload(this.session!, upload, outcome.body, s), signal, () => undefined, 1); + if (!next || next.upload_id !== upload.upload_id || next.attachment_id !== upload.attachment_id || next.occurrence_id !== upload.occurrence_id || + next.reserved_bytes !== upload.reserved_bytes || next.write_capability === upload.write_capability || + !(Date.parse(next.expires_at) > Date.parse(upload.expires_at))) return false; + outcome.upload = next; + return true; + } catch { return false; } + } + /** @internal Manifest PUT authorized by a segment capability only; a 409 on a replayed revision is success (BR-5). */ + async putManifest(capability: string, recordingId: string, body: string, key: string, signal: AbortSignal, replay: boolean): Promise { + if (!this.wire || !this.session) return "incomplete"; + let tries = 0; + try { + await this.attempt(async s => { + const n = tries++; + try { await this.wire!.putManifest(this.session!, capability, recordingId, body, key, s); } + catch (error) { if ((n > 0 || replay) && error instanceof AttachmentTransportError && error.code === "conflict") return; throw error; } + }, signal, () => undefined, 3, this.manifestTimeoutMs); + return "ok"; + } catch (error) { return codeOf(error); } } - /** Tightening discards all queued data and cancels transfers; re-enable never resurrects it. */ + private drop(job: CoreJob, reason: AttachmentCompletion["status"]): void { this.dropped++; job.fail(reason); } + + // --- policy and lifecycle ------------------------------------------------------------------------------ + /** Tightening discards all queued data and cancels transfers; re-enable never resurrects it or turns a disabled manager on. */ updatePolicy(update: Pick): void { - if (update.enabled !== undefined) this.config.enabled = update.enabled; + if (!this.enabled) return; + if (update.enabled === false) this.config.enabled = false; if (update.traceConfig) this.config.traceConfig = { ...this.config.traceConfig, ...update.traceConfig }; + this.epoch++; this.cancelDenied(); } /** A revoked capability/session/generation never resumes or retains capture data. */ invalidate(reason: "deleted" | "expired" | "policy_denied" = "deleted"): void { - this.closedReason = reason; this.closed = true; this.bootController.abort(); this.cancelDenied(); + this.closedReason = reason; this.closed = true; this.bootController?.abort(); this.epoch++; + this.cancelDenied(); + for (const state of [...this.streams]) this.streamAbort(state, "policy_denied"); + for (const id of [...this.reservations.keys()]) this.releaseReservation(id); + for (const writer of [...this.recordings.values()]) writer.abort(); } private cancelDenied(): void { - for (let index = this.queue.length - 1; index >= 0; index--) { - const job = this.queue[index]; const reason = this.denial(job.kind, job.options); - if (reason) { this.queue.splice(index, 1); job.data = undefined; this.bytes -= job.size; this.drop(job, reason); } + for (const lane of [...this.order]) { + for (let index = lane.jobs.length - 1; index >= 0; index--) { + const job = lane.jobs[index]; const reason = job.denied(); + if (reason) { lane.jobs.splice(index, 1); this.queued--; this.endJob(job); this.drop(job, reason); } + } + this.dropLane(lane); } for (const job of this.active) { - const reason = this.denial(job.kind, job.options); + const reason = job.denied(); if (reason) { job.controller.abort(); this.drop(job, reason); } } + for (const state of [...this.streams]) if (this.recheck(state.facts)) this.streamAbort(state, "policy_denied"); this.notify(); } - private notify(): void { for (const notify of this.listeners) notify(); } private deadline(promise: Promise, timeout: number, fallback: T): Promise { return new Promise(resolve => { // An explicit lifecycle wait owns a bounded grace period; background work stays unref'd. @@ -263,30 +1096,49 @@ export class AttachmentManager { }); } async flush(timeoutMs = this.shutdownTimeout): Promise { + if (!this.enabled) return Object.freeze({ unfinished: 0, dropped: 0 }); + this.sweep(); this.cancelDenied(); - if (this.queue.length || this.active.size) { + if (this.queued || this.active.size) { await new Promise(resolve => { const done = () => { clearTimeout(timer); this.listeners.delete(changed); resolve(); }; - const changed = () => { if (!this.queue.length && !this.active.size) done(); }; + const changed = () => { if (!this.queued && !this.active.size) done(); }; const timer = setTimeout(done, bounded(timeoutMs, this.shutdownTimeout, 30000)); this.listeners.add(changed); changed(); }); } - return Object.freeze({ unfinished: this.queue.length + this.active.size, dropped: this.dropped }); + return Object.freeze({ unfinished: this.queued + this.active.size + this.streams.size + this.recordings.size, dropped: this.dropped }); } shutdown(timeoutMs = this.shutdownTimeout): Promise { if (!this.shutdownPromise) { - // Stop new admissions, but give accepted jobs a bounded finalization window. + // Stop new admissions, but give accepted jobs and open recordings a bounded finalization window. this.shutdownPromise = (async () => { - const result = await this.flush(timeoutMs); - this.closed = true; this.bootController.abort(); this.cancelDenied(); + if (!this.enabled) return Object.freeze({ unfinished: 0, dropped: 0 }); + const budget = bounded(timeoutMs, this.shutdownTimeout, 30000); + const deadline = this.clock.now() + budget; + this.stoppingFlag = true; + const writers = [...this.recordings.values()]; + if (writers.length) await this.deadline(Promise.all(writers.map(writer => writer.shutdown(Math.max(1, deadline - this.clock.now())))), Math.max(1, deadline - this.clock.now()), []); + for (const state of [...this.streams]) this.streamAbort(state, "incomplete"); + for (const id of [...this.reservations.keys()]) this.releaseReservation(id); + const result = await this.flush(Math.max(1, Math.floor(deadline - this.clock.now()))); + this.closed = true; this.closedReason = "incomplete"; this.bootController?.abort(); + for (const lane of [...this.order]) { + for (const job of lane.jobs.splice(0)) { this.queued--; this.endJob(job); this.drop(job, "incomplete"); } + this.dropLane(lane); + } + for (const job of this.active) { job.controller.abort(); this.drop(job, "incomplete"); } + for (const writer of [...this.recordings.values()]) writer.abort(); + this.notify(); return Object.freeze({ unfinished: result.unfinished, dropped: this.dropped }); })(); } return this.shutdownPromise; } - /** Aggregate diagnostics only; no per-source metadata, names, hashes or exception text. */ - status(): Readonly<{ state: AttachmentAdmission; queued: number; active: number; ownedBytes: number; dropped: number }> { - return Object.freeze({ state: this.closed ? this.closedReason : this.bootStatus, queued: this.queue.length, active: this.active.size, ownedBytes: this.bytes, dropped: this.dropped }); + /** Aggregate diagnostics only; no per-source metadata, names, hashes, capabilities or exception text. */ + status(): AttachmentStatus { + if (this.enabled) this.sweep(); + return Object.freeze({ state: this.runtimeState(), queued: this.queued, active: this.active.size, openTickets: this.reservations.size, + openStreams: this.streams.size, openRecordings: this.recordings.size, ownedBytes: this.bytes, dropped: this.dropped }); } } diff --git a/typescript/packages/fi-core/src/attachments/node-sources.ts b/typescript/packages/fi-core/src/attachments/node-sources.ts new file mode 100644 index 000000000..2743ee571 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/node-sources.ts @@ -0,0 +1,112 @@ +/** + * Node-only owned recording files (design section 4). Reachable only from `./attachments/node`. + * + * Guarantees, stated honestly (design 4.4): the root must be canonical (realpath equals the path) with no symlink + * component; every intermediate component is lstat'ed (a directory, not a symlink) immediately before the leaf is + * opened with O_NOFOLLOW; the opened handle must match the identified (dev, ino, size) and its realpath must equal the + * expected path. RESIDUAL RACE: an intermediate directory swapped to a symlink between the lstat and the open can + * redirect the leaf; the identity bind and the post-open realpath check are the mitigation. Windows is unsupported. + */ +import { constants, promises as fs } from "node:fs"; +import * as path from "node:path"; +import { registerAttachmentOwnedFile, type AttachmentOwnedLease } from "./owned"; +import { AttachmentTransportError } from "./transport-v1"; +import type { AttachmentFileIdentity, AttachmentOwnedFileRef, AttachmentRecordingFileScope } from "./types"; + +const unavailable = () => new AttachmentTransportError("source_unavailable"); +function validRelative(relativePath: string): string[] | undefined { + if (typeof relativePath !== "string" || !relativePath || relativePath.length > 4096 || relativePath.includes("\0") || path.isAbsolute(relativePath)) return undefined; + const parts = relativePath.split("/"); + if (parts.some(part => !part || part === "." || part === ".." || part.includes("\\"))) return undefined; + return parts; +} +async function noSymlinkDirectory(full: string): Promise { + const info = await fs.lstat(full); + if (info.isSymbolicLink() || !info.isDirectory()) throw unavailable(); +} +async function openLeaf(root: string, parts: readonly string[]) { + let current = root; + for (const part of parts.slice(0, -1)) { current = path.join(current, part); await noSymlinkDirectory(current); } + const full = path.join(root, ...parts); + const handle = await fs.open(full, constants.O_RDONLY | constants.O_NOFOLLOW | (constants.O_NONBLOCK ?? 0)); + try { + const stat = await handle.stat({ bigint: true }); + if (!stat.isFile()) throw unavailable(); + if (await fs.realpath(full) !== full) throw unavailable(); + return { handle, stat, full }; + } catch (error) { await handle.close().catch(() => undefined); throw error; } +} +class Scope implements AttachmentRecordingFileScope { + private closedFlag = false; + private readonly leases = new Set(); + constructor(private readonly root: string) {} + closed(): boolean { return this.closedFlag; } + async identify(relativePath: string): Promise { + const parts = validRelative(relativePath); + if (this.closedFlag || !parts) throw unavailable(); + const { handle, stat } = await openLeaf(this.root, parts).catch(() => { throw unavailable(); }); + await handle.close().catch(() => undefined); + if (stat.size > BigInt(Number.MAX_SAFE_INTEGER)) throw unavailable(); + return Object.freeze({ dev: stat.dev, ino: stat.ino, size: Number(stat.size) }); + } + source(relativePath: string, identity: AttachmentFileIdentity): { kind: "OwnedRecordingFile"; lease: AttachmentOwnedFileRef } { + const parts = validRelative(relativePath); + if (!parts || !identity || typeof identity.dev !== "bigint" || typeof identity.ino !== "bigint" || !Number.isSafeInteger(identity.size)) { + // A forged or invalid identity yields a ref that always fails closed. + return { kind: "OwnedRecordingFile", lease: registerAttachmentOwnedFile({ identity: { dev: BigInt(0), ino: BigInt(0), size: 0 }, open: async () => { throw unavailable(); }, closed: () => true }) }; + } + const fixed = Object.freeze({ dev: identity.dev, ino: identity.ino, size: identity.size }); + const ref = registerAttachmentOwnedFile({ + identity: fixed, + closed: () => this.closedFlag, + open: async () => { + if (this.closedFlag) throw unavailable(); + const { handle, stat } = await openLeaf(this.root, parts).catch(() => { throw unavailable(); }); + // Identity bind: a substituted file must also reproduce (dev, ino, size). + if (stat.dev !== fixed.dev || stat.ino !== fixed.ino || stat.size !== BigInt(fixed.size)) { await handle.close().catch(() => undefined); throw unavailable(); } + const snapshot = { dev: stat.dev, ino: stat.ino, size: stat.size, mtime: stat.mtimeNs, ctime: stat.ctimeNs }; + let open = true; + const lease: AttachmentOwnedLease = { + size: fixed.size, + // Positional reads on our own handle never move an application's file position. + read: async (offset, length) => { + if (!open || this.closedFlag) throw unavailable(); + const buffer = new Uint8Array(length); let done = 0; + while (done < length) { + const { bytesRead } = await handle.read(buffer, done, length - done, offset + done); + if (!bytesRead) break; done += bytesRead; + } + return done === length ? buffer : buffer.subarray(0, done); + }, + check: async () => { + if (!open || this.closedFlag) throw unavailable(); + const now = await handle.stat({ bigint: true }); + if (now.dev !== snapshot.dev || now.ino !== snapshot.ino || now.size !== snapshot.size || now.mtimeNs !== snapshot.mtime || now.ctimeNs !== snapshot.ctime) throw unavailable(); + }, + close: async () => { if (!open) return; open = false; this.leases.delete(lease); await handle.close().catch(() => undefined); }, + }; + this.leases.add(lease); + return lease; + }, + }); + return { kind: "OwnedRecordingFile", lease: ref }; + } + async close(): Promise { + this.closedFlag = true; + await Promise.all([...this.leases].map(lease => lease.close())); + } +} +/** An explicitly bound session's recording root. A report path alone is NOT authority (F01). */ +export async function createRecordingFileScope(root: string, options: { enabled: true }): Promise { + if (!options || options.enabled !== true || process.platform === "win32" || typeof root !== "string" || !path.isAbsolute(root)) throw unavailable(); + const normalized = path.resolve(root); + if (normalized !== root.replace(/\/+$/, "") && normalized !== root) throw unavailable(); + let real: string; + try { real = await fs.realpath(normalized); } catch { throw unavailable(); } + if (real !== normalized) throw unavailable(); + // lstat every root component: none may be a symlink. + const parts = normalized.split(path.sep).filter(Boolean); + let current: string = path.sep; + for (const part of parts) { current = path.join(current, part); await noSymlinkDirectory(current).catch(() => { throw unavailable(); }); } + return new Scope(normalized); +} diff --git a/typescript/packages/fi-core/src/attachments/node.ts b/typescript/packages/fi-core/src/attachments/node.ts index 6d71d0402..a911f7aa2 100644 --- a/typescript/packages/fi-core/src/attachments/node.ts +++ b/typescript/packages/fi-core/src/attachments/node.ts @@ -1,62 +1,99 @@ import { request } from "node:https"; +import { DisabledAttachmentManager } from "./disabled"; import { AttachmentManager } from "./manager"; -import { AttachmentTransportError, createAttachmentHttpTransport } from "./transport-v1"; +import { attachmentEnabled } from "./policy"; +import { ATTACHMENT_CHUNK_BYTES, AttachmentTransportError, createAttachmentHttpTransport } from "./transport-v1"; import type { AttachmentHttpClient } from "./transport-v1"; -import type { AttachmentManagerOptions } from "./types"; +import type { AttachmentContentSource, AttachmentManagerOptions } from "./types"; +type Ranged = Exclude; +function sizeOf(body: string | AttachmentContentSource | undefined): number { + if (body === undefined) return 0; + if (typeof body === "string") return Buffer.byteLength(body); + if (body instanceof Uint8Array) return body.byteLength; + return (body as { size: number }).size; +} /** Node-only bounded transport. No keep-alive pool, redirect following, cookies or environment proxy. */ export const attachmentNodeHttpClient: AttachmentHttpClient = input => new Promise((resolve, reject) => { let settled = false; - const fail = () => { if (!settled) { settled = true; reject(new AttachmentTransportError("failed", !input.signal.aborted)); } }; + const fail = (code: "incomplete" | "unsupported_version" | "source_unavailable" = "incomplete", retryable = !input.signal.aborted) => { + if (!settled) { settled = true; reject(new AttachmentTransportError(code, retryable)); } + }; try { - const size = typeof input.body === "string" ? Buffer.byteLength(input.body) : input.body instanceof Uint8Array ? input.body.byteLength : input.body.size; - const req = request(input.url, { method: input.method, headers: { ...input.headers, "content-length": String(size) }, + const size = sizeOf(input.body); + const req = request(input.url, { method: input.method, headers: { ...input.headers, ...(input.body === undefined ? {} : { "content-length": String(size) }) }, signal: input.signal, agent: false }, response => { - if (!response.statusCode || response.statusCode < 200 || response.statusCode >= 300) { - settled = true; - resolve({ status: response.statusCode ?? 0, body: new Uint8Array(), retryAfter: response.headers["retry-after"] }); - response.destroy(); req.destroy(); return; - } - const chunks: Buffer[] = []; let bytes = 0; + const status = response.statusCode ?? 0; + const ok = status >= 200 && status < 300; + // Success bodies up to 64 KiB; error bodies up to 4 KiB (parsed only as F01 Error). + const limit = ok ? 65536 : 4096; + const chunks: Buffer[] = []; let bytes = 0; let over = false; response.on("data", (chunk: Buffer) => { + if (over) return; bytes += chunk.byteLength; - if (bytes > 65536) { settled = true; reject(new AttachmentTransportError("failed")); response.destroy(); req.destroy(); return; } + if (bytes > limit) { + over = true; + if (ok) { fail("unsupported_version", false); response.destroy(); req.destroy(); return; } + settled = true; resolve({ status, body: new Uint8Array(), retryAfter: response.headers["retry-after"] as string | undefined }); + response.destroy(); req.destroy(); return; + } chunks.push(chunk); }); - response.on("error", fail); - response.on("aborted", fail); + response.on("error", () => fail()); + response.on("aborted", () => fail()); response.on("end", () => { - if (!settled) { settled = true; resolve({ status: response.statusCode ?? 0, body: Buffer.concat(chunks, bytes), retryAfter: response.headers["retry-after"] }); } + if (!settled) { settled = true; resolve({ status, body: Buffer.concat(chunks, bytes), retryAfter: response.headers["retry-after"] as string | undefined }); } }); }); req.on("socket", socket => socket.unref()); - req.on("error", fail); - // A Blob is immutable and read independently, one bounded slice at a time. + req.on("error", () => fail()); + // Per-slice stall timeout on content PUTs. + if (input.stallTimeoutMs) req.setTimeout(input.stallTimeoutMs, () => { req.destroy(); fail("incomplete", true); }); void (async () => { - if (typeof input.body === "string") { req.end(input.body); return; } - for (let offset = 0; offset < size; offset += 256 * 1024) { + const body = input.body; + if (body === undefined) { req.end(); return; } + if (typeof body === "string") { req.end(body); return; } + const ranged = !(body instanceof Uint8Array) && !(typeof Blob !== "undefined" && body instanceof Blob) ? body as Ranged : undefined; + for (let offset = 0; offset < size; offset += ATTACHMENT_CHUNK_BYTES) { if (input.signal.aborted || req.destroyed) return; - const chunk = input.body instanceof Uint8Array ? input.body.subarray(offset, offset + 256 * 1024) : - new Uint8Array(await input.body.slice(offset, offset + 256 * 1024).arrayBuffer()); + const length = Math.min(ATTACHMENT_CHUNK_BYTES, size - offset); + let chunk: Uint8Array; + if (body instanceof Uint8Array) chunk = body.subarray(offset, offset + length); + else if (ranged) { + // check() before every slice: a changed owned file aborts the transfer. + await ranged.check(); + chunk = await ranged.read(offset, length); + if (chunk.byteLength !== length) throw new AttachmentTransportError("source_unavailable"); + } else chunk = new Uint8Array(await (body as Blob).slice(offset, offset + length).arrayBuffer()); + if (input.pacer) await input.pacer.take(chunk.byteLength, input.signal); if (input.signal.aborted || req.destroyed) return; if (!req.write(chunk)) await new Promise(done => { const finish = () => { req.removeListener("drain", finish); req.removeListener("close", finish); done(); }; req.once("drain", finish); req.once("close", finish); }); } + if (ranged) await ranged.check(); req.end(); - })().catch(() => { req.destroy(); fail(); }); + })().catch(error => { req.destroy(); fail(error instanceof AttachmentTransportError ? error.code as "source_unavailable" : "incomplete", false); }); } catch { fail(); } }); export interface NodeAttachmentManagerOptions extends AttachmentManagerOptions { headers?: Readonly> } -/** Provider setup passes only its resolved FI credentials, never a provider/model client's key. */ +/** + * Provider setup passes only its resolved FI credentials, never a provider/model client's key. Returns the inert + * DisabledAttachmentManager when attachments are OFF; `enabled` is resolved once here, from config then env. + */ export function createNodeAttachmentManager(options: NodeAttachmentManagerOptions = {}): AttachmentManager { - const endpoint = options.endpoint ?? process.env.FI_ATTACHMENTS_ENDPOINT; + const environment = options.environment ?? (() => process.env); + let env: Readonly>; + try { env = environment(); } catch { env = { FI_ATTACHMENTS_ENABLED: "false" }; } + if (!attachmentEnabled(options, env)) return new DisabledAttachmentManager(); + const endpoint = options.endpoint ?? env.FI_ATTACHMENTS_ENDPOINT; let transport = options.transport; if (!transport && endpoint) { - try { transport = createAttachmentHttpTransport({ endpoint, headers: options.headers ?? {}, client: attachmentNodeHttpClient }); } + try { transport = createAttachmentHttpTransport({ endpoint, headers: options.headers ?? {}, client: attachmentNodeHttpClient, stallTimeoutMs: options.requestTimeoutMs }); } catch { /* Unconfigured/untrusted origin remains fail closed without affecting tracer setup. */ } } - return new AttachmentManager({ ...options, endpoint, environment: options.environment ?? (() => process.env), transport }); + return new AttachmentManager({ ...options, endpoint, environment, transport }); } +export { createRecordingFileScope } from "./node-sources"; export * from "./index"; diff --git a/typescript/packages/fi-core/src/attachments/owned.ts b/typescript/packages/fi-core/src/attachments/owned.ts new file mode 100644 index 000000000..a3e4fef26 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/owned.ts @@ -0,0 +1,28 @@ +/** Browser-safe registry of owned-file references minted by this core copy's Node scope (design 4.3). */ +import type { AttachmentFileIdentity, AttachmentOwnedFileRef } from "./types"; + +/** @internal An open lease over one owned file. Reads never move any application file position. */ +export interface AttachmentOwnedLease { + readonly size: number; + read(offset: number, size: number): Promise; + /** Throws source_unavailable when (dev, ino, size, mtime, ctime) changed since open. */ + check(): Promise; + close(): Promise; +} +/** @internal */ +export interface AttachmentOwnedFileRecord { + readonly identity: AttachmentFileIdentity; + open(): Promise; + closed(): boolean; +} +const records = new WeakMap(); +/** @internal Only the Node scope calls this. */ +export function registerAttachmentOwnedFile(record: AttachmentOwnedFileRecord): AttachmentOwnedFileRef { + const ref = Object.freeze(Object.create(null)) as AttachmentOwnedFileRef; + records.set(ref as unknown as object, record); + return ref; +} +/** @internal A foreign or forged object resolves to undefined (source_unavailable). */ +export function attachmentOwnedFile(ref: unknown): AttachmentOwnedFileRecord | undefined { + return ref && typeof ref === "object" ? records.get(ref) : undefined; +} diff --git a/typescript/packages/fi-core/src/attachments/recording.ts b/typescript/packages/fi-core/src/attachments/recording.ts new file mode 100644 index 000000000..36500ed1a --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/recording.ts @@ -0,0 +1,821 @@ +import { v4 as uuid } from "uuid"; +import { attachmentTraceUuid, validAttachmentUuid } from "./correlation"; +import type { AttachmentHost, CoreJob, UploadOutcome } from "./internal"; +import { attachmentIdempotencyKey } from "./transport-v1"; +import type { + AttachmentAdmission, AttachmentCapturePolicy, AttachmentCompletion, AttachmentContentSource, AttachmentDirection, AttachmentGapReason, + AttachmentRecordingManifest, AttachmentRecordingOptions, AttachmentRecordingResult, AttachmentRecordingStatus, AttachmentRecordingWriter, + AttachmentTrackSpec, AttachmentUploadRequest, +} from "./types"; + +export const ATTACHMENT_WAV_HEADER_BYTES = 44; +const MAX_SEGMENT_BYTES = 8 * 1024 * 1024; +const MAX_SEGMENT_MS = 30000; +const MAX_RECORDING_MS = 7200000; +const MAX_RECORDING_BYTES = 1024 * 1024 * 1024; +const MAX_MANIFEST_BYTES = 1048576; +const MAX_PLAYOUT = 4096; +const GAP_REASONS: readonly AttachmentGapReason[] = ["queue_full", "bootstrap_pending", "source_unavailable", "policy_denied", "truncated", "incomplete"]; + +function gcd(a: number, b: number): number { while (b) { [a, b] = [b, a % b]; } return a; } +/** Smallest frame count that is a whole number of milliseconds (design 2.5). */ +export function attachmentBoundaryStep(sampleRateHz: number): number { return sampleRateHz / gcd(sampleRateHz, 1000); } +/** Largest multiple of the boundary step within both the duration and the byte bound (header included). */ +export function attachmentSegmentFrames(sampleRateHz: number, frameBytes: number, segmentMs = MAX_SEGMENT_MS, segmentBytes = MAX_SEGMENT_BYTES): number { + const g = attachmentBoundaryStep(sampleRateHz); + const byMs = Math.floor(Math.min(segmentMs, MAX_SEGMENT_MS) * sampleRateHz / 1000); + const byBytes = Math.floor((Math.min(segmentBytes, MAX_SEGMENT_BYTES) - ATTACHMENT_WAV_HEADER_BYTES) / frameBytes); + return Math.floor(Math.min(byMs, byBytes) / g) * g; +} +export function attachmentFramesToMs(frames: number, sampleRateHz: number, roundUp = false): number { + return roundUp ? Math.ceil(frames * 1000 / sampleRateHz) : Math.floor(frames * 1000 / sampleRateHz); +} +/** Canonical 44-byte RIFF/WAVE PCM header (format tag 1, 16-bit). */ +export function attachmentWavHeader(channels: number, sampleRateHz: number, dataBytes: number): Uint8Array { + const header = new Uint8Array(ATTACHMENT_WAV_HEADER_BYTES); const view = new DataView(header.buffer); + const ascii = (offset: number, text: string) => { for (let i = 0; i < text.length; i++) header[offset + i] = text.charCodeAt(i); }; + ascii(0, "RIFF"); view.setUint32(4, 36 + dataBytes, true); ascii(8, "WAVE"); ascii(12, "fmt "); + view.setUint32(16, 16, true); view.setUint16(20, 1, true); view.setUint16(22, channels, true); view.setUint32(24, sampleRateHz, true); + view.setUint32(28, sampleRateHz * channels * 2, true); view.setUint16(32, channels * 2, true); view.setUint16(34, 16, true); + ascii(36, "data"); view.setUint32(40, dataBytes, true); + return header; +} +/** Header plus retained chunks, served as one bounded content source without re-materializing. */ +export function attachmentChunkedSource(header: Uint8Array, chunks: readonly Uint8Array[]): AttachmentContentSource { + let size = header.byteLength; for (const chunk of chunks) size += chunk.byteLength; + if (size <= MAX_SEGMENT_BYTES) { + const out = new Uint8Array(size); out.set(header, 0); let offset = header.byteLength; + for (const chunk of chunks) { out.set(chunk, offset); offset += chunk.byteLength; } + return out; + } + const parts = [header, ...chunks]; + return { + size, + async read(offset: number, length: number) { + const out = new Uint8Array(Math.min(length, size - offset)); let base = 0; let written = 0; + for (const part of parts) { + const end = base + part.byteLength; + if (end > offset && base < offset + out.byteLength) { + const from = Math.max(0, offset - base); const to = Math.min(part.byteLength, offset + out.byteLength - base); + out.set(part.subarray(from, to), written); written += to - from; + } + base = end; if (written === out.byteLength) break; + } + return out; + }, + async check() { /* in-memory */ }, + }; +} + +/** Cross-field manifest rules mirrored from F01 validate.py, applied locally before every PUT. */ +export function validAttachmentManifest(value: AttachmentRecordingManifest): boolean { + try { + if (!value || value.schema_version !== 1 || !validAttachmentUuid(value.recording_id) || value.timebase !== "session_relative_ms") return false; + if (new TextEncoder().encode(JSON.stringify(value)).byteLength > MAX_MANIFEST_BYTES) return false; + if (!Number.isInteger(value.sample_rate_hz) || value.sample_rate_hz < 8000 || value.sample_rate_hz > 192000) return false; + if (!Number.isInteger(value.channels) || value.channels < 1 || value.channels > 32) return false; + if (!Number.isInteger(value.duration_ms) || value.duration_ms < 0 || value.duration_ms > MAX_RECORDING_MS) return false; + if (value.capture_complete && value.gaps.length) return false; + if (!Array.isArray(value.tracks) || value.tracks.length < 1 || value.tracks.length > 32) return false; + const ids = value.tracks.map(track => track.track_id); + if (new Set(ids).size !== ids.length || !ids.every(validAttachmentUuid)) return false; + const channels = value.tracks.flatMap(track => track.channels).sort((a, b) => a - b); + if (channels.length !== value.channels || channels.some((channel, index) => channel !== index)) return false; + if (value.tracks.some(track => new Set(track.channels).size !== track.channels.length || !["user", "assistant", "mixed", "unknown"].includes(track.role) || + !["input", "output", "mixed"].includes(track.direction))) return false; + if (value.layout === "mixed" && value.tracks.some(track => track.direction !== "mixed")) return false; + if (value.layout === "stereo" && value.channels !== 2) return false; + const separate = value.layout === "separate_tracks"; + const seen = new Set(); let previous = -1; + for (let index = 0; index < value.segments.length; index++) { + const s = value.segments[index]; + if (s.index !== index || s.start_ms < previous || seen.has(s.occurrence_id) || !validAttachmentUuid(s.occurrence_id)) return false; + if (!Number.isInteger(s.duration_ms) || s.duration_ms < 1 || s.duration_ms > MAX_SEGMENT_MS) return false; + seen.add(s.occurrence_id); previous = s.start_ms; + } + const timelines = new Map(); + if (separate) for (const id of ids) timelines.set(id, []); else timelines.set(undefined, []); + for (const interval of [...value.segments, ...value.gaps]) { + if (separate ? interval.track_id === undefined : interval.track_id !== undefined) return false; + const list = timelines.get(interval.track_id); if (!list) return false; + if (!Number.isInteger(interval.start_ms) || !Number.isInteger(interval.duration_ms) || interval.duration_ms < 1) return false; + list.push(interval); + } + if (value.gaps.some(gap => !GAP_REASONS.includes(gap.reason))) return false; + for (const list of timelines.values()) { + let end = 0; + for (const interval of [...list].sort((a, b) => a.start_ms - b.start_ms)) { if (interval.start_ms !== end) return false; end += interval.duration_ms; } + if (end !== value.duration_ms) return false; + } + for (const interval of [...value.segments, ...value.gaps, ...value.playout]) { + if (interval.start_ms + interval.duration_ms > value.duration_ms || interval.duration_ms < 1) return false; + } + return true; + } catch { return false; } +} + +interface Piece { + kind: "segment" | "gap"; + track: number; + start: number; + end: number; + reason?: AttachmentGapReason; + // Segments only. + source?: AttachmentContentSource; + bytes?: number; + state?: "sealed" | "queued" | "uploading" | "done" | "failed"; + occurrenceId?: string; + attachmentId?: string; + index?: number; + outcome?: UploadOutcome; + failure?: AttachmentCompletion["status"]; + job?: CoreJob; +} +interface Track { + ordinal: number; + spec: AttachmentTrackSpec; + manifestId?: string; + frameBytes: number; + channels: number; + denied?: AttachmentAdmission; + pos: number; + fill: number; + blockGap?: AttachmentGapReason; + blockData: number; + runStart: number; + runFrames: number; + chunks: Uint8Array[]; + chunkBytes: number; + carry?: Uint8Array; + pieces: Piece[]; +} +export interface RecordingIdentity { traceId: string; spanId: string } +export interface RecordingInit { + host: AttachmentHost; + options: AttachmentRecordingOptions; + identity?: RecordingIdentity; + /** queued (active), bootstrap_pending (pending mode) or a denial (inert). */ + admission: AttachmentAdmission; + /** Per-track denial for separate_tracks. */ + trackDenials?: readonly (AttachmentAdmission | undefined)[]; + /** Re-run full admission (pending writers and policy epoch changes). */ + readmit(): { reason: AttachmentAdmission; trackDenials?: readonly (AttachmentAdmission | undefined)[] }; + /** Applies the carrier stamp once admitted. */ + stamp(): void; + /** Releases the recording slot and descriptor exactly once. */ + close(): void; + sourceKind?: "Bytes" | "OwnedRecordingFile"; +} +export interface RecordingFileSegment { track: number; startFrame: number; endFrame: number; source: AttachmentContentSource } + +/** Validates options, returning the layout's stream shape or undefined (fail closed). */ +export function attachmentRecordingShape(options: AttachmentRecordingOptions): { streams: number; frameBytes: number[]; channels: number } | undefined { + try { + const rate = options?.format?.sampleRateHz; + if (!Number.isInteger(rate) || rate < 8000 || rate > 192000 || (options.format.sampleFormat ?? "s16le") !== "s16le") return undefined; + if (attachmentBoundaryStep(rate) > 4800) return undefined; + if (!["mixed", "stereo", "separate_tracks"].includes(options.layout) || !["observe_consumed", "create_recording"].includes(options.operation)) return undefined; + if (options.purpose !== undefined && options.purpose !== "conversation_recording" && options.purpose !== "turn_recording") return undefined; + const tracks = options.tracks; + if (!Array.isArray(tracks) || tracks.length < 1 || tracks.length > 32) return undefined; + for (const track of tracks) { + if (!track || !Array.isArray(track.channels) || track.channels.length < 1 || new Set(track.channels).size !== track.channels.length || + track.channels.some((c: number) => !Number.isInteger(c) || c < 0 || c > 31) || !["input", "output", "mixed"].includes(track.direction) || + !["user", "assistant", "mixed", "unknown"].includes(track.role)) return undefined; + } + const all = tracks.flatMap(track => [...track.channels]).sort((a, b) => a - b); + if (all.some((channel, index) => channel !== index)) return undefined; + const window = options.windowSegments ?? 2; + if (!Number.isInteger(window) || window < 1 || window > 4) return undefined; + for (const id of [options.conversationId, options.turnId]) if (id !== undefined && !(typeof id === "string" && /^[a-zA-Z0-9_-]{1,128}$/.test(id))) return undefined; + if (options.layout === "mixed") return tracks.length === 1 && tracks[0].direction === "mixed" ? { streams: 1, frameBytes: [all.length * 2], channels: all.length } : undefined; + if (options.layout === "stereo") return tracks.length === 2 && tracks.every(track => track.channels.length === 1) ? { streams: 1, frameBytes: [4], channels: 2 } : undefined; + return { streams: tracks.length, frameBytes: tracks.map(track => track.channels.length * 2), channels: all.length }; + } catch { return undefined; } +} + +/** Live PCM recording writer and the shared segment/manifest pipeline (design section 2). */ +export class AttachmentRecording implements AttachmentRecordingWriter { + readonly recordingId = uuid(); + private mode: "pending" | "active" | "finishing" | "terminal"; + private admissionReason: AttachmentAdmission; + private terminalReason?: AttachmentAdmission | AttachmentCompletion["status"]; + private readonly host: AttachmentHost; + private readonly options: AttachmentRecordingOptions; + private readonly identity?: RecordingIdentity; + private readonly init: RecordingInit; + private readonly rate: number; + private readonly g: number; + private F = 0; + private readonly tracks: Track[] = []; + private readonly manifestTracks: { track_id: string; channels: number[]; direction: AttachmentDirection; role: AttachmentTrackSpec["role"] }[] = []; + private readonly windowSegments: number; + private epoch: number; + private totalBytes = 0; + private truncated = false; + private maxFrames = 0; + private recordingBytesCap = MAX_RECORDING_BYTES; + private completed = 0; + private sinceManifest = 0; + private lastPutEnd = 0; + private revision = 0; + private publishedSegments = 0; + private publishedGapMs = 0; + private publishedComplete = false; + private readonly cuts: number[] = []; + private manifestQueued = false; + private manifestFallback = false; + private manifestDisabled = false; + private indexReused = false; + private readonly playout: { startMs: number; durationMs: number; evidence: "generated" | "played" | "unknown" | "truncated" }[] = []; + private outstanding = 0; + private idleWaiters: (() => void)[] = []; + private resultPromise?: Promise; + private closed = false; + private finalEnd?: number; + private stamped = false; + + constructor(init: RecordingInit) { + this.init = init; this.host = init.host; this.options = init.options; this.identity = init.identity; + this.rate = init.options.format?.sampleRateHz ?? 8000; + this.g = Number.isInteger(this.rate) && this.rate > 0 ? attachmentBoundaryStep(this.rate) : 1; + this.windowSegments = init.options.windowSegments ?? 2; + this.epoch = this.host.policyEpoch(); + this.admissionReason = init.admission; + const shape = attachmentRecordingShape(init.options); + if (!shape) { this.admissionReason = this.admissionReason === "queued" || this.admissionReason === "bootstrap_pending" ? "policy_denied" : this.admissionReason; } + if (shape) { + const specs = init.options.tracks; + for (const spec of specs) this.manifestTracks.push({ track_id: uuid(), channels: [...spec.channels], direction: spec.direction, role: spec.role }); + for (let i = 0; i < shape.streams; i++) { + this.tracks.push({ ordinal: i, spec: init.options.layout === "separate_tracks" ? specs[i] : { channels: specs.flatMap(s => [...s.channels]), direction: "mixed", role: "mixed" }, + manifestId: init.options.layout === "separate_tracks" ? this.manifestTracks[i].track_id : undefined, + frameBytes: shape.frameBytes[i], channels: shape.frameBytes[i] / 2, denied: init.trackDenials?.[i], pos: 0, fill: 0, blockData: 0, + runStart: 0, runFrames: 0, chunks: [], chunkBytes: 0, pieces: [] }); + } + } + if (this.admissionReason === "queued") { this.mode = "active"; this.activate(); } + else if (this.admissionReason === "bootstrap_pending" && shape) this.mode = "pending"; + else { this.mode = "terminal"; this.terminalReason = this.admissionReason; this.closeOnce(); } + } + get reason(): AttachmentAdmission { return this.admissionReason; } + + private activate(): boolean { + const session = this.host.currentSession(); + if (!session) return false; + const limits = session.limits; + const segMs = Math.min(limits.segment_duration_ms ?? MAX_SEGMENT_MS, MAX_SEGMENT_MS); + const segBytes = Math.min(limits.segment_bytes ?? MAX_SEGMENT_BYTES, limits.max_upload_bytes ?? MAX_SEGMENT_BYTES, MAX_SEGMENT_BYTES); + this.F = attachmentSegmentFrames(this.rate, Math.max(...this.tracks.map(t => t.frameBytes)), segMs, segBytes); + if (this.F < this.g) { this.terminal("too_large"); return false; } + const recMs = Math.min(limits.recording_duration_ms ?? MAX_RECORDING_MS, MAX_RECORDING_MS); + this.maxFrames = Math.floor(recMs * this.rate / 1000); + this.recordingBytesCap = Math.min(limits.recording_bytes ?? MAX_RECORDING_BYTES, MAX_RECORDING_BYTES); + if (!this.stamped) { this.stamped = true; try { this.init.stamp(); } catch { /* never throw into audio callbacks */ } } + return true; + } + private terminal(reason: AttachmentAdmission | AttachmentCompletion["status"]): void { + if (this.mode === "terminal") return; + this.mode = "terminal"; this.terminalReason = reason; + } + private closeOnce(): void { if (!this.closed) { this.closed = true; try { this.init.close(); } catch { /* bounded */ } this.host.recordingClosed(this.recordingId); } } + + // --- block timeline ----------------------------------------------------------------------------- + private releaseTrackBlock(track: Track): void { + if (!track.blockData) return; + let drop = track.blockData * track.frameBytes; track.blockData = 0; + this.host.releaseBytes(drop); track.chunkBytes -= drop; + while (drop > 0 && track.chunks.length) { + const last = track.chunks[track.chunks.length - 1]; + if (last.byteLength <= drop) { drop -= last.byteLength; track.chunks.pop(); } + else { track.chunks[track.chunks.length - 1] = last.subarray(0, last.byteLength - drop); drop = 0; } + } + } + private commitBlock(track: Track, frames = this.g): void { + if (track.blockGap === undefined) { track.runFrames += frames; track.blockData = 0; } + else { this.closeRun(track); this.appendGap(track, track.blockGap, frames); } + track.pos += frames; track.fill = 0; track.blockGap = undefined; + if (track.runFrames === 0) track.runStart = track.pos; + if (this.F && track.pos % this.F === 0) { this.closeRun(track); this.boundary(track); } + } + private appendGap(track: Track, reason: AttachmentGapReason, frames: number): void { + const last = track.pieces[track.pieces.length - 1]; + const start = track.pos; + if (last && last.kind === "gap" && last.reason === reason && last.end === start) last.end = start + frames; + else track.pieces.push({ kind: "gap", track: track.ordinal, start, end: start + frames, reason }); + track.runStart = start + frames; + } + private closeRun(track: Track): void { + if (track.runFrames > 0) { + const dataBytes = track.runFrames * track.frameBytes; + const chunks = track.chunks; let kept = 0; const used: Uint8Array[] = []; + for (const chunk of chunks) { if (kept >= dataBytes) break; const take = Math.min(chunk.byteLength, dataBytes - kept); used.push(take === chunk.byteLength ? chunk : chunk.subarray(0, take)); kept += take; } + // Bytes of the still-open block stay with the track. + const rest: Uint8Array[] = []; let skip = dataBytes; + for (const chunk of chunks) { if (skip >= chunk.byteLength) { skip -= chunk.byteLength; continue; } rest.push(skip ? chunk.subarray(skip) : chunk); skip = 0; } + track.chunks = rest; track.chunkBytes -= dataBytes; + const header = attachmentWavHeader(track.channels, this.rate, dataBytes); + this.totalBytes += dataBytes + ATTACHMENT_WAV_HEADER_BYTES; + track.pieces.push({ kind: "segment", track: track.ordinal, start: track.runStart, end: track.runStart + track.runFrames, + source: attachmentChunkedSource(header, used), bytes: dataBytes, state: "sealed", occurrenceId: uuid(), attachmentId: uuid() }); + } + track.runFrames = 0; track.runStart = track.pos; + this.release(); + } + /** Common boundaries: a track more than one window behind the leader is filled with a gap (design 2.5). */ + private boundary(leader: Track): void { + if (this.tracks.length < 2) return; + const target = leader.pos - this.F; + if (target <= 0) return; + for (const track of this.tracks) { + if (track === leader || track.pos + track.fill >= target) continue; + this.gapFrames(track, target - track.pos - track.fill, track.denied ? "policy_denied" : "source_unavailable"); + } + } + private gapFrames(track: Track, frames: number, reason: AttachmentGapReason): void { + while (frames > 0) { + if (track.blockData) this.releaseTrackBlock(track); + if (track.blockGap === undefined) track.blockGap = reason; + const take = Math.min(frames, this.g - track.fill); + track.fill += take; frames -= take; + if (track.fill === this.g) this.commitBlock(track); + } + } + private dataFrames(track: Track, bytes: Uint8Array): void { + const frames = bytes.byteLength / track.frameBytes; let offset = 0; let remaining = frames; + while (remaining > 0) { + const take = Math.min(remaining, this.g - track.fill); + if (track.blockGap === undefined) { + const part = bytes.subarray(offset * track.frameBytes, (offset + take) * track.frameBytes); + track.chunks.push(part); track.chunkBytes += part.byteLength; track.blockData += take; + } else this.host.releaseBytes(take * track.frameBytes); // tainted block: data becomes part of the gap + track.fill += take; remaining -= take; offset += take; + if (track.fill === this.g) this.commitBlock(track); + } + } + private totalFrames(track: Track): number { return track.pos + track.fill; } + private waitingWindows(): number { + const starts = new Set(); + for (const track of this.tracks) for (const piece of track.pieces) if (piece.kind === "segment" && (piece.state === "sealed" || piece.state === "queued" || piece.state === "uploading")) starts.add(piece.start); + return starts.size; + } + + // --- public writer API ---------------------------------------------------------------------------- + tryWrite(pcm: Uint8Array, track = 0): boolean { + try { + if (this.mode === "terminal" || this.mode === "finishing" || !(pcm instanceof Uint8Array) || !Number.isInteger(track)) return false; + const t = this.tracks[track]; + if (!t) return false; + // Partial frames carry to the next call (at most frameBytes - 1 bytes). + let data = pcm; let ok = true; + if (t.carry) { + const need = t.frameBytes - t.carry.byteLength; + if (pcm.byteLength < need) { const joined = new Uint8Array(t.carry.byteLength + pcm.byteLength); joined.set(t.carry); joined.set(pcm, t.carry.byteLength); t.carry = joined; return true; } + const frame = new Uint8Array(t.frameBytes); frame.set(t.carry); frame.set(pcm.subarray(0, need), t.carry.byteLength); t.carry = undefined; + ok = this.writeFrames(t, frame); + data = pcm.subarray(need); + } + const whole = data.byteLength - (data.byteLength % t.frameBytes); + if (data.byteLength !== whole) t.carry = data.slice(whole); + if (!whole) return ok; + return this.writeFrames(t, data.subarray(0, whole)) && ok; + } catch { return false; } + } + private writeFrames(t: Track, data: Uint8Array): boolean { + if (this.mode === "terminal" || this.mode === "finishing") return false; + const whole = data.byteLength; const frames = whole / t.frameBytes; + if (this.mode === "pending") { + const state = this.host.runtimeState(); + if (state === "ready") { + const again = this.init.readmit(); + if (again.reason !== "queued") { this.admissionReason = again.reason; this.discardAll(); this.terminal(again.reason); this.closeOnce(); return false; } + for (let i = 0; i < this.tracks.length; i++) this.tracks[i].denied = again.trackDenials?.[i]; + this.admissionReason = "queued"; this.mode = "active"; this.epoch = this.host.policyEpoch(); + if (!this.activate()) return false; + } else if (state === "bootstrap_pending") { this.gapFrames(t, frames, "bootstrap_pending"); return false; } + else { this.admissionReason = state as AttachmentAdmission; this.discardAll(); this.terminal(this.admissionReason); this.closeOnce(); return false; } + } + if (this.host.policyEpoch() !== this.epoch) { + this.epoch = this.host.policyEpoch(); + const again = this.init.readmit(); + if (again.reason !== "queued") { this.gapFrames(t, frames, "policy_denied"); this.policyStop(); return false; } + for (let i = 0; i < this.tracks.length; i++) if (again.trackDenials?.[i] && !this.tracks[i].denied) { this.tracks[i].denied = again.trackDenials[i]; this.dropTrack(this.tracks[i]); } + } + if (t.denied) { this.gapFrames(t, frames, "policy_denied"); return false; } + if (this.host.runtimeState() !== "ready" || this.host.stopping()) { this.gapFrames(t, frames, "incomplete"); return false; } + if (this.truncated || this.totalFrames(t) + frames > this.maxFrames || this.totalBytes + t.chunkBytes + whole > this.recordingBytesCap) { + this.truncate(t, frames); return false; + } + if (this.waitingWindows() >= this.windowSegments) { this.gapFrames(t, frames, "queue_full"); return false; } + let accepted = true; + for (let offset = 0; offset < whole; offset += this.host.chunkBytes) { + const slice = data.subarray(offset, Math.min(whole, offset + this.host.chunkBytes)); + if (!this.host.reserveBytes(slice.byteLength)) { this.gapFrames(t, slice.byteLength / t.frameBytes, "queue_full"); accepted = false; continue; } + // Mutable input: copied before the caller can reuse it. + this.dataFrames(t, slice.slice()); + } + return accepted; + } + private truncate(track: Track, frames: number): void { + this.truncated = true; + const ceiling = Math.floor(MAX_RECORDING_MS * this.rate / 1000); + const allowed = Math.max(0, Math.min(frames, ceiling - this.totalFrames(track))); + if (allowed) this.gapFrames(track, allowed, "truncated"); + } + markGap(durationMs: number, reason: AttachmentGapReason = "source_unavailable", track?: number): void { + try { + if (this.mode === "terminal" || this.mode === "finishing" || !Number.isFinite(durationMs) || durationMs <= 0 || !GAP_REASONS.includes(reason)) return; + let frames = Math.ceil(durationMs * this.rate / 1000); frames = Math.ceil(frames / this.g) * this.g; + const targets = track === undefined ? this.tracks : [this.tracks[track]].filter(Boolean); + for (const t of targets) { + const ceiling = Math.floor(MAX_RECORDING_MS * this.rate / 1000); + const allowed = Math.max(0, Math.min(frames, ceiling - this.totalFrames(t))); + if (allowed) this.gapFrames(t, allowed, reason); + } + } catch { /* bounded */ } + } + markPlayout(startMs: number, durationMs: number, evidence: "generated" | "played" | "unknown" | "truncated"): void { + if (this.mode === "terminal" || !Number.isInteger(startMs) || !Number.isInteger(durationMs) || startMs < 0 || durationMs < 1 || + startMs + durationMs > MAX_RECORDING_MS || !["generated", "played", "unknown", "truncated"].includes(evidence)) return; + if (this.playout.length >= MAX_PLAYOUT) return; + this.playout.push({ startMs, durationMs, evidence }); + } + status(): AttachmentRecordingStatus { + let sealed = 0; + for (const track of this.tracks) for (const piece of track.pieces) if (piece.kind === "segment") sealed++; + let owned = 0; for (const track of this.tracks) owned += track.chunkBytes; + for (const track of this.tracks) for (const piece of track.pieces) if (piece.kind === "segment" && piece.state !== "done" && piece.state !== "failed") owned += piece.bytes ?? 0; + return Object.freeze({ state: this.mode === "active" ? "recording" as const : this.mode, reason: (this.terminalReason ?? this.admissionReason) as AttachmentRecordingStatus["reason"], sealed, + uploaded: this.completed, gapMs: this.publishedGapMs, manifestRevision: this.revision, ownedBytes: owned }); + } + + // --- upload pipeline ------------------------------------------------------------------------------ + /** Where the next not-yet-sealed interval of this track starts. */ + private openStart(track: Track): number { return track.runFrames > 0 ? track.runStart : track.pos; } + private frontier(track: Track): number { + for (const piece of track.pieces) if (piece.kind === "segment" && piece.state !== "done" && piece.state !== "failed") return Math.min(piece.start, this.openStart(track)); + return this.openStart(track); + } + /** Releases sealed segments in (start, track ordinal) order once no earlier interval can still appear. */ + private release(): void { + if (this.mode === "terminal" && this.terminalReason !== "policy_denied") return; + for (;;) { + let head: Piece | undefined; + for (const track of this.tracks) for (const piece of track.pieces) { + if (piece.kind === "segment" && piece.state === "sealed" && (!head || piece.start < head.start || (piece.start === head.start && piece.track < head.track))) head = piece; + } + if (!head) return; + const finishing = this.mode === "finishing" || this.finalEnd !== undefined; + if (!finishing) { + for (const track of this.tracks) { + if (track.ordinal === head.track) continue; + const open = this.openStart(track); + if (open < head.start || (open === head.start && track.ordinal < head.track)) return; + } + } + this.enqueueSegment(head); + } + } + private enqueueSegment(piece: Piece): void { + piece.state = "queued"; this.outstanding++; + const controller = new AbortController(); + const job: CoreJob = { + lane: this.recordingId, bytes: piece.bytes ?? 0, controller, + run: signal => this.uploadSegment(piece, signal), + denied: () => this.segmentDenied(piece), + fail: reason => this.segmentSettled(piece, reason), + }; + piece.job = job; piece.bytes = 0; // the manager now owns the reservation and releases it when the job ends + this.host.enqueue(job); + } + private segmentDenied(piece: Piece): AttachmentAdmission | undefined { + if (this.mode === "terminal" && this.terminalReason !== "policy_denied" && this.terminalReason !== "conflict") return "incomplete"; + const track = this.tracks[piece.track]; + if (track?.denied) return "policy_denied"; + const reason = this.host.recheck({ mimeType: "audio/wav", direction: this.trackDirection(piece.track), purpose: this.options.purpose ?? "conversation_recording", + capture: this.options.capture, operation: this.init.sourceKind === "OwnedRecordingFile" ? "import_existing" : this.options.operation, + sourceKind: this.init.sourceKind ?? "Bytes" }, { ignoreSessionExpiry: false }); + return reason; + } + private trackDirection(ordinal: number): AttachmentDirection { + return this.options.layout === "separate_tracks" ? this.tracks[ordinal].spec.direction : "mixed"; + } + private segmentRequest(piece: Piece, index: number): AttachmentUploadRequest | undefined { + const session = this.host.currentSession(); + if (!session || !this.identity) return undefined; + const declared = piece.source instanceof Uint8Array ? piece.source.byteLength : (piece.source as { size: number }).size; + return { schema_version: 1, attachment_id: piece.attachmentId!, occurrence_id: piece.occurrenceId!, + association: { capture_session_id: this.host.sessionId, correlation_nonce: this.host.nonce, + parent: { trace_id: attachmentTraceUuid(this.identity.traceId), span_id: this.identity.spanId } }, + policy_version: session.policy.version, deletion_generation: 0, mime_type: "audio/wav", modality: "audio", + direction: this.trackDirection(piece.track), source_kind: this.init.sourceKind ?? "Bytes", + purpose: this.options.purpose ?? "conversation_recording", declared_bytes: declared, + recording_id: this.recordingId, segment_index: index, + ...(this.options.conversationId ? { conversation_id: this.options.conversationId } : {}), + ...(this.options.turnId ? { turn_id: this.options.turnId } : {}) }; + } + private async uploadSegment(piece: Piece, signal: AbortSignal): Promise { + if (piece.state !== "queued") return; + const reason = this.segmentDenied(piece); + if (reason) { this.segmentSettled(piece, reason); return; } + piece.state = "uploading"; + const index = this.completed; const reused = this.indexReused; + const request = this.segmentRequest(piece, index); + if (!request) { this.segmentSettled(piece, "incomplete"); return; } + const body = JSON.stringify(request); + const outcome = await this.host.runUpload({ body, occurrenceId: piece.occurrenceId!, declaredBytes: request.declared_bytes, source: piece.source!, + recheck: () => this.segmentDenied(piece), + beforeComplete: this.manifestFallback ? async (o, s) => { await this.fallbackManifest(piece, o, s); } : undefined }, signal); + piece.outcome = outcome; + if (outcome.status === "validating") { piece.index = index; this.segmentSettled(piece, "validating"); return; } + if (outcome.status === "conflict" && reused && outcome.upload === undefined) this.closeOnConflict(); + this.segmentSettled(piece, outcome.status); + } + private segmentSettled(piece: Piece, status: AttachmentCompletion["status"]): void { + if (piece.state === "done" || piece.state === "failed") return; + piece.source = undefined; + if (status === "validating") { piece.state = "done"; this.completed++; this.sinceManifest++; this.indexReused = false; } + else { piece.state = "failed"; piece.failure = status; this.indexReused = true; this.host.countDrop(); } + this.outstanding--; + this.maybeManifest(false); + if (this.outstanding === 0) { const waiters = this.idleWaiters; this.idleWaiters = []; for (const w of waiters) w(); } + } + private closeOnConflict(): void { + // BR-4 fallback: a rejected reused index closes the recording with the published prefix plus an incomplete gap. + this.discardAll(); this.terminal("conflict"); + } + + // --- manifest ------------------------------------------------------------------------------------- + private settledEnd(): number { + let end = Infinity; + for (const track of this.tracks) end = Math.min(end, this.frontier(track)); + if (!Number.isFinite(end)) return 0; + for (let changed = true; changed;) { + changed = false; + for (const track of this.tracks) for (const piece of track.pieces) { + if (piece.kind === "segment" && piece.start < end && piece.end > end) { end = piece.start; changed = true; } + } + } + return end; + } + private latestCapability(): UploadOutcome | undefined { + let best: Piece | undefined; + for (const track of this.tracks) for (const piece of track.pieces) if (piece.state === "done" && piece.outcome?.upload && (!best || (piece.index ?? -1) > (best.index ?? -1))) best = piece; + return best?.outcome; + } + private capabilities(): UploadOutcome[] { + const all: Piece[] = []; + for (const track of this.tracks) for (const piece of track.pieces) if (piece.state === "done" && piece.outcome?.upload) all.push(piece); + return all.sort((a, b) => (b.index ?? 0) - (a.index ?? 0)).map(piece => piece.outcome!); + } + private ms(frames: number, final: boolean): number { + return final && this.finalEnd !== undefined && frames === this.finalEnd ? attachmentFramesToMs(frames, this.rate, true) : attachmentFramesToMs(frames, this.rate); + } + /** Cumulative, prefix-monotonic manifest over [0, end): earlier cut points always split later gaps. */ + buildManifest(end: number, final: boolean, extra?: Piece): AttachmentRecordingManifest { + const separate = this.options.layout === "separate_tracks"; + const segments: AttachmentRecordingManifest["segments"] = []; + const gaps: AttachmentRecordingManifest["gaps"] = []; + const cuts = [...this.cuts].sort((a, b) => a - b); + for (const track of this.tracks) { + const trackId = separate ? track.manifestId : undefined; + for (const piece of track.pieces) { + if (piece.start >= end) continue; + const pieceEnd = Math.min(piece.end, end); + const done = piece.kind === "segment" && (piece.state === "done" || piece === extra); + if (done && piece.end <= end) { + segments.push({ occurrence_id: piece.occurrenceId!, index: piece === extra ? this.completed : piece.index!, start_ms: this.ms(piece.start, final), + duration_ms: this.ms(piece.end, final) - this.ms(piece.start, final), ...(trackId ? { track_id: trackId } : {}) }); + continue; + } + const reason: AttachmentGapReason = piece.kind === "gap" ? piece.reason! : + piece.failure === "policy_denied" ? "policy_denied" : "incomplete"; + let start = piece.start; + for (const cut of [...cuts.filter(c => c > piece.start && c < pieceEnd), pieceEnd]) { + const a = this.ms(start, final); const b = this.ms(cut, final); + if (b > a) gaps.push({ start_ms: a, duration_ms: b - a, reason, ...(trackId ? { track_id: trackId } : {}) }); + start = cut; + } + } + } + segments.sort((a, b) => a.index - b.index); + const duration = this.ms(end, final); + const playout = this.playout.filter(p => p.startMs + p.durationMs <= duration).map(p => ({ start_ms: p.startMs, duration_ms: p.durationMs, evidence: p.evidence })); + const all = this.options.tracks.flatMap(t => [...t.channels]); + return { schema_version: 1, recording_id: this.recordingId, timebase: "session_relative_ms", sample_format: "s16le", sample_rate_hz: this.rate, + channels: all.length, codec: "pcm", container: "wav", layout: this.options.layout, + tracks: this.manifestTracks.map(t => ({ ...t, channels: [...t.channels] })), segments, gaps, playout, + capture_complete: final && gaps.length === 0 && segments.length > 0, duration_ms: duration }; + } + private maybeManifest(force: boolean): void { + if (this.manifestQueued || this.manifestDisabled || this.manifestFallback || this.finalEnd !== undefined) return; + if (this.mode === "terminal") return; + const end = this.settledEnd(); + if (end <= this.lastPutEnd || !this.latestCapability()) return; + const due = force || this.sinceManifest >= 4 || this.ms(end, false) - this.ms(this.lastPutEnd, false) >= 30000; + if (!due) return; + this.manifestQueued = true; this.outstanding++; + const controller = new AbortController(); + this.host.enqueue({ lane: this.recordingId, bytes: 0, controller, + run: async signal => { try { await this.putManifest(this.settledEnd(), false, signal); } finally { this.manifestQueued = false; this.manifestJobDone(); } }, + denied: () => this.mode === "terminal" && this.terminalReason !== "policy_denied" ? "incomplete" : undefined, + fail: () => { this.manifestQueued = false; this.manifestJobDone(); } }); + } + private manifestJobDone(): void { + this.outstanding--; + if (this.outstanding === 0) { const waiters = this.idleWaiters; this.idleWaiters = []; for (const w of waiters) w(); } + } + private async putManifest(end: number, final: boolean, signal: AbortSignal, extra?: Piece, holder?: UploadOutcome): Promise { + if (end <= 0 && !final) return false; + const manifest = this.buildManifest(end, final, extra); + if (!validAttachmentManifest(manifest)) { this.manifestDisabled = true; return false; } + const body = JSON.stringify(manifest); + if (new TextEncoder().encode(body).byteLength > MAX_MANIFEST_BYTES) { + // Closes as partial at the last successful revision; the writer stops retaining. + this.manifestDisabled = true; this.truncated = true; return false; + } + const revision = this.revision + 1; + const key = attachmentIdempotencyKey("manifest", this.recordingId, revision); + this.cuts.push(end); + const holders = holder ? [holder] : this.capabilities(); + for (const candidate of holders) { + if (signal.aborted) return false; + let result = await this.host.putManifest(candidate.upload!.write_capability, this.recordingId, body, key, signal, false); + if (result === "credential_scope_unsupported" || result === "expired") { + if (await this.host.renewUpload(candidate, signal)) result = await this.host.putManifest(candidate.upload!.write_capability, this.recordingId, body, key, signal, true); + } + if (result === "ok") { + this.revision = revision; this.lastPutEnd = end; this.sinceManifest = 0; + this.publishedSegments = manifest.segments.length; this.publishedComplete = manifest.capture_complete; + this.publishedGapMs = manifest.gaps.reduce((sum, gap) => sum + gap.duration_ms, 0); + return true; + } + if (result === "credential_scope_unsupported" || result === "policy_denied" || result === "conflict") { + // BR-7 fallback: put the manifest before complete for the remaining segments. + if (!final && !this.manifestFallback && !holder) { this.manifestFallback = true; return false; } + if (holder) throw new Error("manifest"); + continue; + } + if (result === "expired") continue; // an older completed segment's capability may still be valid (design 3.3) + return false; + } + return false; + } + private async fallbackManifest(piece: Piece, outcome: UploadOutcome, signal: AbortSignal): Promise { + piece.outcome = outcome; + // Prospectively include this segment; its own capability authorizes the PUT. + const savedState = piece.state; piece.state = "done"; piece.index = this.completed; + const end = this.settledEnd(); + piece.state = savedState; + let ok = false; + try { ok = await this.putManifest(end, false, signal, piece, outcome); } catch { ok = false; } + piece.index = undefined; + if (!ok) { this.manifestDisabled = true; throw new Error("manifest"); } + } + + // --- finish / abort ------------------------------------------------------------------------------ + private discardAll(): void { + for (const track of this.tracks) { + if (track.chunkBytes) this.host.releaseBytes(track.chunkBytes); + track.chunks = []; track.chunkBytes = 0; track.blockData = 0; track.runFrames = 0; track.carry = undefined; + for (const piece of track.pieces) { + if (piece.kind !== "segment") continue; + if (piece.state === "sealed") { if (piece.bytes) this.host.releaseBytes(piece.bytes); piece.bytes = 0; piece.state = "failed"; piece.failure = "incomplete"; piece.source = undefined; } + else if (piece.state === "queued" || piece.state === "uploading") piece.job?.controller.abort(); + } + } + } + private dropTrack(track: Track): void { + if (track.chunkBytes) this.host.releaseBytes(track.chunkBytes); + track.chunks = []; track.chunkBytes = 0; track.blockData = 0; + if (track.runFrames) { this.appendGap(track, "policy_denied", track.runFrames); track.runFrames = 0; } + for (const piece of track.pieces) if (piece.kind === "segment" && (piece.state === "sealed")) { + if (piece.bytes) this.host.releaseBytes(piece.bytes); piece.bytes = 0; piece.state = "failed"; piece.failure = "policy_denied"; piece.source = undefined; + } + } + private policyStop(): void { + // Tightening: retained bytes are freed and the remaining timeline becomes a policy_denied gap. + for (const track of this.tracks) { + if (track.blockData) this.releaseTrackBlock(track); + if (track.runFrames) { const frames = track.runFrames; this.host.releaseBytes(track.chunkBytes); track.chunks = []; track.chunkBytes = 0; track.runFrames = 0; + track.pieces.push({ kind: "gap", track: track.ordinal, start: track.runStart, end: track.runStart + frames, reason: "policy_denied" }); track.runStart = track.pos; } + for (const piece of track.pieces) if (piece.kind === "segment" && piece.state === "sealed") { + if (piece.bytes) this.host.releaseBytes(piece.bytes); piece.bytes = 0; piece.state = "failed"; piece.failure = "policy_denied"; piece.source = undefined; + } + } + this.terminal("policy_denied"); + } + private sealAll(): void { + for (const track of this.tracks) { + if (track.carry) track.carry = undefined; + if (track.fill) { + const frames = track.fill; + if (track.blockGap === undefined) { track.runFrames += frames; track.blockData = 0; track.pos += frames; track.fill = 0; } + else { this.closeRun(track); this.appendGap(track, track.blockGap, frames); track.pos += frames; track.fill = 0; track.blockGap = undefined; } + } + this.closeRun(track); + } + const end = Math.max(0, ...this.tracks.map(track => track.pos)); + for (const track of this.tracks) if (track.pos < end) { this.appendGap(track, track.denied ? "policy_denied" : "source_unavailable", end - track.pos); track.pos = end; } + this.finalEnd = end; + } + private idle(deadline: number): Promise { + if (this.outstanding === 0) return Promise.resolve(); + return new Promise(resolve => { + const done = () => { clearTimeout(timer); resolve(); }; + // The caller awaits finish(): a ref'd timer bounded by its own deadline gives it the grace period. + const timer = setTimeout(() => { this.idleWaiters = this.idleWaiters.filter(w => w !== done); resolve(); }, Math.max(0, deadline - this.host.clock.now())); + this.idleWaiters.push(done); + }); + } + finish(timeoutMs?: number): Promise { + if (!this.resultPromise) this.resultPromise = this.finishInternal(timeoutMs).catch(() => this.result("incomplete")); + return this.resultPromise; + } + private async finishInternal(timeoutMs?: number): Promise { + const budget = Math.min(Number.isFinite(timeoutMs) && timeoutMs! >= 0 ? timeoutMs! : this.host.finalizationTimeoutMs, this.host.finalizationTimeoutMs); + const start = this.host.clock.now(); const deadline = start + budget; + if (this.mode === "pending" || (this.mode === "terminal" && this.revision === 0 && !this.latestCapability() && this.admissionReason !== "queued")) { + const denied = this.admissionReason !== "queued"; + this.discardAll(); this.terminal(this.terminalReason ?? this.admissionReason); this.closeOnce(); + return this.result(denied ? "denied" : undefined); + } + const aborted = this.terminalReason === "incomplete" && this.mode === "terminal"; + if (aborted) { this.closeOnce(); return this.result(); } + const wasTerminal = this.mode === "terminal"; + if (!wasTerminal) this.mode = "finishing"; + this.sealAll(); + this.release(); + const reserve = Math.min(2000, budget / 4); + await this.idle(deadline - reserve); + // At the deadline: queued and in-flight segments are dropped and become incomplete gaps. + for (const track of this.tracks) for (const piece of track.pieces) { + if (piece.kind === "segment" && piece.state !== "done" && piece.state !== "failed") { + piece.job?.controller.abort(); + if (piece.state === "sealed" && piece.bytes) { this.host.releaseBytes(piece.bytes); piece.bytes = 0; } + piece.state = "failed"; piece.failure = "incomplete"; piece.source = undefined; + } + } + if (this.latestCapability() && !this.manifestDisabled) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), Math.max(1, deadline - this.host.clock.now())); + try { await this.raceDeadline(this.putManifest(this.finalEnd ?? 0, true, controller.signal), deadline); } + catch { /* bounded */ } finally { clearTimeout(timer); controller.abort(); } + } + this.terminal(this.terminalReason ?? "incomplete"); + this.closeOnce(); + return this.result(); + } + private raceDeadline(promise: Promise, deadline: number): Promise { + return new Promise(resolve => { + const timer = setTimeout(() => resolve(undefined), Math.max(0, deadline - this.host.clock.now())); + promise.then(value => { clearTimeout(timer); resolve(value); }, () => { clearTimeout(timer); resolve(undefined); }); + }); + } + private result(forced?: "denied" | "incomplete"): AttachmentRecordingResult { + const state: AttachmentRecordingResult["state"] = forced === "denied" || (this.admissionReason !== "queued" && this.revision === 0 && !this.completed) ? "denied" : + this.revision === 0 ? "unpublished" : this.publishedComplete ? "published" : "partial"; + const reason = state === "published" ? undefined : String(this.terminalReason ?? this.admissionReason); + return Object.freeze({ recordingId: this.recordingId, state, ...(reason ? { reason } : {}), segmentsPublished: this.publishedSegments, + gapMs: this.publishedGapMs, captureComplete: this.publishedComplete, manifestRevision: this.revision }); + } + abort(): void { + try { + if (this.mode === "terminal" && this.closed) return; + this.discardAll(); this.terminal("incomplete"); + if (!this.resultPromise) this.resultPromise = Promise.resolve(this.result()); + this.closeOnce(); + } catch { /* abort never throws */ } + } + /** Manager shutdown/fork-style invalidation: finish with the remaining budget. */ + shutdown(timeoutMs: number): Promise { return this.finish(timeoutMs); } + + /** Owned WAV (tier wav_recording): segments are pre-built lease ranges. */ + loadFileSegments(segments: readonly RecordingFileSegment[], totalFrames: number): void { + if (this.mode !== "active") return; + for (const segment of segments) { + const track = this.tracks[segment.track]; + track.pieces.push({ kind: "segment", track: segment.track, start: segment.startFrame, end: segment.endFrame, source: segment.source, bytes: 0, + state: "sealed", occurrenceId: uuid(), attachmentId: uuid() }); + } + for (const track of this.tracks) track.pos = totalFrames; + this.finalEnd = totalFrames; this.mode = "finishing"; + this.release(); + } + /** Owned WAV: wait for every segment and PUT the final manifest; cancellation drops the rest. */ + async completeFile(signal: AbortSignal): Promise { + await new Promise(resolve => { + if (this.outstanding === 0 || signal.aborted) { resolve(); return; } + const done = () => { signal.removeEventListener("abort", done); resolve(); }; + this.idleWaiters.push(done); signal.addEventListener("abort", done, { once: true }); + }); + if (signal.aborted) { this.discardAll(); this.terminal("incomplete"); this.closeOnce(); return "incomplete"; } + let failed: AttachmentCompletion["status"] | undefined; + for (const track of this.tracks) for (const piece of track.pieces) if (piece.state === "failed") failed = failed ?? piece.failure; + if (this.latestCapability()) await this.putManifest(this.finalEnd ?? 0, true, signal); + this.terminal(failed ?? "incomplete"); this.closeOnce(); + return this.revision === 0 ? (failed ?? "incomplete") : failed ?? (this.publishedComplete ? "validating" : "incomplete"); + } + /** @internal capture policy snapshot for owned file admission. */ + get capture(): AttachmentCapturePolicy | undefined { return this.options.capture; } +} diff --git a/typescript/packages/fi-core/src/attachments/stream.ts b/typescript/packages/fi-core/src/attachments/stream.ts new file mode 100644 index 000000000..733c52f05 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/stream.ts @@ -0,0 +1,44 @@ +import type { AttachmentAdmission, AttachmentHandle, AttachmentStream } from "./types"; + +/** @internal Mutable state the manager keeps (and sweeps) for one consumed stream. */ +export interface StreamState { + reason: AttachmentAdmission; + readonly attachmentId?: string; + readonly occurrenceId?: string; + chunks: Uint8Array[]; + length: number; + readonly maxBytes: number; + /** Bytes still reserved for this stream (the whole maxBytes until finish). */ + reserved: number; + lastWrite: number; + readonly idleTtl: number; + readonly epoch: number; + open: boolean; + handle?: AttachmentHandle; +} +/** @internal */ +export interface StreamHost { + streamWrite(state: StreamState, chunk: Uint8Array): boolean; + streamFinish(state: StreamState): AttachmentHandle; + streamAbort(state: StreamState, reason: "incomplete" | "policy_denied" | "too_large"): void; +} +/** + * Single-object upload of chunks the application ALREADY consumed (design section 6). The core never pulls, clones + * or tees a stream; adapters forward fulfilled reads to tryWrite. + */ +export class AttachmentConsumedStream implements AttachmentStream { + constructor(private readonly host: StreamHost | undefined, /** @internal */ readonly state: StreamState) {} + get reason(): AttachmentAdmission { return this.state.reason; } + get attachmentId(): string | undefined { return this.state.reason === "queued" || this.state.handle ? this.state.attachmentId : undefined; } + get occurrenceId(): string | undefined { return this.state.reason === "queued" || this.state.handle ? this.state.occurrenceId : undefined; } + tryWrite(chunk: Uint8Array): boolean { + try { return !!this.host && this.state.open && this.host.streamWrite(this.state, chunk); } catch { return false; } + } + finish(): AttachmentHandle { + if (!this.host) return this.state.handle!; + return this.host.streamFinish(this.state); + } + abort(reason: "incomplete" | "policy_denied" = "incomplete"): void { + try { if (this.host && this.state.open) this.host.streamAbort(this.state, reason === "policy_denied" ? "policy_denied" : "incomplete"); } catch { /* never throws */ } + } +} diff --git a/typescript/packages/fi-core/src/attachments/transport-v1.ts b/typescript/packages/fi-core/src/attachments/transport-v1.ts index e11333c7f..662f7d660 100644 --- a/typescript/packages/fi-core/src/attachments/transport-v1.ts +++ b/typescript/packages/fi-core/src/attachments/transport-v1.ts @@ -1,82 +1,210 @@ import { context } from "@opentelemetry/api"; import { suppressTracing } from "@opentelemetry/core"; import { validAttachmentUuid } from "./correlation"; -import type { AttachmentAdmission, AttachmentBootstrap, AttachmentSession, AttachmentTransport, AttachmentUpload, AttachmentUploadRequest } from "./types"; +import type { + AttachmentAdmission, AttachmentBootstrap, AttachmentCompletionStatus, AttachmentContentSource, AttachmentImportJob, AttachmentPacer, + AttachmentSession, AttachmentTransport, AttachmentTransportV1, AttachmentUpload, AttachmentUploadRequest, +} from "./types"; +export type AttachmentErrorCode = AttachmentAdmission | AttachmentCompletionStatus | "failed"; /** Safe errors have only fixed codes, never response bodies, credentials or URLs. */ export class AttachmentTransportError extends Error { - constructor(readonly code: AttachmentAdmission | "failed", readonly retryable = false, readonly retryAfterMs = 0) { + constructor(readonly code: AttachmentErrorCode, readonly retryable = false, readonly retryAfterMs = 0, readonly httpStatus = 0) { super(code); this.name = "AttachmentTransportError"; } } export interface AttachmentHttpRequest { - url: string; method: "POST" | "PUT"; headers: Record; - body: string | Uint8Array | Blob; signal: AbortSignal; + url: string; method: "GET" | "POST" | "PUT"; headers: Record; + body?: string | Uint8Array | Blob | AttachmentContentSource; signal: AbortSignal; + /** Content PUT only: per-slice stall timeout and pacing. */ + stallTimeoutMs?: number; pacer?: AttachmentPacer; } export interface AttachmentHttpResponse { status: number; body: Uint8Array; retryAfter?: string } export type AttachmentHttpClient = (request: AttachmentHttpRequest) => Promise; +export const ATTACHMENT_CHUNK_BYTES = 256 * 1024; +const SUCCESS_BODY = 65536; +const ERROR_BODY = 4096; +const TOKEN = /^[a-zA-Z0-9_-]{1,128}$/; +const REASONS = new Set(["queued", "disabled", "bootstrap_pending", "policy_denied", "credential_scope_unsupported", "source_unavailable", "unsampled", + "too_large", "queue_full", "unsupported_version", "endpoint_unconfigured", "scope_unresolved", "correlation_unavailable", "redaction_unavailable", + "recording_disabled", "binding_pending", "not_found", "conflict", "throttled", "validation_failed", "expired", "deleted", "incomplete", "ready"]); +const STATES = new Set(["pending_upload", "importing", "validating", "ready", "failed", "rejected", "expired", "deleted"]); + +/** Strict JSON: duplicate members and non-JSON numbers are rejected (F01 validate.py strict_json). */ +export function attachmentStrictJson(text: string): unknown { + let i = 0; + const ws = () => { while (i < text.length && " \t\n\r".includes(text[i])) i++; }; + const fail = (): never => { throw new SyntaxError("invalid JSON"); }; + const str = (): string => { + const start = i; i++; + for (; i < text.length; i++) { + const c = text[i]; + if (c === "\\") { i++; continue; } + if (c === '"') { i++; return JSON.parse(text.slice(start, i)); } + if (c < " ") fail(); + } + return fail(); + }; + const value = (depth: number): unknown => { + if (depth > 64) fail(); + ws(); const c = text[i]; + if (c === "{") { + i++; const out: Record = {}; const keys = new Set(); ws(); + if (text[i] === "}") { i++; return out; } + for (;;) { + ws(); if (text[i] !== '"') fail(); const key = str(); if (keys.has(key)) fail(); keys.add(key); + ws(); if (text[i] !== ":") fail(); i++; + Object.defineProperty(out, key, { value: value(depth + 1), enumerable: true, writable: true, configurable: true }); + ws(); if (text[i] === ",") { i++; continue; } if (text[i] === "}") { i++; return out; } fail(); + } + } + if (c === "[") { + i++; const out: unknown[] = []; ws(); + if (text[i] === "]") { i++; return out; } + for (;;) { out.push(value(depth + 1)); ws(); if (text[i] === ",") { i++; continue; } if (text[i] === "]") { i++; return out; } fail(); } + } + if (c === '"') return str(); + const m = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?|^true|^false|^null/.exec(text.slice(i, i + 400)); + if (!m) return fail(); + i += m[0].length; return JSON.parse(m[0]); + }; + const result = value(0); ws(); if (i !== text.length) fail(); + return result; +} + +const REFINE: Readonly> = { + 400: ["unsupported_version"], 401: ["credential_scope_unsupported", "expired"], + 403: ["policy_denied", "redaction_unavailable", "recording_disabled", "correlation_unavailable", "credential_scope_unsupported"], + 404: ["not_found", "scope_unresolved"], 409: ["conflict", "deleted", "expired", "policy_denied"], 410: ["expired", "deleted"], 413: ["too_large"], +}; +const DEFAULT_REASON: Readonly> = { + 400: "unsupported_version", 401: "credential_scope_unsupported", 403: "policy_denied", 404: "not_found", 409: "conflict", 410: "expired", 413: "too_large", 429: "throttled", +}; +/** Integer delta-seconds only, capped at 5 s; an HTTP-date is treated as 1 s (design 0.4). */ +export function attachmentRetryAfterMs(value: string | undefined, status: number): number { + if (value === undefined || value === "") return status === 429 ? 1000 : 0; + const trimmed = String(value).trim(); + if (/^\d{1,9}$/.test(trimmed)) return Math.min(Number(trimmed) * 1000, 5000); + return 1000; +} +/** One HTTP status -> reason table for every route (design 0.5). */ +export function attachmentHttpError(status: number, body?: Uint8Array, retryAfter?: string): AttachmentTransportError { + let code: AttachmentErrorCode = DEFAULT_REASON[status] ?? "incomplete"; + const refine = REFINE[status]; + if (refine && body && body.byteLength && body.byteLength <= ERROR_BODY) { + try { + const parsed = attachmentStrictJson(new TextDecoder("utf-8", { fatal: true }).decode(body)) as Record; + if (object(parsed) && Object.keys(parsed).every(key => key === "code" || key === "request_id") && validAttachmentUuid(parsed.request_id) && + typeof parsed.code === "string" && REASONS.has(parsed.code) && refine.includes(parsed.code)) code = parsed.code as AttachmentErrorCode; + } catch { /* Unparseable error bodies are ignored. */ } + } + const retryable = status === 429 || status === 408 || status === 500 || status === 502 || status === 503 || status === 504; + return new AttachmentTransportError(code, retryable, retryable ? attachmentRetryAfterMs(retryAfter, status) : 0, status); +} + +/** Idempotency keys (design 0.6). F01 Token: ^[a-zA-Z0-9_-]{1,128}$. */ +export function attachmentIdempotencyKey(route: "bootstrap" | "upload" | "complete" | "abort" | "manifest" | "import", id: string, sequence?: number): string { + const hex = String(id).replace(/-/g, ""); + const prefix = { bootstrap: "bs", upload: "up", complete: "cp", abort: "ab", manifest: "mf", import: "im" }[route]; + return sequence === undefined ? `${prefix}-${hex}` : `${prefix}-${hex}-${sequence}`; +} + +async function readBounded(response: Response, limit: number): Promise { + const reader = response.body?.getReader(); + const chunks: Uint8Array[] = []; let size = 0; + if (reader) { + try { + for (;;) { + const { value, done } = await reader.read(); if (done) break; + size += value.byteLength; + if (size > limit) return undefined; + chunks.push(value); + } + } finally { await reader.cancel().catch(() => undefined); } + } + const body = new Uint8Array(size); let offset = 0; + for (const chunk of chunks) { body.set(chunk, offset); offset += chunk.length; } + return body; +} +/** Materialize an in-memory content source (browser fetch has no streaming request bodies everywhere). */ +export async function attachmentContentBytes(source: AttachmentContentSource, signal: AbortSignal): Promise { + if (source instanceof Uint8Array || (typeof Blob !== "undefined" && source instanceof Blob)) return source; + const ranged = source as Exclude; + const out = new Uint8Array(ranged.size); + for (let offset = 0; offset < ranged.size; offset += ATTACHMENT_CHUNK_BYTES) { + if (signal.aborted) throw new AttachmentTransportError("incomplete"); + await ranged.check(); + const part = await ranged.read(offset, Math.min(ATTACHMENT_CHUNK_BYTES, ranged.size - offset)); + if (part.byteLength !== Math.min(ATTACHMENT_CHUNK_BYTES, ranged.size - offset)) throw new AttachmentTransportError("source_unavailable"); + out.set(part, offset); + } + await ranged.check(); + return out; +} /** No cookies, redirects, arbitrary destination paths or unbounded JSON reads. */ export const attachmentFetchClient: AttachmentHttpClient = async request => { try { + const body = request.body === undefined || typeof request.body === "string" ? request.body : await attachmentContentBytes(request.body, request.signal); + if (body !== undefined && typeof body !== "string" && request.pacer) await request.pacer.take(body instanceof Uint8Array ? body.byteLength : body.size, request.signal); const response = await fetch(request.url, { - method: request.method, headers: request.headers, body: request.body, + method: request.method, headers: request.headers, body: body as unknown as RequestInit["body"], signal: request.signal, redirect: "error", credentials: "omit", }); - if (response.status < 200 || response.status >= 300) { - await response.body?.cancel().catch(() => undefined); + const ok = response.status >= 200 && response.status < 300; + const data = await readBounded(response, ok ? SUCCESS_BODY : ERROR_BODY); + if (!data) { + if (ok) throw new AttachmentTransportError("unsupported_version"); return { status: response.status, body: new Uint8Array(), retryAfter: response.headers.get("retry-after") ?? undefined }; } - const reader = response.body?.getReader(); - const chunks: Uint8Array[] = []; let size = 0; - if (reader) { - try { - for (;;) { - const { value, done } = await reader.read(); if (done) break; - size += value.byteLength; - if (size > 65536) throw new AttachmentTransportError("failed"); - chunks.push(value); - } - } finally { await reader.cancel().catch(() => undefined); } - } - const body = new Uint8Array(size); let offset = 0; - for (const chunk of chunks) { body.set(chunk, offset); offset += chunk.length; } - return { status: response.status, body, retryAfter: response.headers.get("retry-after") ?? undefined }; + return { status: response.status, body: data, retryAfter: response.headers.get("retry-after") ?? undefined }; } catch (error) { if (error instanceof AttachmentTransportError) throw error; - throw new AttachmentTransportError("failed", !request.signal.aborted); + // A refused redirect surfaces as a network TypeError; the abort signal tells cancellation apart. + throw new AttachmentTransportError("incomplete", !request.signal.aborted); } }; function object(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } -export function validAttachmentSession(value: unknown, sessionId: string): value is AttachmentSession { +const LIMIT_MAX: Readonly> = { max_upload_bytes: 52428800, segment_bytes: 8388608, segment_duration_ms: 30000, + recording_bytes: 1073741824, recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, + reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, + upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, + grant_ttl_seconds: 300, revocation_check_seconds: 5 }; +export function validAttachmentSession(value: unknown, sessionId: string, now = Date.now()): value is AttachmentSession { if (!object(value) || value.schema_version !== 1 || value.capture_session_id !== sessionId || !object(value.scope) || ![value.scope.organization_id, value.scope.workspace_id, value.scope.project_id].every(validAttachmentUuid) || - !Number.isFinite(Date.parse(value.expires_at)) || Date.parse(value.expires_at) <= Date.now() || + !Number.isFinite(Date.parse(value.expires_at)) || Date.parse(value.expires_at) <= now || !Array.isArray(value.transfer_modes) || value.transfer_modes.length !== 1 || value.transfer_modes[0] !== "authenticated_stream") return false; const p = value.policy; - if (!object(p) || typeof p.version !== "string" || p.version.trim() !== p.version || !/^[a-zA-Z0-9_-]{1,128}$/.test(p.version) || + if (!object(p) || typeof p.version !== "string" || p.version.trim() !== p.version || !TOKEN.test(p.version) || !["enabled", "require_media_redaction", "require_metadata_removal", "allow_unredacted_media_with_text_pii", "record_audio", "record_video"].every(key => typeof p[key] === "boolean") || !Array.isArray(p.allowed_sources) || p.allowed_sources.length > 6 || !p.allowed_sources.every((s: unknown) => ["Bytes", "Blob", "RemoteUrl", "LocalFile", "OwnedRecordingFile", "ProviderFile"].includes(s as string)) || !Array.isArray(p.allowed_modalities) || p.allowed_modalities.length > 5 || !p.allowed_modalities.every((m: unknown) => ["audio", "image", "video", "document", "file"].includes(m as string))) return false; - const limits: Record = { max_upload_bytes: 52428800, segment_bytes: 8388608, segment_duration_ms: 30000, - recording_bytes: 1073741824, recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, - reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, - upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, - grant_ttl_seconds: 300, revocation_check_seconds: 5 }; - return object(value.limits) && Object.entries(limits).every(([key, max]) => Number.isInteger(value.limits[key]) && value.limits[key] > 0 && value.limits[key] <= max); -} -export function validAttachmentUpload(value: unknown, request: AttachmentUploadRequest): value is AttachmentUpload { + return object(value.limits) && Object.entries(LIMIT_MAX).every(([key, max]) => Number.isInteger(value.limits[key]) && value.limits[key] > 0 && value.limits[key] <= max); +} +export function validAttachmentUpload(value: unknown, request: AttachmentUploadRequest, now = Date.now()): value is AttachmentUpload { return object(value) && validAttachmentUuid(value.upload_id) && value.attachment_id === request.attachment_id && value.occurrence_id === request.occurrence_id && value.state === "pending_upload" && value.reserved_bytes === request.declared_bytes && - Number.isFinite(Date.parse(value.expires_at)) && Date.parse(value.expires_at) > Date.now() && value.transfer_mode === "authenticated_stream" && typeof value.write_capability === "string" && value.write_capability.trim() === value.write_capability && /^[a-zA-Z0-9_.-]{16,4096}$/.test(value.write_capability); + Number.isFinite(Date.parse(value.expires_at)) && Date.parse(value.expires_at) > now && value.transfer_mode === "authenticated_stream" && + typeof value.write_capability === "string" && value.write_capability.trim() === value.write_capability && /^[a-zA-Z0-9_.-]{16,4096}$/.test(value.write_capability); +} +function validStatus(value: unknown, id: string, key: "upload_id" | "job_id", state?: string): value is { state: string; reason: string } { + return object(value) && Object.keys(value).length === 3 && value[key] === id && typeof value.state === "string" && STATES.has(value.state) && + typeof value.reason === "string" && REASONS.has(value.reason) && (state === undefined || value.state === state); } export interface AttachmentHttpTransportOptions { endpoint: string; /** Explicitly supplied credential/capability. Never read environment in browser code. */ headers: Readonly>; client?: AttachmentHttpClient; + /** Per-slice stall timeout for content PUTs (default 2 s). */ + stallTimeoutMs?: number; } -export function createAttachmentHttpTransport(options: AttachmentHttpTransportOptions): AttachmentTransport { +/** Symbol under which a legacy-shaped transport carries its full versioned surface. */ +export const ATTACHMENT_TRANSPORT_V1: unique symbol = Symbol.for("@traceai/fi-core/attachments/transport-v1"); + +/** The full design section 9 surface over HTTPS. */ +export function createAttachmentHttpTransportV1(options: AttachmentHttpTransportOptions): AttachmentTransportV1 { const url = new URL(options.endpoint); if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash || url.pathname !== "/") { throw new AttachmentTransportError("endpoint_unconfigured"); @@ -88,52 +216,179 @@ export function createAttachmentHttpTransport(options: AttachmentHttpTransportOp if (allowed.has(name.toLowerCase()) && typeof value === "string" && value.length <= 8192 && !/[\r\n]/.test(value)) headers[name.toLowerCase()] = value; } const client = options.client ?? attachmentFetchClient; - async function send(path: string, body: string | Uint8Array | Blob, signal: AbortSignal, expected: number, key?: string, capability?: string): Promise { + const stall = Number.isInteger(options.stallTimeoutMs) && options.stallTimeoutMs! > 0 ? options.stallTimeoutMs! : 2000; + const bootstraps = new Map(); + async function send(method: "GET" | "POST" | "PUT", path: string, body: AttachmentHttpRequest["body"], signal: AbortSignal, expected: number, + extra: { key?: string; capability?: string; binary?: boolean; pacer?: AttachmentPacer } = {}): Promise { if (signal.aborted) throw new AttachmentTransportError("incomplete"); - const binary = typeof body !== "string"; let response: AttachmentHttpResponse; + const auth = extra.capability !== undefined ? { authorization: `Bearer ${extra.capability}` } : headers; try { response = await context.with(suppressTracing(context.active()), () => client({ - url: origin + path, method: binary ? "PUT" : "POST", signal, body, - headers: { ...(capability ? { authorization: `Bearer ${capability}` } : headers), "content-type": binary ? "application/octet-stream" : "application/json", "cache-control": "no-store", ...(key ? { "idempotency-key": key } : {}) }, + url: origin + path, method, signal, body, + headers: { ...auth, ...(body === undefined ? {} : { "content-type": extra.binary ? "application/octet-stream" : "application/json" }), + "cache-control": "no-store", ...(extra.key ? { "idempotency-key": extra.key } : {}) }, + ...(extra.binary ? { stallTimeoutMs: stall, pacer: extra.pacer } : {}), })); - } catch (error) { throw error instanceof AttachmentTransportError ? error : new AttachmentTransportError("failed", !signal.aborted); } + } catch (error) { + if (error instanceof AttachmentTransportError) throw error.code === "failed" ? new AttachmentTransportError("incomplete", error.retryable) : error; + throw new AttachmentTransportError("incomplete", !signal.aborted); + } if (response.status !== expected) { - const code = response.status === 401 || response.status === 403 ? "credential_scope_unsupported" : response.status === 413 ? "too_large" : response.status === 409 ? "policy_denied" : response.status === 410 ? "expired" : "failed"; - const retryable = response.status === 429 || response.status >= 500 && response.status <= 599; - const after = Number(response.retryAfter); - throw new AttachmentTransportError(code, retryable, Number.isFinite(after) && after >= 0 ? Math.min(after * 1000, 2000) : 0); + if (response.status >= 200 && response.status < 300) throw new AttachmentTransportError("unsupported_version"); + throw attachmentHttpError(response.status, response.body, response.retryAfter); } - if (response.body.byteLength > 65536) throw new AttachmentTransportError("failed"); - try { return JSON.parse(new TextDecoder().decode(response.body)); } - catch { throw new AttachmentTransportError("failed"); } + if (response.body.byteLength > SUCCESS_BODY) throw new AttachmentTransportError("unsupported_version"); + if (expected === 204) return undefined; + try { return attachmentStrictJson(new TextDecoder("utf-8", { fatal: true }).decode(response.body)); } + catch { throw new AttachmentTransportError("unsupported_version"); } } const base = (session: AttachmentSession) => { if (!validAttachmentUuid(session.scope.project_id)) throw new AttachmentTransportError("scope_unresolved"); - return `/tracer/v1/projects/${session.scope.project_id}/attachments/uploads`; + return `/tracer/v1/projects/${session.scope.project_id}/attachments`; }; const uploadPath = (session: AttachmentSession, upload: AttachmentUpload) => { - if (!validAttachmentUuid(upload.upload_id)) throw new AttachmentTransportError("failed"); - return `${base(session)}/${upload.upload_id}`; + if (!validAttachmentUuid(upload.upload_id)) throw new AttachmentTransportError("unsupported_version"); + return `${base(session)}/uploads/${upload.upload_id}`; }; - return { - async bootstrap(request, signal) { - const result = await send("/tracer/v1/attachment-sessions", JSON.stringify(request), signal, 200, request.capture_session_id); - if (!validAttachmentSession(result, request.capture_session_id)) throw new AttachmentTransportError("unsupported_version"); + const parseUploadBody = (body: string): AttachmentUploadRequest => { + try { return attachmentStrictJson(body) as AttachmentUploadRequest; } catch { throw new AttachmentTransportError("unsupported_version"); } + }; + const completeBody = (session: AttachmentSession) => JSON.stringify({ policy_version: session.policy.version, deletion_generation: 0 }); + const v1: AttachmentTransportV1 = { + async bootstrap(body, key, signal) { + const result = await send("POST", "/tracer/v1/attachment-sessions", JSON.stringify(body), signal, 200, { key }); + if (!validAttachmentSession(result, body.capture_session_id)) throw new AttachmentTransportError("unsupported_version"); + bootstraps.set(body.capture_session_id, { ...body }); + return result; + }, + async renewSession(current, sequence, signal) { + // F01 v1 has no renewal route: replay bootstrap with the SAME session id and nonce under a new key (BR-2, CQ-1). + const body = bootstraps.get(current.capture_session_id); + if (!body) throw new AttachmentTransportError("unsupported_version"); + const result = await send("POST", "/tracer/v1/attachment-sessions", JSON.stringify(body), signal, 200, + { key: attachmentIdempotencyKey("bootstrap", current.capture_session_id, sequence) }); + if (!validAttachmentSession(result, body.capture_session_id)) throw new AttachmentTransportError("unsupported_version"); return result; }, - async initialize(session, request, signal) { - const result = await send(base(session), JSON.stringify(request), signal, 201, request.occurrence_id); - if (!validAttachmentUpload(result, request)) throw new AttachmentTransportError("failed"); + async initialize(session, body, key, signal) { + const request = parseUploadBody(body); + const result = await send("POST", `${base(session)}/uploads`, body, signal, 201, { key }); + if (!validAttachmentUpload(result, request)) throw new AttachmentTransportError("unsupported_version"); return result; }, - async upload(session, upload, data, signal) { - const result = await send(`${uploadPath(session, upload)}/content`, data, signal, 200, undefined, upload.write_capability); - if (!object(result) || result.upload_id !== upload.upload_id || result.state !== "pending_upload") throw new AttachmentTransportError("failed"); + async renewUpload(session, upload, body, signal) { + // Replay the byte-identical initialize under the same key with the ORIGINAL identity (BR-1). + const request = parseUploadBody(body); + const result = await send("POST", `${base(session)}/uploads`, body, signal, 201, { key: attachmentIdempotencyKey("upload", request.occurrence_id) }); + if (!validAttachmentUpload(result, request)) throw new AttachmentTransportError("unsupported_version"); + if (result.upload_id !== upload.upload_id || result.write_capability === upload.write_capability || + Date.parse(result.expires_at) <= Date.parse(upload.expires_at)) throw new AttachmentTransportError("expired"); + return result; + }, + async putContent(session, upload, source, signal, pacer) { + const size = source instanceof Uint8Array ? source.byteLength : (source as { size: number }).size; + if (size !== upload.reserved_bytes) throw new AttachmentTransportError("too_large"); + const result = await send("PUT", `${uploadPath(session, upload)}/content`, source, signal, 200, { capability: upload.write_capability, binary: true, pacer }); + if (!validStatus(result, upload.upload_id, "upload_id", "pending_upload")) throw new AttachmentTransportError("unsupported_version"); + }, + async complete(session, upload, key, signal) { + const result = await send("POST", `${uploadPath(session, upload)}/complete`, completeBody(session), signal, 202, { key, capability: upload.write_capability }); + // validating, never ready: late-root and root-first orders both still need validation (F01 lifecycle fixtures). + if (!validStatus(result, upload.upload_id, "upload_id", "validating")) throw new AttachmentTransportError("unsupported_version"); + return "validating"; + }, + async abort(session, upload, key, signal) { + const result = await send("POST", `${uploadPath(session, upload)}/abort`, completeBody(session), signal, 200, { key, capability: upload.write_capability }); + if (!validStatus(result, upload.upload_id, "upload_id")) throw new AttachmentTransportError("unsupported_version"); + }, + async uploadStatus(session, upload, signal) { + const result = await send("GET", `${uploadPath(session, upload)}/status`, undefined, signal, 200, { capability: upload.write_capability }); + if (!validStatus(result, upload.upload_id, "upload_id")) throw new AttachmentTransportError("unsupported_version"); + return { state: result.state, reason: result.reason }; + }, + async putManifest(session, capability, recordingId, body, key, signal) { + if (!validAttachmentUuid(recordingId) || !/^[a-zA-Z0-9_.-]{16,4096}$/.test(capability)) throw new AttachmentTransportError("unsupported_version"); + if (new TextEncoder().encode(body).byteLength > 1048576) throw new AttachmentTransportError("too_large"); + await send("PUT", `${base(session)}/recordings/${recordingId}/manifest`, body, signal, 204, { key, capability }); }, - async complete(session, upload, request, signal) { - const result = await send(`${uploadPath(session, upload)}/complete`, JSON.stringify({ policy_version: request.policy_version, deletion_generation: request.deletion_generation }), signal, 202, request.occurrence_id, upload.write_capability); - if (!object(result) || result.upload_id !== upload.upload_id || result.state !== "validating") throw new AttachmentTransportError("failed"); + async submitImport(session, body, key, signal) { + let recordingId: string; + try { recordingId = (attachmentStrictJson(body) as { recording: { recording_id: string } }).recording.recording_id; } + catch { throw new AttachmentTransportError("unsupported_version"); } + const result = await send("POST", `${base(session)}/imports`, body, signal, 202, { key }); + if (!object(result) || Object.keys(result).length !== 4 || result.mode !== "completed_recording" || result.recording_id !== recordingId || + result.state !== "importing" || !validAttachmentUuid(result.job_id)) throw new AttachmentTransportError("unsupported_version"); + return { jobId: result.job_id, recordingId, state: "importing" } as AttachmentImportJob; + }, + async importStatus(session, jobId, signal) { + if (!validAttachmentUuid(jobId)) throw new AttachmentTransportError("unsupported_version"); + const result = await send("GET", `${base(session)}/imports/${jobId}/status`, undefined, signal, 200); + if (!validStatus(result, jobId, "job_id")) throw new AttachmentTransportError("unsupported_version"); + return { state: result.state, reason: result.reason }; }, }; + return v1; +} + +/** + * Legacy four-method shape, kept for existing doubles and adapters. The returned object also carries the full + * versioned surface under ATTACHMENT_TRANSPORT_V1, which the manager prefers. + */ +export function createAttachmentHttpTransport(options: AttachmentHttpTransportOptions): AttachmentTransport { + const v1 = createAttachmentHttpTransportV1(options); + const legacy: AttachmentTransport & { [ATTACHMENT_TRANSPORT_V1]: AttachmentTransportV1 } = { + bootstrap: (request, signal) => v1.bootstrap(request, attachmentIdempotencyKey("bootstrap", request.capture_session_id, 0), signal), + initialize: (session, request, signal) => v1.initialize(session, JSON.stringify(request), attachmentIdempotencyKey("upload", request.occurrence_id), signal), + upload: (session, upload, data, signal) => v1.putContent(session, upload, data, signal, attachmentNoPacer), + complete: async (session, upload, _request, signal) => { await v1.complete(session, upload, attachmentIdempotencyKey("complete", upload.upload_id), signal); }, + [ATTACHMENT_TRANSPORT_V1]: v1, + }; + return legacy; +} +export const attachmentNoPacer: AttachmentPacer = { take: async () => undefined }; + +/** Adapt a legacy four-method transport. Routes it cannot express fail closed. */ +export function attachmentLegacyTransport(transport: AttachmentTransport): AttachmentTransportV1 { + const requests = new WeakMap(); + const unsupported = async (): Promise => { throw new AttachmentTransportError("unsupported_version"); }; + return { + bootstrap: (body, _key, signal) => transport.bootstrap(body, signal), + renewSession: unsupported, + async initialize(session, body, _key, signal) { + let request: AttachmentUploadRequest; + try { request = JSON.parse(body); } catch { throw new AttachmentTransportError("unsupported_version"); } + const upload = await transport.initialize(session, request, signal); + if (object(upload)) requests.set(upload, request); + return upload; + }, + // No fresh capability can be minted through the legacy shape: renewal yields `expired` (design section 3). + renewUpload: async () => { throw new AttachmentTransportError("expired"); }, + async putContent(session, upload, source, signal) { + if (!(source instanceof Uint8Array) && !(typeof Blob !== "undefined" && source instanceof Blob)) { + // Ranged sources (file leases) are never materialized for a transport that cannot stream them. + if ((source as { size: number }).size > 8 * 1024 * 1024) throw new AttachmentTransportError("source_unavailable"); + } + return transport.upload(session, upload, await attachmentContentBytes(source, signal), signal); + }, + async complete(session, upload, _key, signal) { + const request = requests.get(upload); + if (!request) throw new AttachmentTransportError("unsupported_version"); + await transport.complete(session, upload, request, signal); + return "validating"; + }, + abort: async () => undefined, + uploadStatus: unsupported, + putManifest: unsupported, + submitImport: unsupported, + importStatus: unsupported, + }; +} +/** Pick the versioned surface of whatever transport shape the caller provided. */ +export function attachmentWireTransport(transport: AttachmentTransport | AttachmentTransportV1 | undefined): AttachmentTransportV1 | undefined { + if (!transport) return undefined; + const carried = (transport as { [ATTACHMENT_TRANSPORT_V1]?: AttachmentTransportV1 })[ATTACHMENT_TRANSPORT_V1]; + if (carried && typeof carried.putContent === "function") return carried; + if (typeof (transport as AttachmentTransportV1).putContent === "function") return transport as AttachmentTransportV1; + return attachmentLegacyTransport(transport as AttachmentTransport); } diff --git a/typescript/packages/fi-core/src/attachments/types.ts b/typescript/packages/fi-core/src/attachments/types.ts index 4c1b906d3..f4cedb3b0 100644 --- a/typescript/packages/fi-core/src/attachments/types.ts +++ b/typescript/packages/fi-core/src/attachments/types.ts @@ -1,5 +1,6 @@ import type { Span, SpanContext } from "@opentelemetry/api"; import type { TraceConfigOptions } from "../trace/trace-config/types"; +import type { AttachmentCorrelation } from "./correlation"; export type AttachmentDirection = "input" | "output" | "mixed"; export type AttachmentModality = "audio" | "image" | "video" | "document" | "file"; @@ -11,10 +12,14 @@ export type AttachmentSource = | { kind: "Blob"; data: Blob } | { kind: "RemoteUrl"; url: string } | { kind: "LocalFile"; path: string } - | { kind: "OwnedRecordingFile"; lease: unknown } + | { kind: "OwnedRecordingFile"; lease: AttachmentOwnedFileRef } | { kind: "ProviderFile"; provider: string; fileId: string }; export type AttachmentAdmission = "queued" | "disabled" | "bootstrap_pending" | "policy_denied" | "credential_scope_unsupported" | "source_unavailable" | "unsampled" | "too_large" | "queue_full" | "unsupported_version" | "endpoint_unconfigured" | "scope_unresolved" | "correlation_unavailable" | "redaction_unavailable" | "recording_disabled" | "expired" | "deleted" | "incomplete"; -export type AttachmentCompletion = Readonly<{ status: AttachmentAdmission | "validating" | "failed" | "deleted" }>; +/** Terminal transfer outcomes (design 0.3). `validating` means transferred, NOT readable. `importing` is an accepted import job. */ +export type AttachmentCompletionStatus = "validating" | "importing" | "incomplete" | "expired" | "deleted" | "policy_denied" | "too_large" | "throttled" | + "conflict" | "not_found" | "credential_scope_unsupported" | "validation_failed" | "unsupported_version" | "source_unavailable" | "redaction_unavailable"; +/** `failed` is retired (design 0.3): it is never produced and stays in the type only for source compatibility. */ +export type AttachmentCompletion = Readonly<{ status: AttachmentAdmission | AttachmentCompletionStatus | "failed" }>; export interface AttachmentHandle { readonly attachmentId: string; readonly occurrenceId: string; @@ -61,6 +66,18 @@ export interface AttachmentConfig { workers?: number; requestTimeoutMs?: number; shutdownTimeoutMs?: number; + /** One content PUT (default 120 s), capped by the negotiated transfer_timeout_seconds. */ + contentTimeoutMs?: number; + /** One manifest PUT (default 10 s, max 30 s). */ + manifestTimeoutMs?: number; + /** Largest mutable buffer copied synchronously inside a caller's call (default 8 MiB, max 32 MiB). */ + syncCopyBytes?: number; + /** Unused preflight ticket lifetime (default 10 s, max 30 s). */ + ticketTtlMs?: number; + /** Consumed-stream idle lifetime (default 60 s, max 300 s). */ + streamIdleTtlMs?: number; + /** Session-end grace for adapter finalization (default and max 30 s). */ + finalizationTimeoutMs?: number; } export interface AttachmentReference { attachment_id: string; @@ -120,6 +137,8 @@ export interface AttachmentUploadRequest { purpose?: AttachmentPurpose; message_index?: number; part_index?: number; + recording_id?: string; + segment_index?: number; conversation_id?: string; turn_id?: string; tool_call_id?: string; @@ -148,7 +167,169 @@ export interface AttachmentManagerOptions extends AttachmentConfig { traceConfig?: TraceConfigOptions; /** Node uses process.env via the provider; browsers do not inspect any ambient credentials. */ environment?: () => Readonly>; - transport?: AttachmentTransport; + /** The legacy four-method shape or the full versioned wire surface (design section 9). */ + transport?: AttachmentTransport | AttachmentTransportV1; + /** @internal Test clock. `now` is monotonic milliseconds; `wall` is epoch milliseconds. */ + clock?: AttachmentClock; } export interface AttachmentFlushResult { readonly unfinished: number; readonly dropped: number } export type AttachmentSpan = Span | SpanContext; + +// --------------------------------------------------------------------------- +// Design sections 0-9 (sdk-streaming-design.md). Additive; earlier names are unchanged. +// --------------------------------------------------------------------------- +export interface AttachmentClock { now(): number; wall(): number } +/** Runtime state (design 0.2). `deleted` and `policy_denied` remain only for an explicit `invalidate()`. */ +export type AttachmentRuntimeState = "disabled" | "endpoint_unconfigured" | "credential_scope_unsupported" | "scope_unresolved" | "bootstrap_pending" | + "ready" | "expired" | "unsupported_version" | "incomplete" | "deleted" | "policy_denied" | "source_unavailable"; +export interface AttachmentStatus { + readonly state: AttachmentRuntimeState; + readonly queued: number; readonly active: number; + readonly openTickets: number; readonly openStreams: number; readonly openRecordings: number; + readonly ownedBytes: number; readonly dropped: number; +} +export type AttachmentOperation = "observe_consumed" | "import_existing" | "create_recording"; +export type AttachmentRole = NonNullable; +export type AttachmentTarget = AttachmentSpan | AttachmentCorrelation | undefined; +export type AttachmentGapReason = "queue_full" | "bootstrap_pending" | "source_unavailable" | "policy_denied" | "truncated" | "incomplete"; +export interface AttachmentPreflightOptions { + mimeType: string; + direction: AttachmentDirection; + /** Upper bound on the bytes the ticket may later carry; > 0. */ + maxBytes: number; + operation?: AttachmentOperation; + sourceKind?: AttachmentSourceKind; + purpose?: AttachmentPurpose; + capture?: AttachmentCapturePolicy; + ttlMs?: number; +} +export interface AttachmentFields { + messageIndex?: number; partIndex?: number; role?: AttachmentRole; channel?: number; + conversationId?: string; turnId?: string; toolCallId?: string; +} +export interface AttachmentStreamOptions extends AttachmentFields { + mimeType: string; direction: AttachmentDirection; maxBytes: number; + purpose?: AttachmentPurpose; capture?: AttachmentCapturePolicy; idleTtlMs?: number; +} +export interface AttachmentStream { + readonly reason: AttachmentAdmission; readonly attachmentId?: string; readonly occurrenceId?: string; + /** Synchronous, never throws, does no I/O; copies the chunk. */ + tryWrite(chunk: Uint8Array): boolean; + /** Non-blocking: enqueues the single upload. */ + finish(): AttachmentHandle; + abort(reason?: "incomplete" | "policy_denied"): void; +} +export interface AttachmentAdmissionTicket { + readonly admitted: boolean; + readonly reason: AttachmentAdmission; + readonly maxBytes: number; + readonly correlation?: AttachmentCorrelation; + attach(source: Extract, fields?: AttachmentFields): AttachmentHandle; + startStream(options: AttachmentStreamOptions): AttachmentStream; + release(): void; +} +export interface AttachmentPcmFormat { sampleRateHz: number; sampleFormat?: "s16le" } +export interface AttachmentTrackSpec { channels: readonly number[]; direction: AttachmentDirection; role: "user" | "assistant" | "mixed" | "unknown" } +export interface AttachmentRecordingOptions { + format: AttachmentPcmFormat; + layout: "mixed" | "stereo" | "separate_tracks"; + tracks: readonly AttachmentTrackSpec[]; + operation: "observe_consumed" | "create_recording"; + purpose?: "conversation_recording" | "turn_recording"; + capture?: AttachmentCapturePolicy; + conversationId?: string; turnId?: string; + windowSegments?: number; +} +export interface AttachmentRecordingStatus { + readonly state: "pending" | "recording" | "finishing" | "terminal"; + readonly reason: AttachmentAdmission | AttachmentCompletionStatus; + readonly sealed: number; readonly uploaded: number; readonly gapMs: number; readonly manifestRevision: number; readonly ownedBytes: number; +} +export interface AttachmentRecordingResult { + readonly recordingId: string; + readonly state: "published" | "partial" | "unpublished" | "denied"; + readonly reason?: string; + readonly segmentsPublished: number; + readonly gapMs: number; + readonly captureComplete: boolean; + /** 0 = never published. */ + readonly manifestRevision: number; +} +export interface AttachmentRecordingWriter { + /** Client UUID; never projected on spans. */ + readonly recordingId: string; + readonly reason: AttachmentAdmission; + tryWrite(pcm: Uint8Array, track?: number): boolean; + markGap(durationMs: number, reason?: AttachmentGapReason, track?: number): void; + markPlayout(startMs: number, durationMs: number, evidence: "generated" | "played" | "unknown" | "truncated"): void; + /** Bounded; never rejects. */ + finish(timeoutMs?: number): Promise; + abort(): void; + status(): AttachmentRecordingStatus; +} +/** Opaque file reference minted only by a recording file scope from this core copy. */ +declare const attachmentOwnedFileBrand: unique symbol; +export interface AttachmentOwnedFileRef { readonly [attachmentOwnedFileBrand]: true } +export interface AttachmentFileIdentity { dev: bigint; ino: bigint; size: number } +export interface AttachmentRecordingFileScope { + identify(relativePath: string): Promise; + source(relativePath: string, identity: AttachmentFileIdentity): { kind: "OwnedRecordingFile"; lease: AttachmentOwnedFileRef }; + close(): Promise; +} +export interface AttachmentOwnedRecordingOptions { + mimeType: string; direction: AttachmentDirection; wavTracks?: readonly AttachmentTrackSpec[]; + purpose?: "conversation_recording" | "turn_recording"; capture?: AttachmentCapturePolicy; conversationId?: string; +} +export interface AttachmentOwnedRecordingHandle { + /** Provisional for WAV until its header is read after admission. */ + readonly tier: "single" | "wav_recording" | "denied"; + readonly reason: AttachmentAdmission; + readonly recordingId?: string; + readonly completion: Promise; + /** Bounded wait; resolves `incomplete` at the deadline (and cancels). */ + wait(timeoutMs?: number): Promise; + cancel(): void; +} +export interface AttachmentCompletedRecordingRequest { provider: string; providerRef: string; direction?: AttachmentDirection } +export type AttachmentImportSourceRef = { kind: "ProviderFile"; connectionId: string; registeredOutputId: string } | { kind: "RemoteUrlEnvelope"; envelopeId: string }; +export interface AttachmentResolvedCompletedRecording { source: AttachmentImportSourceRef; declaredBytes: number; declaredDurationMs: number; mimeType: string; direction: AttachmentDirection } +export interface AttachmentCompletedRecordingResolver { + resolve(request: AttachmentCompletedRecordingRequest, signal: AbortSignal): Promise; +} +export interface AttachmentImportOptions { resolver: AttachmentCompletedRecordingResolver; capture?: AttachmentCapturePolicy; timeoutMs?: number } +export interface AttachmentImportHandle { + readonly reason: AttachmentAdmission | AttachmentCompletionStatus; + readonly recordingId?: string; + readonly jobId?: string; + readonly completion: Promise; + /** Exactly one status GET per call. */ + status(timeoutMs?: number): Promise<{ state: string; reason: string }>; +} +export interface AttachmentImportJob { jobId: string; recordingId: string; state: "importing" } +/** In-memory bytes, an immutable Blob, or a bounded range reader (owned files, WAV ranges). */ +export type AttachmentContentSource = Uint8Array | Blob | { readonly size: number; read(offset: number, size: number): Promise; check(): Promise }; +export interface AttachmentPacer { take(bytes: number, signal: AbortSignal): Promise } +/** The ONE versioned wire surface (design section 9). Every HTTP shape stays behind this. */ +export interface AttachmentTransportV1 { + bootstrap(body: AttachmentBootstrap, key: string, signal: AbortSignal): Promise; + renewSession(current: AttachmentSession, sequence: number, signal: AbortSignal): Promise; + initialize(session: AttachmentSession, body: string, key: string, signal: AbortSignal): Promise; + renewUpload(session: AttachmentSession, upload: AttachmentUpload, body: string, signal: AbortSignal): Promise; + putContent(session: AttachmentSession, upload: AttachmentUpload, source: AttachmentContentSource, signal: AbortSignal, pacer: AttachmentPacer): Promise; + complete(session: AttachmentSession, upload: AttachmentUpload, key: string, signal: AbortSignal): Promise<"validating">; + abort(session: AttachmentSession, upload: AttachmentUpload, key: string, signal: AbortSignal): Promise; + uploadStatus(session: AttachmentSession, upload: AttachmentUpload, signal: AbortSignal): Promise<{ state: string; reason: string }>; + putManifest(session: AttachmentSession, capability: string, recordingId: string, body: string, key: string, signal: AbortSignal): Promise; + submitImport(session: AttachmentSession, body: string, key: string, signal: AbortSignal): Promise; + importStatus(session: AttachmentSession, jobId: string, signal: AbortSignal): Promise<{ state: string; reason: string }>; +} +export interface AttachmentRecordingTrack { track_id: string; channels: number[]; direction: AttachmentDirection; role: AttachmentTrackSpec["role"] } +export interface AttachmentRecordingManifest { + schema_version: 1; recording_id: string; timebase: "session_relative_ms"; + sample_format: "s16le"; sample_rate_hz: number; channels: number; codec: "pcm"; container: "wav"; + layout: AttachmentRecordingOptions["layout"]; tracks: AttachmentRecordingTrack[]; + segments: { occurrence_id: string; index: number; start_ms: number; duration_ms: number; track_id?: string }[]; + gaps: { start_ms: number; duration_ms: number; reason: AttachmentGapReason; track_id?: string }[]; + playout: { start_ms: number; duration_ms: number; evidence: "generated" | "played" | "unknown" | "truncated" }[]; + capture_complete: boolean; duration_ms: number; +} diff --git a/typescript/packages/fi-core/src/attachments/wire.ts b/typescript/packages/fi-core/src/attachments/wire.ts new file mode 100644 index 000000000..50f2ad58d --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/wire.ts @@ -0,0 +1,132 @@ +/** + * Hand validators for every F01 shape the SDK sends or parses (design 9, T-TR-01). They mirror + * contract.schema.json plus the cross-field rules of validate.py, with no schema engine in the runtime. + */ +import { validAttachmentManifest } from "./recording"; +import { attachmentStrictJson } from "./transport-v1"; +import type { AttachmentRecordingManifest } from "./types"; + +type V = (value: unknown) => boolean; +const obj = (v: unknown): v is Record => !!v && typeof v === "object" && !Array.isArray(v); +const UUID: V = v => typeof v === "string" && v.length === 36 && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(v) && v !== "00000000-0000-0000-0000-000000000000"; +const TRACE: V = v => (typeof v === "string" && /^[0-9a-f]{32}$/.test(v) && !/^0{32}$/.test(v)) || UUID(v); +const SPAN: V = v => typeof v === "string" && /^[0-9a-f]{16}$/.test(v) && !/^0{16}$/.test(v); +const TOKEN: V = v => typeof v === "string" && /^[a-zA-Z0-9_-]{1,128}$/.test(v); +const int = (min: number, max: number): V => v => typeof v === "number" && Number.isInteger(v) && v >= min && v <= max; +const SAFE = int(0, Number.MAX_SAFE_INTEGER); +const TIMESTAMP: V = v => typeof v === "string" && v.length <= 40 && + /^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(\.\d+)?([Zz]|[+-]\d{2}:\d{2})$/.test(v) && Number.isFinite(Date.parse(v)); +const oneOf = (...values: unknown[]): V => v => values.includes(v); +const DIRECTION = oneOf("input", "output", "mixed"); +const MODALITY = oneOf("audio", "image", "video", "document", "file"); +const AUDIO = ["audio/wav", "audio/ogg", "audio/mpeg", "audio/mp4", "audio/webm", "audio/flac", "audio/L16"]; +const MIME_BY: Record = { audio: AUDIO, image: ["image/png", "image/jpeg", "image/webp"], video: ["video/mp4", "video/webm"], + document: ["application/pdf"], file: ["application/octet-stream"] }; +const MIME = oneOf(...Object.values(MIME_BY).flat()); +const PURPOSE = oneOf("conversation_recording", "turn_recording", "model_input", "model_output", "tool_result", "computer_screenshot", "diagnostic", "document"); +const SOURCE = oneOf("Bytes", "Blob", "RemoteUrl", "LocalFile", "OwnedRecordingFile", "ProviderFile"); +const STATUS = oneOf("pending_upload", "importing", "validating", "ready", "failed", "rejected", "expired", "deleted"); +const REASON = oneOf("queued", "disabled", "bootstrap_pending", "policy_denied", "credential_scope_unsupported", "source_unavailable", "unsampled", "too_large", + "queue_full", "unsupported_version", "endpoint_unconfigured", "scope_unresolved", "correlation_unavailable", "redaction_unavailable", "recording_disabled", + "binding_pending", "not_found", "conflict", "throttled", "validation_failed", "expired", "deleted", "incomplete", "ready"); +/** Closed object: every required key present, no unknown key, every present key valid. */ +function closed(required: string[], properties: Record): V { + return v => obj(v) && required.every(key => Object.prototype.hasOwnProperty.call(v, key)) && + Object.keys(v).every(key => Object.prototype.hasOwnProperty.call(properties, key) && properties[key](v[key])); +} +const array = (item: V, min: number, max: number, unique = false): V => v => Array.isArray(v) && v.length >= min && v.length <= max && v.every(item) && + (!unique || new Set(v.map(x => JSON.stringify(x))).size === v.length); +const SCOPE = closed(["organization_id", "workspace_id", "project_id"], { organization_id: UUID, workspace_id: UUID, project_id: UUID }); +const PARENT = closed(["trace_id", "span_id"], { trace_id: TRACE, span_id: SPAN }); +const ASSOCIATION = closed(["capture_session_id", "correlation_nonce", "parent"], { capture_session_id: UUID, correlation_nonce: UUID, parent: PARENT }); +const mimeMatches = (v: Record) => (MIME_BY[v.modality as string] ?? []).includes(v.mime_type as string); +const REFERENCE_SHAPE = closed(["attachment_id", "occurrence_id", "direction", "modality", "mime_type"], { attachment_id: UUID, occurrence_id: UUID, + direction: DIRECTION, modality: MODALITY, mime_type: MIME, purpose: PURPOSE, message_index: int(0, 65535), part_index: int(0, 65535), + role: oneOf("user", "assistant", "tool", "system", "unknown"), channel: int(0, 31), conversation_id: TOKEN, turn_id: TOKEN, tool_call_id: TOKEN }); +const REFERENCE: V = v => REFERENCE_SHAPE(v) && mimeMatches(v as Record); +const REFERENCES: V = v => array(REFERENCE, 0, 64, true)(v) && new Set((v as { occurrence_id: string }[]).map(r => r.occurrence_id)).size === (v as unknown[]).length; +const UPLOAD_REQUEST_SHAPE = closed(["schema_version", "attachment_id", "occurrence_id", "association", "policy_version", "deletion_generation", "mime_type", + "modality", "direction", "source_kind", "declared_bytes"], { schema_version: oneOf(1), attachment_id: UUID, occurrence_id: UUID, association: ASSOCIATION, + policy_version: TOKEN, deletion_generation: SAFE, mime_type: MIME, modality: MODALITY, direction: DIRECTION, source_kind: SOURCE, purpose: PURPOSE, + declared_bytes: int(1, 52428800), message_index: int(0, 65535), part_index: int(0, 65535), recording_id: UUID, segment_index: int(0, 65535), + conversation_id: TOKEN, turn_id: TOKEN, tool_call_id: TOKEN }); +const UPLOAD_REQUEST: V = v => { + if (!UPLOAD_REQUEST_SHAPE(v)) return false; + const r = v as Record; + if (("recording_id" in r) !== ("segment_index" in r)) return false; + if (!mimeMatches(r)) return false; + if (r.modality === "image" && (r.declared_bytes as number) > 20971520) return false; + if ("recording_id" in r && (r.declared_bytes as number) > 8388608) return false; + return true; +}; +const POLICY = closed(["version", "enabled", "allowed_sources", "allowed_modalities", "require_media_redaction", "require_metadata_removal", + "allow_unredacted_media_with_text_pii", "record_audio", "record_video"], { version: TOKEN, enabled: v => typeof v === "boolean", + allowed_sources: array(SOURCE, 0, 6, true), allowed_modalities: array(MODALITY, 0, 5, true), require_media_redaction: v => typeof v === "boolean", + require_metadata_removal: v => typeof v === "boolean", allow_unredacted_media_with_text_pii: v => typeof v === "boolean", + record_audio: v => typeof v === "boolean", record_video: v => typeof v === "boolean" }); +const LIMIT_MAX: Record = { max_upload_bytes: 52428800, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, + recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, + links_per_minute: 1000, upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, + revocation_check_seconds: 5 }; +const LIMITS = closed(Object.keys(LIMIT_MAX), Object.fromEntries(Object.entries(LIMIT_MAX).map(([k, max]) => [k, int(1, max)]))); +const IMPORT_SOURCE: V = v => closed(["kind", "envelope_id"], { kind: oneOf("RemoteUrlEnvelope"), envelope_id: UUID })(v) || + closed(["kind", "connection_id", "registered_output_id"], { kind: oneOf("ProviderFile"), connection_id: UUID, registered_output_id: UUID })(v); +const COMPLETED = closed(["schema_version", "association", "policy_version", "deletion_generation", "mime_type", "direction", "source_kind", "recording_id", + "modality", "declared_bytes", "declared_duration_ms"], { schema_version: oneOf(1), association: ASSOCIATION, policy_version: TOKEN, deletion_generation: SAFE, + mime_type: oneOf(...AUDIO), direction: DIRECTION, source_kind: oneOf("ProviderFile", "RemoteUrl"), recording_id: UUID, modality: oneOf("audio"), + declared_bytes: int(1, 1073741824), declared_duration_ms: int(1, 7200000) }); +const RECORDING_IMPORT: V = v => { + if (!closed(["mode", "recording", "source"], { mode: oneOf("completed_recording"), recording: COMPLETED, source: IMPORT_SOURCE })(v)) return false; + const r = v as { recording: { source_kind: string }; source: { kind: string } }; + return r.recording.source_kind === (r.source.kind === "RemoteUrlEnvelope" ? "RemoteUrl" : "ProviderFile"); +}; +const TRACK = closed(["track_id", "channels", "direction", "role"], { track_id: UUID, channels: array(int(0, 31), 1, 32, true), direction: DIRECTION, + role: oneOf("user", "assistant", "mixed", "unknown") }); +const MANIFEST_SHAPE = closed(["schema_version", "recording_id", "timebase", "sample_format", "sample_rate_hz", "channels", "codec", "container", "layout", + "tracks", "segments", "gaps", "playout", "capture_complete", "duration_ms"], { schema_version: oneOf(1), recording_id: UUID, timebase: oneOf("session_relative_ms"), + sample_format: oneOf("s16le", "s24le", "s32le", "f32le", "encoded"), sample_rate_hz: int(8000, 192000), channels: int(1, 32), + codec: oneOf("pcm", "opus", "mp3", "aac", "flac"), container: oneOf("wav", "ogg", "mp3", "mp4", "webm", "flac", "raw"), layout: oneOf("mixed", "stereo", "separate_tracks"), + tracks: array(TRACK, 1, 32), segments: array(closed(["occurrence_id", "index", "start_ms", "duration_ms"], { occurrence_id: UUID, index: int(0, 65535), + start_ms: int(0, 7200000), duration_ms: int(1, 30000), track_id: UUID }), 0, 65536), + gaps: array(closed(["start_ms", "duration_ms", "reason"], { start_ms: int(0, 7200000), duration_ms: int(1, 7200000), + reason: oneOf("queue_full", "bootstrap_pending", "source_unavailable", "policy_denied", "truncated", "incomplete"), track_id: UUID }), 0, 65536), + playout: array(closed(["start_ms", "duration_ms", "evidence"], { start_ms: int(0, 7200000), duration_ms: int(1, 7200000), + evidence: oneOf("generated", "played", "unknown", "truncated") }), 0, 65536), + capture_complete: v => typeof v === "boolean", duration_ms: int(0, 7200000) }); +const SPAN_ATTRIBUTES: V = v => { + if (!closed(["fi.attachment.schema_version", "fi.attachment.refs"], { "fi.attachment.schema_version": oneOf(1), + "fi.attachment.refs": x => typeof x === "string" && x.length <= 16384, "fi.attachment.capture_session_id": UUID, "fi.attachment.correlation_nonce": UUID })(v)) return false; + const a = v as Record; + if (("fi.attachment.capture_session_id" in a) !== ("fi.attachment.correlation_nonce" in a)) return false; + if (new TextEncoder().encode(a["fi.attachment.refs"]).byteLength > 16384) return false; + try { return REFERENCES(attachmentStrictJson(a["fi.attachment.refs"])); } catch { return false; } +}; +const VALIDATORS: Record = { + BootstrapRequest: closed(["schema_versions", "project_name", "project_type", "capture_session_id", "correlation_nonce"], { + schema_versions: array(int(1, 65535), 1, 8, true), project_name: v => typeof v === "string" && v.length >= 1 && v.length <= 255, + project_type: v => typeof v === "string" && v.length >= 1 && v.length <= 64, workspace_id: UUID, capture_session_id: UUID, correlation_nonce: UUID }), + BootstrapResponse: closed(["schema_version", "scope", "capture_session_id", "policy", "limits", "expires_at", "transfer_modes"], { schema_version: oneOf(1), + scope: SCOPE, capture_session_id: UUID, policy: POLICY, limits: LIMITS, expires_at: TIMESTAMP, + transfer_modes: v => Array.isArray(v) && v.length === 1 && v[0] === "authenticated_stream" }), + UploadRequest: UPLOAD_REQUEST, + UploadResponse: closed(["upload_id", "attachment_id", "occurrence_id", "state", "reserved_bytes", "expires_at", "transfer_mode", "write_capability"], { + upload_id: UUID, attachment_id: UUID, occurrence_id: UUID, state: oneOf("pending_upload"), reserved_bytes: int(1, 52428800), expires_at: TIMESTAMP, + transfer_mode: oneOf("authenticated_stream"), write_capability: v => typeof v === "string" && /^[a-zA-Z0-9_.-]{16,4096}$/.test(v) }), + CompleteRequest: closed(["policy_version", "deletion_generation"], { policy_version: TOKEN, deletion_generation: SAFE }), + StatusResponse: closed(["upload_id", "state", "reason"], { upload_id: UUID, state: STATUS, reason: REASON }), + RecordingManifest: v => MANIFEST_SHAPE(v) && validAttachmentManifest(v as AttachmentRecordingManifest), + RecordingImportRequest: RECORDING_IMPORT, + RecordingImportResponse: closed(["mode", "job_id", "recording_id", "state"], { mode: oneOf("completed_recording"), job_id: UUID, recording_id: UUID, state: oneOf("importing") }), + ImportStatusResponse: closed(["job_id", "state", "reason"], { job_id: UUID, state: STATUS, reason: REASON }), + Error: closed(["code", "request_id"], { code: REASON, request_id: UUID }), + SpanAttributes: SPAN_ATTRIBUTES, + References: REFERENCES, + Reference: REFERENCE, +}; +/** Shapes this SDK sends or parses. */ +export const ATTACHMENT_WIRE_SHAPES = Object.freeze(Object.keys(VALIDATORS)); +export function validAttachmentWire(name: string, value: unknown): boolean { + const validate = VALIDATORS[name]; + if (!validate) throw new TypeError("unsupported wire shape"); + try { return validate(value); } catch { return false; } +} diff --git a/typescript/packages/fi-core/test/attachments-browser.cjs b/typescript/packages/fi-core/test/attachments-browser.cjs index 9f6dd2e25..c7e5a2bfe 100644 --- a/typescript/packages/fi-core/test/attachments-browser.cjs +++ b/typescript/packages/fi-core/test/attachments-browser.cjs @@ -3,7 +3,7 @@ const assert = require('node:assert/strict'); const { chromium } = require(process.argv[2]); const bundle = process.argv[3]; const id = '10000000-0000-4000-8000-000000000001'; -const expires = new Date(Date.now() + 60000).toISOString(); +const expires = new Date(Date.now() + 3600000).toISOString(); const policy = { version: 'v1', enabled: true, allowed_sources: ['Bytes', 'Blob'], allowed_modalities: ['audio', 'image'], require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }; const limits = { max_upload_bytes: 52428800, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, diff --git a/typescript/packages/fi-core/test/attachments-design.test.cjs b/typescript/packages/fi-core/test/attachments-design.test.cjs new file mode 100644 index 000000000..5b3b8a1f2 --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-design.test.cjs @@ -0,0 +1,729 @@ +/* Design qualification (sdk-streaming-design.md section 10). Synthetic data and in-memory doubles only. */ +const { test, after } = require('node:test'); +const assert = require('node:assert/strict'); +const { core, v1, managerV1, span, pcm, writeSeconds, readAll, clock, mono } = require('./attachments-v1.cjs'); +const { identity, ids, policy, limits, source } = require('./attachments-fixtures.cjs'); +const { AttachmentManager, AttachmentTransportError, attachmentWavHeader, attachmentSegmentFrames, attachmentFramesToMs, validAttachmentManifest, + validAttachmentWire, createAttachmentHttpTransport, createAttachmentHttpTransportV1, attachmentHttpError, attachmentIdempotencyKey, AttachmentCorrelation } = core; +const live = setInterval(() => {}, 1000); // Explicit test owner, not an SDK keepalive. +after(() => clearInterval(live)); +const hex = value => value.replace(/-/g, ''); +const tick = () => new Promise(resolve => setImmediate(resolve)); +const options = { mimeType: 'image/png', direction: 'input' }; + +// ---- 10.1 Preflight ----------------------------------------------------------------------------------- +test('T-PF-01 disabled manager: preflight denies without touching any source', async () => { + const m = new AttachmentManager({ transport: v1().transport }); + const t = m.preflight(identity, { mimeType: 'image/png', direction: 'input', maxBytes: 10 }); + assert.equal(t.reason, 'disabled'); assert.equal(t.admitted, false); + let read = false; const h = t.attach({ kind: 'Bytes', get data() { read = true; return new Uint8Array(1); } }); + assert.equal(h.admission, 'disabled'); assert.equal(read, false); +}); +test('T-PF-02 bootstrap pending: denied and nothing reserved', async () => { + const f = await managerV1({ bootstrap: () => new Promise(() => {}) }, { ready: false }); + const t = f.manager.preflight(identity, { mimeType: 'image/png', direction: 'input', maxBytes: 100 }); + assert.equal(t.reason, 'bootstrap_pending'); assert.equal(f.manager.status().ownedBytes, 0); assert.equal(f.manager.status().openTickets, 0); + await f.manager.shutdown(1); +}); +test('T-PF-04 create_recording needs client AND server record_audio; T-PF-05 import is not recording', async () => { + const off = await managerV1({}, {}, { recordAudio: false }); + assert.equal(off.manager.preflight(identity, { mimeType: 'audio/wav', direction: 'mixed', maxBytes: 10, operation: 'create_recording' }).reason, 'recording_disabled'); + assert.equal(off.manager.preflight(identity, { mimeType: 'audio/wav', direction: 'mixed', maxBytes: 10, operation: 'import_existing' }).reason, 'queued'); + const server = await managerV1({}, { policy: { record_audio: false } }, { recordAudio: true }); + assert.equal(server.manager.preflight(identity, { mimeType: 'audio/wav', direction: 'mixed', maxBytes: 10, operation: 'create_recording' }).reason, 'recording_disabled'); + const both = await managerV1({}, {}, { recordAudio: true }); + assert.equal(both.manager.preflight(identity, { mimeType: 'audio/wav', direction: 'mixed', maxBytes: 10, operation: 'create_recording' }).reason, 'queued'); + for (const f of [off, server, both]) await f.manager.shutdown(1); +}); +test('T-PF-06 mixed direction with hideInputs; T-PF-07 computer_screenshot output with hideInputImages', async () => { + const a = await managerV1({}, {}, { traceConfig: { hideInputs: true } }); + assert.equal(a.manager.preflight(identity, { mimeType: 'audio/wav', direction: 'mixed', maxBytes: 10 }).reason, 'policy_denied'); + const b = await managerV1({}, {}, { traceConfig: { hideInputImages: true } }); + assert.equal(b.manager.preflight(identity, { mimeType: 'image/png', direction: 'output', purpose: 'computer_screenshot', maxBytes: 10 }).reason, 'policy_denied'); + assert.equal(b.manager.preflight(identity, { mimeType: 'image/png', direction: 'output', purpose: 'model_output', maxBytes: 10 }).reason, 'queued'); + await a.manager.shutdown(1); await b.manager.shutdown(1); +}); +test('T-PF-08 descriptors or bytes exhausted -> queue_full', async () => { + const f = await managerV1({}, {}, { maxDescriptors: 1, maxOwnedBytes: 100 }); + const first = f.manager.preflight(identity, { ...options, maxBytes: 10 }); assert.equal(first.reason, 'queued'); + assert.equal(f.manager.preflight(identity, { ...options, maxBytes: 10 }).reason, 'queue_full'); + first.release(); + assert.equal(f.manager.preflight(identity, { ...options, maxBytes: 101 }).reason, 'too_large'); + const g = await managerV1({}, {}, { maxOwnedBytes: 100 }); + const held = g.manager.preflight(identity, { ...options, maxBytes: 90 }); assert.equal(held.reason, 'queued'); + assert.equal(g.manager.preflight(identity, { ...options, maxBytes: 20 }).reason, 'queue_full'); + held.release(); assert.equal(g.manager.status().ownedBytes, 0); + await f.manager.shutdown(1); await g.manager.shutdown(1); +}); +test('T-PF-11 ticket used twice -> conflict and the source is untouched', async () => { + const f = await managerV1(); + const t = f.manager.preflight(identity, { ...options, maxBytes: 8 }); + assert.equal(t.attach({ kind: 'Bytes', data: new Uint8Array([1, 2, 3]) }).admission, 'queued'); + let read = false; + assert.equal(t.attach({ kind: 'Bytes', get data() { read = true; return new Uint8Array(1); } }).admission, 'conflict'); assert.equal(read, false); + await f.manager.shutdown(); +}); +test('T-PF-12 ticket TTL with a fake clock: expired, budget returned lazily, no timer armed', async () => { + const c = clock(); const f = await managerV1({}, { clock: c }); + const timers = []; const original = global.setTimeout; + global.setTimeout = (...args) => { timers.push(args[1]); return original(...args); }; + let t; + try { t = f.manager.preflight(identity, { ...options, maxBytes: 64, ttlMs: 100 }); } finally { global.setTimeout = original; } + assert.equal(timers.length, 0, 'preflight armed a timer'); + assert.equal(f.manager.status().ownedBytes, 64); + c.advance(150); + assert.equal(f.manager.status().ownedBytes, 0); assert.equal(f.manager.status().openTickets, 0); + assert.equal(t.attach({ kind: 'Bytes', data: new Uint8Array(4) }).admission, 'expired'); + await f.manager.shutdown(1); +}); +test('T-PF-13 policy tightened between preflight and attach: new denial, zero reads', async () => { + const f = await managerV1(); + const t = f.manager.preflight(identity, { ...options, maxBytes: 8 }); + f.manager.updatePolicy({ traceConfig: { hideInputs: true } }); + let read = false; + assert.equal(t.attach({ kind: 'Bytes', get data() { read = true; return new Uint8Array(1); } }).admission, 'policy_denied'); + assert.equal(read, false); assert.equal(f.manager.status().ownedBytes, 0); await f.manager.shutdown(1); +}); +test('T-PF-14 actual length above the ticket -> too_large, no copy; T-PF-15 mutable buffer above syncCopyBytes', async () => { + const f = await managerV1({}, {}, { syncCopyBytes: 16 }); + const t = f.manager.preflight(identity, { ...options, maxBytes: 4 }); + assert.equal(t.attach({ kind: 'Bytes', data: new Uint8Array(5) }).admission, 'too_large'); + assert.equal(f.manager.status().ownedBytes, 0); + const big = f.manager.preflight(identity, { ...options, maxBytes: 64 }); + assert.equal(big.attach({ kind: 'Bytes', data: new Uint8Array(32) }).admission, 'too_large'); + const blob = f.manager.preflight(identity, { ...options, maxBytes: 64, sourceKind: 'Blob' }); + const h = blob.attach({ kind: 'Blob', data: new Blob([new Uint8Array(32)]) }); + assert.equal(h.admission, 'queued'); assert.equal((await h.completion).status, 'validating'); + await f.manager.shutdown(); +}); +test('preflight pattern: denied ticket means no decode happened (adapter shape from design 1.5)', async () => { + const f = await managerV1({}, {}, { traceConfig: { hideInputImages: true } }); + let decoded = 0; const decode = () => { decoded++; return new Uint8Array(4); }; + const t = f.manager.preflight(identity, { mimeType: 'image/png', direction: 'input', purpose: 'model_input', maxBytes: 4 }); + if (t.admitted) t.attach({ kind: 'Bytes', data: decode() }); + assert.equal(decoded, 0); await f.manager.shutdown(1); +}); + +// ---- 10.2 Streaming recordings ---------------------------------------------------------------------------- +test('T-ST-01..04 worked vectors: segment frames, bytes and milliseconds', () => { + assert.equal(attachmentSegmentFrames(24000, 2), 720000); assert.equal(44 + 720000 * 2, 1440044); + assert.equal(attachmentSegmentFrames(48000, 4), 1440000); assert.equal(44 + 1440000 * 4, 5760044); + assert.equal(attachmentSegmentFrames(48000, 16), 524256); assert.equal(44 + 524256 * 16, 8388140); assert.equal(attachmentFramesToMs(524256, 48000), 10922); + assert.equal(attachmentFramesToMs(10000, 44100, true), 227); + const h = attachmentWavHeader(1, 24000, 1440000); const v = new DataView(h.buffer); + assert.equal(Buffer.from(h.subarray(0, 4)).toString(), 'RIFF'); assert.equal(v.getUint32(4, true), 1440036); assert.equal(Buffer.from(h.subarray(8, 16)).toString(), 'WAVEfmt '); + assert.deepEqual([v.getUint32(16, true), v.getUint16(20, true), v.getUint16(22, true), v.getUint32(24, true), v.getUint32(28, true), v.getUint16(32, true), v.getUint16(34, true)], [16, 1, 1, 24000, 48000, 2, 16]); + assert.equal(Buffer.from(h.subarray(36, 40)).toString(), 'data'); assert.equal(v.getUint32(40, true), 1440000); +}); +test('T-ST-01 24 kHz mono 30 s: one 1 440 044-byte segment with an exact header; T-ST-20 carrier stamp, no references', async () => { + const f = await managerV1(); const s = span(); + const w = f.manager.startRecording(s, mono(24000)); assert.equal(w.reason, 'queued'); + assert.equal(s.attrs['fi.attachment.refs'], '[]'); assert.equal(s.attrs['fi.attachment.schema_version'], 1); + assert.equal(s.attrs['fi.attachment.capture_session_id'], f.manager.sessionId); assert.equal(s.attrs['fi.attachment.correlation_nonce'], f.manager.nonce); + assert.equal(validAttachmentWire('SpanAttributes', s.attrs), true); + await writeSeconds(f.manager, w, 24000, 30); + const result = await w.finish(10000); + assert.equal(result.state, 'published'); assert.equal(result.captureComplete, true); + const sizes = f.calls.filter(c => c.op === 'putContent').map(c => c.size); assert.deepEqual(sizes, [1440044]); + const body = [...f.contents.values()][0]; assert.deepEqual([...body.subarray(0, 44)], [...attachmentWavHeader(1, 24000, 1440000)]); + const manifest = f.manifests.at(-1).body; + assert.deepEqual(manifest.segments.map(s => [s.index, s.start_ms, s.duration_ms]), [[0, 0, 30000]]); + assert.equal(manifest.duration_ms, 30000); assert.equal(validAttachmentWire('RecordingManifest', manifest), true); + assert.equal(s.attrs['fi.attachment.refs'], '[]', 'segments are never projected as references'); + const request = f.requests[0]; + assert.equal(request.recording_id, w.recordingId); assert.equal(request.segment_index, 0); assert.equal(request.declared_bytes, 1440044); + assert.equal(validAttachmentWire('UploadRequest', request), true); + await f.manager.shutdown(); +}); +test('T-ST-02 24 kHz mono 61 s: 1 440 044 / 1 440 044 / 48 044, duration 61 000, capture_complete', async () => { + const f = await managerV1(); + const w = f.manager.startRecording(identity, mono(24000)); + await writeSeconds(f.manager, w, 24000, 61); + const result = await w.finish(10000); + assert.deepEqual(f.calls.filter(c => c.op === 'putContent').map(c => c.size), [1440044, 1440044, 48044]); + const manifest = f.manifests.at(-1).body; + assert.equal(manifest.duration_ms, 61000); assert.equal(manifest.capture_complete, true); assert.deepEqual(manifest.gaps, []); + assert.deepEqual(manifest.segments.map(s => s.duration_ms), [30000, 30000, 1000]); assert.equal(result.state, 'published'); + await f.manager.shutdown(); +}); +test('T-ST-03 44.1 kHz mono 10 000 frames: 20 044 bytes, duration 227', async () => { + const f = await managerV1(); + const w = f.manager.startRecording(identity, mono(44100)); + assert.equal(w.tryWrite(pcm(10000)), true); + const result = await w.finish(10000); + assert.deepEqual(f.calls.filter(c => c.op === 'putContent').map(c => c.size), [20044]); + assert.equal(f.manifests.at(-1).body.duration_ms, 227); assert.equal(f.manifests.at(-1).body.segments[0].duration_ms, 227); assert.equal(result.captureComplete, true); + await f.manager.shutdown(); +}); +test('T-ST-05 negotiated segment_duration_ms 10 000 and segment_bytes 1 MiB are honoured', async () => { + const f = await managerV1({}, { limits: { segment_duration_ms: 10000, segment_bytes: 1048576 } }); + const w = f.manager.startRecording(identity, mono(24000)); + await writeSeconds(f.manager, w, 24000, 25, { flushEvery: 5 }); + await w.finish(10000); + const sizes = f.calls.filter(c => c.op === 'putContent').map(c => c.size); + assert.deepEqual(sizes, [480044, 480044, 240044]); assert.ok(sizes.every(s => s <= 1048576)); + assert.ok(f.manifests.at(-1).body.segments.every(s => s.duration_ms <= 10000)); + await f.manager.shutdown(); +}); +const twoTracks = { format: { sampleRateHz: 16000 }, layout: 'separate_tracks', operation: 'observe_consumed', + tracks: [{ channels: [0], direction: 'input', role: 'user' }, { channels: [1], direction: 'output', role: 'assistant' }] }; +test('T-ST-06 separate tracks with one track starved for 5 s; T-ST-07 track_id rules', async () => { + const f = await managerV1(); + const w = f.manager.startRecording(identity, twoTracks); + for (let i = 0; i < 250; i++) { w.tryWrite(pcm(320), 0); w.tryWrite(pcm(320), 1); } // 5 s each + for (let i = 0; i < 250; i++) w.tryWrite(pcm(320), 0); // track 1 starved for 5 s + const result = await w.finish(10000); + const m = f.manifests.at(-1).body; + assert.equal(validAttachmentManifest(m), true); assert.equal(validAttachmentWire('RecordingManifest', m), true); + assert.ok(m.segments.every(s => s.track_id) && m.gaps.every(g => g.track_id)); + const user = m.tracks[0].track_id; const bot = m.tracks[1].track_id; + assert.deepEqual(m.segments.filter(s => s.track_id === user).map(s => [s.start_ms, s.duration_ms]), [[0, 10000]]); + assert.deepEqual(m.segments.filter(s => s.track_id === bot).map(s => [s.start_ms, s.duration_ms]), [[0, 5000]]); + assert.deepEqual(m.gaps.map(g => [g.track_id === bot, g.start_ms, g.duration_ms, g.reason]), [[true, 5000, 5000, 'source_unavailable']]); + assert.equal(result.state, 'partial'); assert.equal(m.capture_complete, false); + const mutated = JSON.parse(JSON.stringify(m)); delete mutated.gaps[0].track_id; + assert.equal(validAttachmentManifest(mutated), false, 'F01 separate-track-needs-track-identity'); + await f.manager.shutdown(); +}); +test('T-ST-08 stereo: two tracks [0] and [1], channels 2; a [0],[0] mutation fails locally', async () => { + const f = await managerV1(); + const w = f.manager.startRecording(identity, { ...twoTracks, layout: 'stereo' }); + assert.equal(w.reason, 'queued'); + for (let i = 0; i < 50; i++) assert.equal(w.tryWrite(pcm(320, 2)), true); + await w.finish(10000); + const m = f.manifests.at(-1).body; + assert.equal(m.channels, 2); assert.deepEqual(m.tracks.map(t => t.channels), [[0], [1]]); assert.ok(m.segments.every(s => !('track_id' in s))); + assert.equal(f.requests[0].direction, 'mixed'); assert.equal(validAttachmentManifest(m), true); + const mutated = JSON.parse(JSON.stringify(m)); mutated.tracks[1].channels = [0]; assert.equal(validAttachmentManifest(mutated), false); + const mixedMutation = JSON.parse(JSON.stringify(m)); mixedMutation.segments[0].track_id = m.tracks[0].track_id; assert.equal(validAttachmentManifest(mixedMutation), false); + await f.manager.shutdown(); +}); +test('T-ST-09 budget exhausted mid-recording: tryWrite false, queue_full gap of the exact duration, no exception', async () => { + const f = await managerV1({}, {}, { maxOwnedBytes: 32000 }); // 1 s of 16 kHz mono + const w = f.manager.startRecording(identity, mono(16000)); + const a = await writeSeconds(f.manager, w, 16000, 1, { flushEvery: 0 }); assert.equal(a.rejected, 0); + const b = await writeSeconds(f.manager, w, 16000, 1, { flushEvery: 0 }); assert.equal(b.accepted, 0); + await w.finish(10000); + const m = f.manifests.at(-1).body; + assert.deepEqual(m.gaps.map(g => [g.start_ms, g.duration_ms, g.reason]), [[1000, 1000, 'queue_full']]); + await f.manager.shutdown(); +}); +test('T-ST-10 window full (blocked transport): queue_full gaps, at most windowSegments sealed segments held', async () => { + const f = await managerV1(base => ({ putContent: (_s, _u, _src, signal) => new Promise((_, reject) => signal.addEventListener('abort', () => reject(new AttachmentTransportError('incomplete')), { once: true })) }), + { limits: { segment_duration_ms: 1000 } }); + const w = f.manager.startRecording(identity, mono(16000)); + const r = await writeSeconds(f.manager, w, 16000, 5, { flushEvery: 0 }); + assert.ok(r.rejected > 0); + assert.ok(f.manager.status().ownedBytes <= 2 * (32000 + 44) + 262144, `owned ${f.manager.status().ownedBytes}`); + const result = await w.finish(500); assert.ok(['partial', 'unpublished'].includes(result.state)); + await f.manager.shutdown(100); +}); +test('T-ST-11 bootstrap pending at start, then ready: leading bootstrap_pending gap and no retained bytes before ready', async () => { + let release; const gate = new Promise(resolve => { release = resolve; }); + const f = await managerV1(base => ({ bootstrap: async (body, key) => { await gate; return base.bootstrap(body, key); } }), { ready: false }); + const w = f.manager.startRecording(identity, mono(16000)); + assert.equal(w.reason, 'bootstrap_pending'); + await writeSeconds(f.manager, w, 16000, 1, { flushEvery: 0 }); + assert.equal(f.manager.status().ownedBytes, 0); + release(); assert.equal(await f.manager.ready(), true); + const r = await writeSeconds(f.manager, w, 16000, 1, { flushEvery: 0 }); assert.equal(r.rejected, 0); + assert.equal(w.reason, 'queued'); + await w.finish(10000); + const m = f.manifests.at(-1).body; + assert.deepEqual(m.gaps.map(g => [g.start_ms, g.duration_ms, g.reason]), [[0, 1000, 'bootstrap_pending']]); + assert.deepEqual(m.segments.map(s => [s.start_ms, s.duration_ms]), [[1000, 1000]]); + await f.manager.shutdown(); +}); +test('T-ST-12 policy tightened mid-recording: writer terminal, bytes freed, final manifest ends with policy_denied', async () => { + const f = await managerV1({}, { limits: { segment_duration_ms: 1000 } }); + const w = f.manager.startRecording(identity, mono(16000)); + await writeSeconds(f.manager, w, 16000, 2, { flushEvery: 1 }); + f.manager.updatePolicy({ traceConfig: { hideInputs: true } }); + assert.equal(w.tryWrite(pcm(320)), false); assert.equal(w.tryWrite(pcm(320)), false); + await f.manager.flush(); assert.equal(f.manager.status().ownedBytes, 0); + await w.finish(10000); + const m = f.manifests.at(-1).body; + assert.equal(m.segments.length, 2); assert.equal(m.gaps.at(-1).reason, 'policy_denied'); assert.equal(m.capture_complete, false); + await f.manager.shutdown(); +}); +test('T-ST-13 recording cap (recording_duration_ms 60 000): truncated gap to the end, capture_complete false', async () => { + const f = await managerV1({}, { limits: { recording_duration_ms: 60000 } }); + const w = f.manager.startRecording(identity, mono(8000)); + await writeSeconds(f.manager, w, 8000, 61); + await w.finish(10000); + const m = f.manifests.at(-1).body; + assert.deepEqual(m.gaps.map(g => [g.start_ms, g.duration_ms, g.reason]), [[60000, 1000, 'truncated']]); assert.equal(m.capture_complete, false); + await f.manager.shutdown(); +}); +test('T-ST-14 a segment failing after initialize is aborted and its index reused; a 409 on reuse closes partial', async () => { + let failures = 0; + const f = await managerV1(base => ({ putContent: async (s, u, src, sig) => { + if (base._first === undefined) base._first = u.occurrence_id; + if (u.occurrence_id === base._first) { failures++; throw new AttachmentTransportError('incomplete', true, 0, 500); } + return base.putContent(s, u, src, sig); + } }), { limits: { segment_duration_ms: 1000 } }); + const w = f.manager.startRecording(identity, mono(16000)); + await writeSeconds(f.manager, w, 16000, 3, { flushEvery: 1 }); + await w.finish(10000); + assert.equal(failures, 3); assert.equal(f.calls.filter(c => c.op === 'abort').length, 1); + assert.deepEqual(f.requests.map(r => r.segment_index), [0, 0, 1]); + const m = f.manifests.at(-1).body; + assert.deepEqual(m.gaps.map(g => [g.start_ms, g.duration_ms, g.reason]), [[0, 1000, 'incomplete']]); + assert.deepEqual(m.segments.map(s => [s.index, s.start_ms]), [[0, 1000], [1, 2000]]); + await f.manager.shutdown(); + // Reuse rejected: seg0 ok, seg1 fails, seg2 (reusing index 1) gets 409 -> partial with the published prefix. + let n = 0; + const g = await managerV1(base => ({ + putContent: async (s, u, src, sig) => { if (u.occurrence_id === base._second) throw new AttachmentTransportError('incomplete', true, 0, 500); return base.putContent(s, u, src, sig); }, + initialize: async (s, body, key, sig) => { + n++; const r = JSON.parse(body); + if (n === 2) base._second = r.occurrence_id; + if (n === 3) throw new AttachmentTransportError('conflict', false, 0, 409); + return base.initialize(s, body, key, sig); + } }), { limits: { segment_duration_ms: 1000 } }); + const w2 = g.manager.startRecording(identity, mono(16000)); + await writeSeconds(g.manager, w2, 16000, 3, { flushEvery: 1 }); + const result = await w2.finish(10000); + assert.equal(result.state, 'partial'); + const last = g.manifests.at(-1).body; + assert.deepEqual(last.segments.map(s => s.index), [0]); assert.equal(last.gaps.at(-1).reason, 'incomplete'); assert.equal(last.duration_ms, 3000); + await g.manager.shutdown(); +}); +test('T-ST-15 intermediate manifests are prefix-monotonic across 10 PUTs; T-ST-16 latest completed capability only', async () => { + const f = await managerV1({}, { limits: { segment_duration_ms: 1000 } }); + const w = f.manager.startRecording(identity, mono(8000)); + await writeSeconds(f.manager, w, 8000, 40, { flushEvery: 1 }); + await w.finish(10000); + assert.ok(f.manifests.length >= 10, `manifests ${f.manifests.length}`); + for (let i = 1; i < f.manifests.length; i++) { + const a = f.manifests[i - 1].body; const b = f.manifests[i].body; + const has = (list, item) => list.some(x => JSON.stringify(x) === JSON.stringify(item)); + assert.ok(a.segments.every(x => has(b.segments, x)) && a.gaps.every(x => has(b.gaps, x)), `manifest ${i} dropped an entry`); + assert.ok(b.duration_ms >= a.duration_ms); + } + assert.deepEqual(f.manifests.map(m => m.key), f.manifests.map((_, i) => `mf-${hex(w.recordingId)}-${i + 1}`)); + for (const m of f.manifests) { + const top = m.body.segments.at(-1); + assert.equal(m.capability, f.completed.get(top.occurrence_id), 'manifest must use the latest completed segment capability'); + } + assert.equal(f.manifests.at(-1).body.capture_complete, true); + await f.manager.shutdown(); +}); +test('T-ST-17 capture_complete only with zero gaps (F01 unmarked-capture-gap, complete-without-any-media)', () => { + const rid = '40000000-0000-4000-8000-000000000001'; const track = '40000000-0000-4000-8000-000000000002'; + const base = { schema_version: 1, recording_id: rid, timebase: 'session_relative_ms', sample_format: 's16le', sample_rate_hz: 16000, channels: 1, codec: 'pcm', container: 'wav', + layout: 'mixed', tracks: [{ track_id: track, channels: [0], direction: 'mixed', role: 'mixed' }], segments: [], gaps: [{ start_ms: 0, duration_ms: 10, reason: 'incomplete' }], playout: [], capture_complete: true, duration_ms: 10 }; + assert.equal(validAttachmentManifest(base), false); assert.equal(validAttachmentManifest({ ...base, capture_complete: false }), true); +}); +test('T-ST-18 tryWrite never performs I/O on the caller (10 000 calls, blocked transport)', async () => { + const f = await managerV1(() => ({ putContent: (_s, _u, _src, signal) => new Promise((_, reject) => signal.addEventListener('abort', () => reject(new AttachmentTransportError('incomplete')), { once: true })) })); + const w = f.manager.startRecording(identity, mono(16000)); + const before = f.calls.length; const times = []; + const chunk = pcm(320); + for (let i = 0; i < 10000; i++) { const t = process.hrtime.bigint(); w.tryWrite(chunk); times.push(Number(process.hrtime.bigint() - t) / 1e6); } + assert.equal(f.calls.length, before, 'tryWrite performed transport I/O synchronously'); + times.sort((a, b) => a - b); console.log(`# T-ST-18 p99 tryWrite ms ${times[Math.floor(times.length * 0.99)].toFixed(4)} (recorded, not asserted)`); + w.abort(); await f.manager.shutdown(100); +}); +test('T-ST-19 finish(1000) with a blocked transport returns within 1.25 s', async () => { + const f = await managerV1(() => ({ initialize: () => new Promise(() => {}) }), { limits: { segment_duration_ms: 1000 } }); + const w = f.manager.startRecording(identity, mono(16000)); + await writeSeconds(f.manager, w, 16000, 2, { flushEvery: 0 }); + const start = Date.now(); const result = await w.finish(1000); const elapsed = Date.now() - start; + assert.ok(elapsed <= 1250, `finish took ${elapsed} ms`); assert.ok(['partial', 'unpublished'].includes(result.state)); + await f.manager.shutdown(100); +}); +test('recording abort is synchronous and PUTs no manifest', async () => { + const f = await managerV1(); + const w = f.manager.startRecording(identity, mono(16000)); + await writeSeconds(f.manager, w, 16000, 1, { flushEvery: 0 }); + w.abort(); assert.equal(f.manager.status().ownedBytes, 0); + assert.equal((await w.finish()).state, 'unpublished'); assert.equal(f.manifests.length, 0); + await f.manager.shutdown(); +}); + +// ---- 10.3 Renewal ------------------------------------------------------------------------------------- +function httpDouble(handler) { + const requests = []; + const client = async request => { + const body = request.body === undefined ? undefined : typeof request.body === 'string' ? request.body : Buffer.from(await readAll(request.body)); + const entry = { url: new URL(request.url).pathname, method: request.method, headers: request.headers, body }; requests.push(entry); + const out = await handler(entry, requests); + return { status: out.status, body: Buffer.from(out.body === undefined ? '' : typeof out.body === 'string' ? out.body : JSON.stringify(out.body)), retryAfter: out.retryAfter }; + }; + return { requests, client }; +} +const sessionBody = (sid, extra = {}) => ({ schema_version: 1, scope: { organization_id: ids[0], workspace_id: ids[1], project_id: ids[2] }, capture_session_id: sid, + policy: { ...policy, allowed_sources: ['Bytes', 'Blob', 'OwnedRecordingFile', 'ProviderFile', 'RemoteUrl'], record_audio: true }, limits, + expires_at: new Date(Date.now() + 3600000).toISOString(), transfer_modes: ['authenticated_stream'], ...extra }); +let capCounter = 0; +function standardRoutes(entry, overrides = {}) { + const path = entry.url; + if (path.endsWith('/attachment-sessions')) return overrides.session ? overrides.session(entry) : { status: 200, body: sessionBody(JSON.parse(entry.body).capture_session_id) }; + if (path.endsWith('/uploads')) { + if (overrides.upload) { const out = overrides.upload(entry); if (out) return out; } + const r = JSON.parse(entry.body); + return { status: 201, body: { upload_id: '50000000-0000-4000-8000-000000000001', attachment_id: r.attachment_id, occurrence_id: r.occurrence_id, state: 'pending_upload', + reserved_bytes: r.declared_bytes, expires_at: new Date(Date.now() + 600000 + (++capCounter) * 1000).toISOString(), transfer_mode: 'authenticated_stream', write_capability: `synthetic-capability-${capCounter}-abcdef` } }; + } + if (path.endsWith('/content')) return overrides.content ? overrides.content(entry) : { status: 200, body: { upload_id: path.split('/')[7], state: 'pending_upload', reason: 'queued' } }; + if (path.endsWith('/complete')) return { status: 202, body: { upload_id: path.split('/')[7], state: 'validating', reason: 'binding_pending' } }; + if (path.endsWith('/abort')) return { status: 200, body: { upload_id: path.split('/')[7], state: 'failed', reason: 'incomplete' } }; + if (path.endsWith('/manifest')) return { status: 204 }; + if (path.endsWith('/imports')) { const b = JSON.parse(entry.body); return overrides.imports ? overrides.imports(entry) : { status: 202, body: { mode: 'completed_recording', job_id: '60000000-0000-4000-8000-000000000001', recording_id: b.recording.recording_id, state: 'importing' } }; } + if (path.includes('/imports/') && path.endsWith('/status')) return { status: 200, body: { job_id: path.split('/')[7], state: 'importing', reason: 'binding_pending' } }; + return { status: 404 }; +} +async function httpManager(overrides = {}, managerOptions = {}) { + const d = httpDouble(entry => standardRoutes(entry, overrides)); + const transport = createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic-original', 'x-api-key': 'synthetic-original-key' }, client: d.client }); + const manager = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport, recordAudio: true, ...managerOptions }); + await manager.ready(); + return { manager, ...d }; +} +test('T-RN-01 session near expiry renews with the same id and nonce under key bs-{sid}-1', async () => { + const f = await httpManager({ session: entry => ({ status: 200, body: sessionBody(JSON.parse(entry.body).capture_session_id, { expires_at: new Date(Date.now() + 50000).toISOString() }) }) }); + f.manager.preflight(identity, { ...options, maxBytes: 4 }).release(); + for (let i = 0; i < 20 && f.requests.filter(r => r.url.endsWith('/attachment-sessions')).length < 2; i++) await tick(); + const boots = f.requests.filter(r => r.url.endsWith('/attachment-sessions')); + assert.equal(boots.length, 2); + assert.equal(boots[0].headers['idempotency-key'], `bs-${hex(f.manager.sessionId)}-0`); assert.equal(boots[1].headers['idempotency-key'], `bs-${hex(f.manager.sessionId)}-1`); + assert.equal(boots[1].body, boots[0].body); assert.equal(JSON.parse(boots[1].body).correlation_nonce, f.manager.nonce); + assert.equal(boots[1].headers.authorization, 'Bearer synthetic-original'); + await f.manager.shutdown(); +}); +test('T-RN-02 renewal with a different scope invalidates the manager (scope_unresolved); T-RN-03 tighter policy drops queued work', async () => { + let renew; + const f = await managerV1(base => ({ renewSession: () => new Promise(resolve => { renew = resolve; }) }), { lifetimeMs: 50000 }); + f.manager.preflight(identity, { ...options, maxBytes: 4 }).release(); await tick(); + renew({ ...sessionBody(f.manager.sessionId), scope: { organization_id: ids[0], workspace_id: ids[1], project_id: '70000000-0000-4000-8000-000000000001' } }); + for (let i = 0; i < 10; i++) await tick(); + assert.equal(f.manager.status().state, 'scope_unresolved'); + assert.equal(f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array(2) }, options).admission, 'scope_unresolved'); + let renew2; let unblock; + const block = new Promise(resolve => { unblock = resolve; }); + const g = await managerV1(base => ({ renewSession: () => new Promise(resolve => { renew2 = resolve; }), initialize: async (...a) => { await block; return base.initialize(...a); } }), { lifetimeMs: 50000 }); + const handles = [1, 2, 3].map(() => g.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array(2) }, options)); + await tick(); + renew2({ ...sessionBody(g.manager.sessionId), policy: { ...sessionBody(g.manager.sessionId).policy, version: 'v2', allowed_modalities: ['audio'] } }); + for (let i = 0; i < 10; i++) await tick(); + unblock(); + assert.deepEqual((await Promise.all(handles.map(h => h.completion))).map(c => c.status), ['policy_denied', 'policy_denied', 'policy_denied']); + assert.equal(g.manager.status().ownedBytes, 0); + assert.equal(g.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array(2) }, { mimeType: 'audio/wav', direction: 'mixed' }).admission, 'queued'); + await f.manager.shutdown(1); await g.manager.shutdown(); +}); +test('T-RN-04 renewal fails until expiry: state expired, recordings get incomplete gaps, initialized uploads continue', async () => { + const c = clock(Date.now()); let unblock; const block = new Promise(resolve => { unblock = resolve; }); + const f = await managerV1(base => ({ renewSession: async () => { throw new AttachmentTransportError('incomplete', true); }, + putContent: async (...a) => { await block; return base.putContent(...a); } }), { clock: c, lifetimeMs: 30000 }, { clock: c }); + const h = f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array(4) }, options); + const w = f.manager.startRecording(identity, mono(16000)); + assert.equal(w.tryWrite(pcm(16000)), true); + for (let i = 0; i < 20 && !f.calls.some(x => x.op === 'putContent' || x.op === 'initialize'); i++) await tick(); + c.advance(30000); + assert.equal(f.manager.status().state, 'expired'); + assert.equal(f.manager.preflight(identity, { ...options, maxBytes: 4 }).reason, 'expired'); + assert.equal(w.tryWrite(pcm(16000)), false); + unblock(); assert.equal((await h.completion).status, 'validating'); + const result = await w.finish(2000); assert.notEqual(result.state, 'published'); + await f.manager.shutdown(100); +}); +test('T-RN-05 capability near expiry before content: replay with the byte-identical body under up-{occ}; same upload_id', async () => { + let inits = 0; + const f = await httpManager({ upload: entry => { + inits++; const r = JSON.parse(entry.body); + const expires = inits === 1 ? Date.now() + 3000 : Date.now() + 600000; + return { status: 201, body: { upload_id: '50000000-0000-4000-8000-000000000001', attachment_id: r.attachment_id, occurrence_id: r.occurrence_id, state: 'pending_upload', + reserved_bytes: r.declared_bytes, expires_at: new Date(expires).toISOString(), transfer_mode: 'authenticated_stream', write_capability: `synthetic-capability-renewed-${inits}` } }; + } }); + const h = f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1, 2, 3]) }, options); + assert.equal((await h.completion).status, 'validating'); + const uploads = f.requests.filter(r => r.url.endsWith('/uploads')); + assert.equal(uploads.length, 2); assert.equal(uploads[1].body, uploads[0].body); + assert.equal(uploads[0].headers['idempotency-key'], `up-${hex(h.occurrenceId)}`); assert.equal(uploads[1].headers['idempotency-key'], uploads[0].headers['idempotency-key']); + assert.equal(uploads[1].headers.authorization, 'Bearer synthetic-original'); + assert.equal(f.requests.find(r => r.url.endsWith('/content')).headers.authorization, 'Bearer synthetic-capability-renewed-2'); + await f.manager.shutdown(); +}); +test('T-RN-06 replay returning the same capability: upload expired and aborted, no new upload id', async () => { + const f = await httpManager({ upload: entry => { + const r = JSON.parse(entry.body); + return { status: 201, body: { upload_id: '50000000-0000-4000-8000-000000000001', attachment_id: r.attachment_id, occurrence_id: r.occurrence_id, state: 'pending_upload', + reserved_bytes: r.declared_bytes, expires_at: new Date(Date.now() + 3000).toISOString(), transfer_mode: 'authenticated_stream', write_capability: 'synthetic-capability-same-0000' } }; + } }); + const h = f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, options); + assert.equal((await h.completion).status, 'expired'); + assert.equal(f.requests.filter(r => r.url.endsWith('/abort')).length, 1); + assert.equal(new Set(f.requests.filter(r => r.url.endsWith('/uploads')).map(r => JSON.parse(r.body).occurrence_id)).size, 1); + await f.manager.shutdown(); +}); +test('T-RN-07 401 on content: one renewal then one retry; a second 401 is credential_scope_unsupported', async () => { + let puts = 0; + const ok = await httpManager({ content: entry => (++puts === 1 ? { status: 401 } : { status: 200, body: { upload_id: entry.url.split('/')[7], state: 'pending_upload', reason: 'queued' } }) }); + assert.equal((await ok.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, options).completion).status, 'validating'); + assert.equal(ok.requests.filter(r => r.url.endsWith('/uploads')).length, 2); + const bad = await httpManager({ content: () => ({ status: 401 }) }); + assert.equal((await bad.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, options).completion).status, 'credential_scope_unsupported'); + assert.equal(bad.requests.filter(r => r.url.endsWith('/content')).length, 2); + await ok.manager.shutdown(); await bad.manager.shutdown(); +}); +test('T-RN-08 the write capability never appears in status, handles, span attributes or errors', async () => { + const f = await httpManager({ content: () => ({ status: 500 }) }); + const s = span(); + const h = f.manager.attach(s, { kind: 'Bytes', data: new Uint8Array([1]) }, options); + const completion = await h.completion; + const caps = f.requests.filter(r => r.url.endsWith('/content')).map(r => r.headers.authorization.slice(7)); + const haystack = JSON.stringify([f.manager.status(), h, completion, s.attrs, String(new AttachmentTransportError(completion.status))]); + for (const cap of caps) assert.equal(haystack.includes(cap), false); + await f.manager.shutdown(); +}); + +// ---- 10.5 Completed import ------------------------------------------------------------------------------- +const providerResolved = { source: { kind: 'ProviderFile', connectionId: '80000000-0000-4000-8000-000000000001', registeredOutputId: '80000000-0000-4000-8000-000000000002' }, + declaredBytes: 172800044, declaredDurationMs: 3600000, mimeType: 'audio/wav', direction: 'mixed' }; +const resolver = value => ({ calls: 0, async resolve() { this.calls++; return typeof value === 'function' ? value() : value; } }); +test('T-IM-01 provider import: RecordingImportRequest body, key im-{rid}, original credentials; T-IM-09 no capability; T-IM-07 one status GET', async () => { + const f = await httpManager(); const s = span(); const r = resolver(providerResolved); + const h = await f.manager.importCompletedRecording(s, { provider: 'livekit_egress', providerRef: 'EG_synthetic1' }, { resolver: r }); + assert.equal((await h.completion).status, 'importing'); assert.equal(h.jobId, '60000000-0000-4000-8000-000000000001'); + const post = f.requests.find(x => x.url.endsWith('/imports')); const body = JSON.parse(post.body); + assert.equal(validAttachmentWire('RecordingImportRequest', body), true); + assert.equal(post.headers['idempotency-key'], `im-${hex(body.recording.recording_id)}`); assert.equal(post.headers.authorization, 'Bearer synthetic-original'); + assert.equal(body.recording.declared_bytes, 172800044); assert.equal(body.recording.source_kind, 'ProviderFile'); assert.equal(body.recording.deletion_generation, 0); + assert.equal(s.attrs['fi.attachment.refs'], '[]'); assert.equal(s.attrs['fi.attachment.capture_session_id'], f.manager.sessionId); + const before = f.requests.length; assert.deepEqual(await h.status(), { state: 'importing', reason: 'binding_pending' }); assert.equal(f.requests.length, before + 1); + assert.equal(f.requests.at(-1).headers.authorization, 'Bearer synthetic-original'); + // T-IM-08: no URL, path, header or credential in the body. + assert.equal(/https?:|\/tmp|synthetic-original|credential/.test(post.body), false); + await f.manager.shutdown(); +}); +test('T-IM-02 denied preflight never calls the resolver; T-IM-03 timeout or throw -> source_unavailable', async () => { + const f = await managerV1({}, {}, { traceConfig: { hideOutputs: true } }); const r = resolver(providerResolved); + assert.equal((await f.manager.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_1' }, { resolver: r })).reason, 'policy_denied'); assert.equal(r.calls, 0); + const g = await managerV1(); + const slow = { resolve: () => new Promise(() => {}) }; + const start = Date.now(); const h = await g.manager.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_1' }, { resolver: slow, timeoutMs: 200 }); + assert.equal(h.reason, 'source_unavailable'); assert.ok(Date.now() - start < 1000); + const thrower = { resolve: async () => { throw Error('synthetic-secret-detail'); } }; + const t = await g.manager.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_1' }, { resolver: thrower }); + assert.equal(t.reason, 'source_unavailable'); assert.equal(JSON.stringify(t).includes('synthetic-secret-detail'), false); + await f.manager.shutdown(1); await g.manager.shutdown(1); +}); +test('T-IM-04 out-of-range size or duration -> too_large before submit; T-IM-05 narrowing mixed -> policy_denied; T-IM-06 envelope', async () => { + const f = await managerV1(); + for (const bad of [{ declaredBytes: 1073741825 }, { declaredDurationMs: 7200001 }]) { + assert.equal((await f.manager.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_1' }, { resolver: resolver({ ...providerResolved, ...bad }) })).reason, 'too_large'); + } + assert.equal((await f.manager.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_1', direction: 'mixed' }, { resolver: resolver({ ...providerResolved, direction: 'output' }) })).reason, 'policy_denied'); + const widened = await f.manager.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_1', direction: 'output' }, { resolver: resolver({ ...providerResolved, direction: 'mixed' }) }); + assert.equal((await widened.completion).status, 'importing'); assert.equal(f.imports.at(-1).body.recording.direction, 'mixed'); + const envelope = await f.manager.importCompletedRecording(identity, { provider: 'trusted_importer', providerRef: 'ENV_1' }, { resolver: resolver({ ...providerResolved, source: { kind: 'RemoteUrlEnvelope', envelopeId: '80000000-0000-4000-8000-000000000003' } }) }); + assert.equal((await envelope.completion).status, 'importing'); + assert.deepEqual(f.imports.at(-1).body.source, { kind: 'RemoteUrlEnvelope', envelope_id: '80000000-0000-4000-8000-000000000003' }); assert.equal(f.imports.at(-1).body.recording.source_kind, 'RemoteUrl'); + assert.equal(f.calls.filter(c => c.op === 'submitImport').length, 2); + await f.manager.shutdown(); +}); +test('T-IM-10 429 with Retry-After 1 three times -> throttled within the deadline', async () => { + const f = await httpManager({ imports: () => ({ status: 429, retryAfter: '1' }) }); + const start = Date.now(); + const h = await f.manager.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_1' }, { resolver: resolver(providerResolved) }); + assert.equal((await h.completion).status, 'throttled'); assert.equal(f.requests.filter(r => r.url.endsWith('/imports')).length, 3); + assert.ok(Date.now() - start < 5000); + await f.manager.shutdown(); +}); + +// ---- 10.6 Consumed streams ------------------------------------------------------------------------------- +test('T-CS-01 app-owned reader: chunks observed after the app reads; no extra reads, no clone or tee', async () => { + const f = await managerV1(); const data = [new Uint8Array([1, 2]), new Uint8Array([3, 4, 5])]; + let pulls = 0; + const body = new ReadableStream({ pull(controller) { pulls++; const next = data.shift(); if (next) controller.enqueue(next); else controller.close(); } }); + const stream = f.manager.startAttachmentStream(identity, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 64 }); + const reader = body.getReader(); // the APPLICATION obtained the reader + const read = reader.read.bind(reader); let appReads = 0; + reader.read = async () => { appReads++; const r = await read(); if (!r.done) stream.tryWrite(r.value); else stream.finish(); return r; }; + while (!(await reader.read()).done) { /* app consumes */ } + assert.ok(pulls <= appReads + 1); + const h = stream.finish(); assert.equal((await h.completion).status, 'validating'); + assert.deepEqual([...f.contents.get(h.occurrenceId)], [1, 2, 3, 4, 5]); + await f.manager.shutdown(); +}); +test('T-CS-02 over the cap: all discarded, too_large, nothing uploaded; T-CS-03 app cancel -> incomplete, budget released', async () => { + const f = await managerV1(); const s = span(); + const stream = f.manager.startAttachmentStream(s, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 4 }); + assert.equal(stream.tryWrite(new Uint8Array(3)), true); assert.equal(stream.tryWrite(new Uint8Array(3)), false); + assert.equal(stream.reason, 'too_large'); assert.equal((await stream.finish().completion).status, 'too_large'); + assert.equal(s.attrs['fi.attachment.refs'], undefined, 'an over-cap stream projects nothing'); + const other = f.manager.startAttachmentStream(identity, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 64 }); + other.tryWrite(new Uint8Array(8)); other.abort(); + assert.equal(other.reason, 'incomplete'); assert.equal(f.manager.status().ownedBytes, 0); assert.equal(f.manager.status().openStreams, 0); + await f.manager.flush(); assert.equal(f.ops().includes('initialize'), false); + await f.manager.shutdown(); +}); +test('T-CS-04 abandoned stream aborted at the next entry point after idle TTL (fake clock, no timer)', async () => { + const c = clock(); const f = await managerV1({}, { clock: c }); + const stream = f.manager.startAttachmentStream(identity, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 64, idleTtlMs: 1000 }); + stream.tryWrite(new Uint8Array(4)); c.advance(1001); + assert.equal(f.manager.status().openStreams, 0); assert.equal(stream.reason, 'incomplete'); assert.equal(f.manager.status().ownedBytes, 0); + await f.manager.shutdown(1); +}); +test('T-CS-06 BYOB buffer reused after tryWrite: uploaded bytes equal the bytes at write time', async () => { + const f = await managerV1(); + const stream = f.manager.startAttachmentStream(identity, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 64 }); + const buffer = new Uint8Array([9, 9, 9]); stream.tryWrite(buffer); buffer.fill(0); stream.tryWrite(buffer.subarray(0, 1)); + const h = stream.finish(); assert.equal((await h.completion).status, 'validating'); + assert.deepEqual([...f.contents.get(h.occurrenceId)], [9, 9, 9, 0]); + await f.manager.shutdown(); +}); + +// ---- 10.7 Channel and role ----------------------------------------------------------------------------------- +test('T-CH-01 UploadRequest never carries role or channel; T-CH-02 the live reference does', async () => { + const f = await managerV1(); const s = span(); + const h = f.manager.attach(s, { kind: 'Bytes', data: new Uint8Array([1]) }, { mimeType: 'audio/wav', direction: 'output', role: 'assistant', channel: 1, messageIndex: 2 }); + await h.completion; + assert.equal('role' in f.requests[0] || 'channel' in f.requests[0], false); assert.equal(validAttachmentWire('UploadRequest', f.requests[0]), true); + const refs = JSON.parse(s.attrs['fi.attachment.refs']); assert.equal(refs[0].role, 'assistant'); assert.equal(refs[0].channel, 1); + assert.equal(validAttachmentWire('SpanAttributes', s.attrs), true); + await f.manager.shutdown(); +}); +test('T-CH-03 span ended before attach: no projection, the upload still proceeds; T-CH-05 carrier stamp only on admitted targets', async () => { + const f = await managerV1(); const ended = span(); ended.end(); + const h = f.manager.attach(ended, { kind: 'Bytes', data: new Uint8Array([1]) }, options); + assert.equal((await h.completion).status, 'validating'); assert.equal(ended.writes, 0); + const bystander = span(); f.manager.correlation(bystander); assert.equal(bystander.writes, 0, 'non-target spans are never stamped'); + const denied = span(); f.manager.attach(denied, { kind: 'Bytes', data: new Uint8Array([1]) }, { ...options, capture: { captureContent: false } }); + assert.equal(denied.writes, 0, 'a denied capture stamps nothing'); + const aborted = span(); const st = f.manager.startAttachmentStream(aborted, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 8 }); st.tryWrite(new Uint8Array(1)); st.abort(); + assert.equal(aborted.writes, 0, 'an aborted stream projects nothing'); + for (const make of [s => f.manager.attach(s, { kind: 'Bytes', data: new Uint8Array([1]) }, options), + s => { const x = f.manager.startAttachmentStream(s, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 8 }); x.tryWrite(new Uint8Array(1)); x.finish(); }, + s => f.manager.startRecording(s, mono(16000)).abort()]) { + const target = span(); make(target); + assert.equal(validAttachmentWire('SpanAttributes', target.attrs), true, JSON.stringify(target.attrs)); + } + await f.manager.flush(); await f.manager.shutdown(); +}); +test('T-CH-06 explicit workspace with a different response scope -> scope_unresolved', async () => { + const f = await managerV1({}, { ready: false }, { workspaceId: '90000000-0000-4000-8000-000000000001' }); + assert.equal(await f.manager.ready(), false); assert.equal(f.manager.status().state, 'scope_unresolved'); + assert.equal(f.manager.attach(identity, source(), { mimeType: 'audio/wav', direction: 'mixed' }).admission, 'scope_unresolved'); + await f.manager.shutdown(1); +}); + +// ---- 10.8 Lifecycle ---------------------------------------------------------------------------------------- +test('T-LC-02/03/04/05 disabled null object: no timers, sockets, listeners, finalizers or span writes; every call immediate', async () => { + const counts = { timers: 0, intervals: 0, registry: 0 }; + const saved = { st: global.setTimeout, si: global.setInterval, fr: global.FinalizationRegistry }; + global.setTimeout = (...a) => { counts.timers++; return saved.st(...a); }; + global.setInterval = (...a) => { counts.intervals++; return saved.si(...a); }; + global.FinalizationRegistry = class extends saved.fr { constructor(...a) { counts.registry++; super(...a); } }; + const listenersBefore = process.eventNames().map(n => process.listenerCount(n)).reduce((a, b) => a + b, 0); + try { + const { createNodeAttachmentManager } = require('./attachments-v1.cjs').nodeCore; + const m = createNodeAttachmentManager({ environment: () => ({}) }); + assert.equal(m.constructor.name, 'DisabledAttachmentManager'); + const s = span(); const start = process.hrtime.bigint(); + assert.equal(await m.ready(), false); assert.equal(m.status().state, 'disabled'); + let touched = false; + const handle = m.attach(s, { kind: 'Bytes', get data() { touched = true; return new Uint8Array(1); } }, options); + assert.equal(handle.admission, 'disabled'); assert.deepEqual(await handle.completion, { status: 'disabled' }); + assert.equal(m.preflight(s, { ...options, maxBytes: 1 }).reason, 'disabled'); + const w = m.startRecording(s, mono(16000)); assert.equal(w.tryWrite(pcm(10)), false); assert.equal((await w.finish()).state, 'denied'); + const st = m.startAttachmentStream(s, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 8 }); assert.equal(st.tryWrite(new Uint8Array(1)), false); + let resolved = false; + assert.equal((await m.importCompletedRecording(s, { provider: 'x', providerRef: 'y' }, { resolver: { resolve: async () => { resolved = true; return 'source_unavailable'; } } })).reason, 'disabled'); + assert.equal(m.attachOwnedRecording(s, { kind: 'OwnedRecordingFile', lease: {} }, { mimeType: 'audio/wav', direction: 'mixed' }).reason, 'disabled'); + m.updatePolicy({ enabled: true }); assert.equal(m.status().state, 'disabled'); + assert.deepEqual(await m.flush(), { unfinished: 0, dropped: 0 }); assert.deepEqual(await m.shutdown(), { unfinished: 0, dropped: 0 }); + const elapsed = Number(process.hrtime.bigint() - start) / 1e6; + assert.equal(touched, false); assert.equal(resolved, false); assert.equal(s.writes, 0); + assert.ok(elapsed < 50, `disabled calls took ${elapsed} ms`); + } finally { global.setTimeout = saved.st; global.setInterval = saved.si; global.FinalizationRegistry = saved.fr; } + const listenersAfter = process.eventNames().map(n => process.listenerCount(n)).reduce((a, b) => a + b, 0); + assert.deepEqual(counts, { timers: 0, intervals: 0, registry: 0 }); assert.equal(listenersAfter, listenersBefore); +}); +test('T-LC-12 status().state is "ready" after bootstrap', async () => { + const f = await managerV1(); assert.equal(f.manager.status().state, 'ready'); await f.manager.shutdown(); +}); +test('T-LC-15 shutdown is idempotent and concurrent; T-LC-16 returns within timeout + 250 ms with a transport that ignores abort', async () => { + const f = await managerV1(() => ({ putContent: () => new Promise(() => {}) })); + f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array(4) }, options); + await tick(); await tick(); + const start = Date.now(); const a = f.manager.shutdown(500); const b = f.manager.shutdown(500); + assert.equal(a, b); const result = await a; const elapsed = Date.now() - start; + assert.ok(elapsed <= 750, `shutdown took ${elapsed} ms`); assert.ok(result.unfinished >= 1); +}); + +// ---- 10.9 Transport and wire ------------------------------------------------------------------------------- +test('T-TR-02 status -> reason table, Error.code refinement only within class', () => { + const body = code => Buffer.from(JSON.stringify({ code, request_id: '10000000-0000-4000-8000-000000000009' })); + const rows = [[400, 'unsupported_version', false], [401, 'credential_scope_unsupported', false], [403, 'policy_denied', false], [404, 'not_found', false], + [409, 'conflict', false], [410, 'expired', false], [413, 'too_large', false], [429, 'throttled', true], [408, 'incomplete', true], [500, 'incomplete', true], + [502, 'incomplete', true], [503, 'incomplete', true], [504, 'incomplete', true], [418, 'incomplete', false], [302, 'incomplete', false]]; + for (const [status, code, retryable] of rows) { const e = attachmentHttpError(status); assert.equal(e.code, code, String(status)); assert.equal(e.retryable, retryable, String(status)); } + assert.equal(attachmentHttpError(409, body('deleted')).code, 'deleted'); assert.equal(attachmentHttpError(403, body('redaction_unavailable')).code, 'redaction_unavailable'); + assert.equal(attachmentHttpError(401, body('expired')).code, 'expired'); assert.equal(attachmentHttpError(404, body('scope_unresolved')).code, 'scope_unresolved'); + assert.equal(attachmentHttpError(403, body('deleted')).code, 'policy_denied', 'out-of-class code ignored'); + assert.equal(attachmentHttpError(500, body('deleted')).code, 'incomplete'); + assert.equal(attachmentHttpError(409, Buffer.from('{"code":"deleted","code":"conflict","request_id":"10000000-0000-4000-8000-000000000009"}')).code, 'conflict', 'duplicate members ignored'); + assert.equal(attachmentHttpError(429, undefined, '3').retryAfterMs, 3000); assert.equal(attachmentHttpError(429, undefined, '60').retryAfterMs, 5000); + assert.equal(attachmentHttpError(429, undefined, 'Wed, 21 Oct 2015 07:28:00 GMT').retryAfterMs, 1000); assert.equal(attachmentHttpError(429).retryAfterMs, 1000); +}); +test('T-TR-03 idempotency keys on every route and byte-identical replays', async () => { + let inits = 0; + const f = await httpManager({ upload: entry => (++inits === 1 ? { status: 503 } : undefined), content: () => ({ status: 409 }) }); + const h = f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, options); + assert.equal((await h.completion).status, 'conflict'); + const uploads = f.requests.filter(r => r.url.endsWith('/uploads')); + assert.equal(uploads.length, 2); assert.equal(uploads[0].body, uploads[1].body); assert.equal(uploads[0].headers['idempotency-key'], `up-${hex(h.occurrenceId)}`); + const abort = f.requests.find(r => r.url.endsWith('/abort')); + assert.equal(abort.headers['idempotency-key'], `ab-${hex('50000000-0000-4000-8000-000000000001')}`); assert.match(abort.headers.authorization, /^Bearer synthetic-capability-/); + assert.equal(attachmentIdempotencyKey('complete', '50000000-0000-4000-8000-000000000001'), 'cp-50000000000040008000000000000001'); + const g = await httpManager({}, {}); + await g.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, options).completion; + const complete = g.requests.find(r => r.url.endsWith('/complete')); + assert.equal(complete.headers['idempotency-key'], `cp-${hex('50000000-0000-4000-8000-000000000001')}`); assert.deepEqual(JSON.parse(complete.body), { policy_version: 'v1', deletion_generation: 0 }); + const w = g.manager.startRecording(identity, mono(8000)); w.tryWrite(pcm(8000)); await w.finish(5000); + const manifest = g.requests.find(r => r.url.endsWith('/manifest')); + assert.equal(manifest.method, 'PUT'); assert.equal(manifest.headers['idempotency-key'], `mf-${hex(w.recordingId)}-1`); assert.match(manifest.headers.authorization, /^Bearer synthetic-capability-/); + assert.equal(manifest.headers['x-api-key'], undefined, 'capability routes never carry original credentials'); + await f.manager.shutdown(); await g.manager.shutdown(); +}); +test('T-TR-05 content PUT pacing honours upload_bytes_per_second', async () => { + const d = httpDouble(entry => standardRoutes(entry, { session: e => ({ status: 200, body: sessionBody(JSON.parse(e.body).capture_session_id, { limits: { ...limits, upload_bytes_per_second: 400000 } }) }) })); + const transport = createAttachmentHttpTransportV1({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic-original' }, client: async request => { + if (request.pacer && request.body && typeof request.body !== 'string') { const size = request.body.byteLength ?? request.body.size; for (let o = 0; o < size; o += 262144) await request.pacer.take(Math.min(262144, size - o), request.signal); } + return d.client(request); + } }); + const m = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport }); + await m.ready(); const start = Date.now(); + assert.equal((await m.attach(identity, { kind: 'Bytes', data: new Uint8Array(800000) }, { mimeType: 'audio/mpeg', direction: 'output' }).completion).status, 'validating'); + const elapsed = Date.now() - start; assert.ok(elapsed >= 700, `paced upload took ${elapsed} ms`); + const put = d.requests.find(r => r.url.endsWith('/content')); assert.equal(put.body.length, 800000); + await m.shutdown(); +}); +test('T-TR-06 oversized or duplicate-member responses -> unsupported_version; T-TR-07 missing capability rejected; T-TR-08 validating, never ready', async () => { + const big = httpDouble(entry => (entry.url.endsWith('/attachment-sessions') ? { status: 200, body: 'x'.repeat(70000) } : { status: 404 })); + const t1 = createAttachmentHttpTransportV1({ endpoint: 'https://attachments.example', headers: {}, client: big.client }); + await assert.rejects(t1.bootstrap({ capture_session_id: ids[0] }, 'bs-x-0', new AbortController().signal), e => e.code === 'unsupported_version'); + const dup = httpDouble(() => ({ status: 200, body: '{"schema_version":1,"schema_version":1}' })); + const t2 = createAttachmentHttpTransportV1({ endpoint: 'https://attachments.example', headers: {}, client: dup.client }); + await assert.rejects(t2.bootstrap({ capture_session_id: ids[0] }, 'bs-x-0', new AbortController().signal), e => e.code === 'unsupported_version'); + const f = await httpManager({ upload: entry => { const r = JSON.parse(entry.body); return { status: 201, body: { upload_id: '50000000-0000-4000-8000-000000000001', attachment_id: r.attachment_id, + occurrence_id: r.occurrence_id, state: 'pending_upload', reserved_bytes: r.declared_bytes, expires_at: new Date(Date.now() + 600000).toISOString(), transfer_mode: 'authenticated_stream' } }; } }); + assert.equal((await f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, options).completion).status, 'unsupported_version'); + assert.equal(f.requests.some(r => r.url.endsWith('/content')), false); + const g = await httpManager(); + assert.equal((await g.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, options).completion).status, 'validating'); + await f.manager.shutdown(); await g.manager.shutdown(); +}); +test('legacy four-method transports keep working through the adapter (stacked lanes)', async () => { + const { fake } = require('./attachments-fixtures.cjs'); const f = fake(); + const m = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: f.transport }); + assert.equal(await m.ready(), true); assert.equal(m.status().state, 'ready'); + assert.equal((await m.attach(identity, source(), { mimeType: 'audio/wav', direction: 'mixed' }).completion).status, 'validating'); + assert.deepEqual(f.calls, ['bootstrap', 'initialize', 'upload', 'complete']); + const w = m.startRecording(identity, mono(8000)); w.tryWrite(pcm(800)); const r = await w.finish(2000); + assert.equal(r.state, 'unpublished', 'legacy transports cannot PUT manifests: fail closed'); + await m.shutdown(); +}); diff --git a/typescript/packages/fi-core/test/attachments-exit-probe.cjs b/typescript/packages/fi-core/test/attachments-exit-probe.cjs new file mode 100644 index 000000000..ead56e30e --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-exit-probe.cjs @@ -0,0 +1,49 @@ +// One exit-probe case per process (design 8.5). Prints DONE and returns; the process must then exit by itself. +const path = require('node:path'); +const { createRequire } = require('node:module'); +const load = createRequire(process.env.FI_CORE_PACKAGE_JSON || path.resolve(__dirname, '../package.json')); +const { register, ProjectType } = load('@traceai/fi-core'); +const { v1, pcm } = require('./attachments-v1.cjs'); +const identity = { traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }; +const blocked = () => new Promise(() => {}); // ignores its abort signal entirely +const provider = (transport, extra = {}) => register({ projectName: 'probe', projectType: ProjectType.OBSERVE, setGlobalTracerProvider: false, spanProcessors: [], + attachments: { enabled: true, environment: () => ({}), recordAudio: true, transport, ...extra } }); +(async () => { + const name = process.argv[2]; + if (name === 'disabled') { + const p = register({ projectName: 'probe', projectType: ProjectType.OBSERVE, setGlobalTracerProvider: false, spanProcessors: [] }); + p.getTracer('probe').startSpan('s').end(); + await p.shutdown(); console.log(`DONE ${p.attachments.status().state}`); return; + } + if (name === 'enabled-idle') { const p = provider(v1().transport); await p.attachments.ready(); await p.shutdown(); console.log('DONE'); return; } + if (name === 'enabled-blocked') { + const p = provider(v1({ putContent: blocked }).transport, { shutdownTimeoutMs: 1000 }); + await p.attachments.ready(); p.attachments.attach(identity, { kind: 'Bytes', data: new Uint8Array(8) }, { mimeType: 'audio/wav', direction: 'mixed' }); + await new Promise(resolve => setImmediate(resolve)); + const result = await p.attachments.shutdown(1000); await p.shutdown(); + if (result.unfinished < 1) throw Error('expected unfinished'); + console.log(`DONE unfinished=${result.unfinished}`); return; + } + if (name === 'enabled-no-shutdown') { + const p = provider(v1({ putContent: blocked }).transport); await p.attachments.ready(); + p.attachments.attach(identity, { kind: 'Bytes', data: new Uint8Array(8) }, { mimeType: 'audio/wav', direction: 'mixed' }); + console.log('DONE'); return; // no shutdown: unref'd handles must not hold the process + } + if (name === 'mid-recording') { + const d = v1(); const p = provider(d.transport, { shutdownTimeoutMs: 10000 }); await p.attachments.ready(); + const w = p.attachments.startRecording(identity, { format: { sampleRateHz: 16000 }, layout: 'mixed', tracks: [{ channels: [0], direction: 'mixed', role: 'mixed' }], operation: 'observe_consumed' }); + for (let i = 0; i < 250; i++) w.tryWrite(pcm(320)); + await p.shutdown(); + const result = await w.finish(); + if (!(d.manifests.length >= 1 || result.state === 'partial')) throw Error(`no manifest: ${result.state}`); + console.log(`DONE ${result.state} manifests=${d.manifests.length}`); return; + } + if (name === 'import-pending') { + const p = provider(v1().transport); await p.attachments.ready(); + const h = await p.attachments.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_probe' }, { resolver: { resolve: () => new Promise(() => {}) }, timeoutMs: 1000 }); + await p.shutdown(); + if (h.reason !== 'source_unavailable') throw Error(h.reason); + console.log(`DONE ${h.reason}`); return; + } + throw Error(`unknown case ${name}`); +})().catch(error => { console.error(String(error && error.message)); process.exitCode = 1; }); diff --git a/typescript/packages/fi-core/test/attachments-exit.test.cjs b/typescript/packages/fi-core/test/attachments-exit.test.cjs new file mode 100644 index 000000000..5e1f2ccf0 --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-exit.test.cjs @@ -0,0 +1,17 @@ +/* Process-exit regression probes (design 8.5, T-LC-06..11). Each case runs in a separate Node process. */ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const packageJson = process.env.FI_CORE_PACKAGE_JSON || path.resolve(__dirname, '../package.json'); +const probe = path.join(__dirname, 'attachments-exit-probe.cjs'); +for (const [name, limit] of [['disabled', 10000], ['enabled-idle', 10000], ['enabled-blocked', 10000], ['enabled-no-shutdown', 10000], ['mid-recording', 15000], ['import-pending', 10000]]) { + test(`exit probe ${name} exits 0 under ${limit} ms`, () => { + const start = Date.now(); + const result = spawnSync(process.execPath, [probe, name], { env: { ...process.env, FI_CORE_PACKAGE_JSON: packageJson }, timeout: limit, encoding: 'utf8' }); + const elapsed = Date.now() - start; + assert.equal(result.signal, null, `${name} was killed (kept alive): ${result.stderr}`); + assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /DONE/); assert.ok(elapsed < limit); + console.log(`# probe ${name} ${elapsed} ms ${result.stdout.trim().split('\n').pop()}`); + }); +} diff --git a/typescript/packages/fi-core/test/attachments-files.test.cjs b/typescript/packages/fi-core/test/attachments-files.test.cjs new file mode 100644 index 000000000..94ff0bd3b --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-files.test.cjs @@ -0,0 +1,138 @@ +/* Owned recording files (design section 4, T-LS-*). Synthetic files under a private temp root only. */ +const { test, after } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { nodeCore, core, managerV1, readAll } = require('./attachments-v1.cjs'); +const { identity } = require('./attachments-fixtures.cjs'); +const { createRecordingFileScope } = nodeCore; +const { attachmentWavHeader, validAttachmentManifest, AttachmentTransportError } = core; +const live = setInterval(() => {}, 1000); +after(() => clearInterval(live)); +const MiB = 1024 * 1024; +function root() { return fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'owned-rec-'))); } +function pattern(size, seed = 1) { const b = Buffer.alloc(size); for (let i = 0; i < size; i++) b[i] = (i * 13 + seed + (i >> 11)) & 0xff; return b; } +function wavFile(file, { rate = 24000, channels = 2, frames, bits = 16, format = 1, badLength = false }) { + const data = pattern(frames * channels * (bits / 8), 7); + const header = Buffer.from(attachmentWavHeader(channels, rate, badLength ? data.length + 100 : data.length)); + header.writeUInt16LE(format, 20); header.writeUInt16LE(bits, 34); header.writeUInt16LE(channels * bits / 8, 32); + fs.writeFileSync(file, Buffer.concat([header, data])); + return data; +} +const stereo = [{ channels: [0], direction: 'input', role: 'user' }, { channels: [1], direction: 'output', role: 'assistant' }]; + +test('T-LS-02 symlink leaf and T-LS-03 symlinked intermediate are source_unavailable; foreign refs fail closed', async () => { + const dir = root(); fs.writeFileSync(path.join(dir, 'real.ogg'), pattern(100)); + fs.symlinkSync(path.join(dir, 'real.ogg'), path.join(dir, 'link.ogg')); + fs.mkdirSync(path.join(dir, 'real-dir')); fs.writeFileSync(path.join(dir, 'real-dir', 'a.ogg'), pattern(100)); fs.symlinkSync(path.join(dir, 'real-dir'), path.join(dir, 'dir-link')); + const scope = await createRecordingFileScope(dir, { enabled: true }); + await assert.rejects(scope.identify('link.ogg'), e => e.code === 'source_unavailable'); + await assert.rejects(scope.identify('dir-link/a.ogg'), e => e.code === 'source_unavailable'); + await assert.rejects(scope.identify('../etc/passwd'), e => e.code === 'source_unavailable'); + await assert.rejects(createRecordingFileScope(path.join(dir, 'dir-link'), { enabled: true }), e => e.code === 'source_unavailable'); + const f = await managerV1(); + const h = f.manager.attachOwnedRecording(identity, { kind: 'OwnedRecordingFile', lease: {} }, { mimeType: 'audio/ogg', direction: 'mixed' }); + assert.equal(h.reason, 'source_unavailable'); assert.equal(h.tier, 'denied'); + await scope.close(); await f.manager.shutdown(); +}); +test('T-LS-06 12 MiB OGG: tier single, streamed from the lease, owned memory never above one slice; T-CH-04 mixed, no channel', async () => { + const dir = root(); const data = pattern(12 * MiB); fs.writeFileSync(path.join(dir, 'call.ogg'), data); + let peak = 0; let manager; + const f = await managerV1(base => ({ putContent: async (s, u, src, sig) => { + const out = Buffer.alloc(src.size); + for (let o = 0; o < src.size; o += 262144) { await src.check(); const p = await src.read(o, Math.min(262144, src.size - o)); Buffer.from(p).copy(out, o); peak = Math.max(peak, manager.status().ownedBytes); } + base.contents.set ? null : null; f.contents.set(u.occurrence_id, out); + } })); + manager = f.manager; + const scope = await createRecordingFileScope(dir, { enabled: true }); + const id = await scope.identify('call.ogg'); assert.equal(id.size, 12 * MiB); + const h = f.manager.attachOwnedRecording(identity, scope.source('call.ogg', id), { mimeType: 'audio/ogg', direction: 'mixed' }); + assert.equal(h.tier, 'single'); assert.equal((await h.wait(30000)).status, 'validating'); + assert.ok(peak <= 262144, `owned peak ${peak}`); + assert.ok(Buffer.from(f.contents.values().next().value).equals(data)); + const request = f.requests[0]; + assert.equal(request.direction, 'mixed'); assert.equal(request.source_kind, 'OwnedRecordingFile'); assert.equal('channel' in request || 'role' in request, false); + assert.equal(request.recording_id, undefined); + await scope.close(); await f.manager.shutdown(); +}); +test('T-LS-07 70 MiB OGG: too_large with zero bytes read', async () => { + const dir = root(); const file = path.join(dir, 'big.ogg'); const fd = fs.openSync(file, 'w'); fs.ftruncateSync(fd, 70 * MiB); fs.closeSync(fd); + const f = await managerV1(); + const scope = await createRecordingFileScope(dir, { enabled: true }); + const h = f.manager.attachOwnedRecording(identity, scope.source('big.ogg', await scope.identify('big.ogg')), { mimeType: 'audio/ogg', direction: 'mixed' }); + assert.equal(h.tier, 'denied'); assert.equal(h.reason, 'too_large'); await f.manager.flush(); + assert.equal(f.ops().includes('initialize'), false); + await scope.close(); await f.manager.shutdown(); +}); +test('T-LS-08 90 MiB 24 kHz stereo WAV with wavTracks: segmented from lease ranges, stereo manifest, bytes identical', async () => { + const dir = root(); const frames = Math.floor(90 * MiB / 4); + const data = wavFile(path.join(dir, 'call.wav'), { rate: 24000, channels: 2, frames }); + const f = await managerV1(); + const scope = await createRecordingFileScope(dir, { enabled: true }); + const h = f.manager.attachOwnedRecording(identity, scope.source('call.wav', await scope.identify('call.wav')), { mimeType: 'audio/wav', direction: 'mixed', wavTracks: stereo }); + assert.equal(h.tier, 'wav_recording'); + assert.equal((await h.wait(30000)).status, 'validating'); assert.ok(h.recordingId); + const m = f.manifests.at(-1).body; + assert.equal(m.layout, 'stereo'); assert.deepEqual(m.tracks.map(t => [t.channels, t.role]), [[[0], 'user'], [[1], 'assistant']]); + assert.equal(m.capture_complete, true); assert.equal(m.duration_ms, Math.ceil(frames * 1000 / 24000)); assert.equal(validAttachmentManifest(m), true); + const joined = Buffer.concat(m.segments.map(s => f.contents.get(s.occurrence_id).subarray(44))); + assert.ok(joined.equals(data), 'decoded segment frames must equal the source frames'); + assert.ok(f.requests.every(r => r.declared_bytes <= 8 * 1024 * 1024 && r.recording_id === h.recordingId)); + await scope.close(); await f.manager.shutdown(); +}); +test('T-LS-09 bad data length, non-PCM and 24-bit WAV: single when within the cap, too_large above it', async () => { + const dir = root(); + wavFile(path.join(dir, 'bad.wav'), { frames: 1000, badLength: true }); + wavFile(path.join(dir, 'float.wav'), { frames: 1000, format: 3 }); + wavFile(path.join(dir, 'b24.wav'), { frames: 1000, bits: 24 }); + const f = await managerV1(); + const scope = await createRecordingFileScope(dir, { enabled: true }); + for (const name of ['bad.wav', 'float.wav', 'b24.wav']) { + const h = f.manager.attachOwnedRecording(identity, scope.source(name, await scope.identify(name)), { mimeType: 'audio/wav', direction: 'mixed', wavTracks: stereo }); + assert.equal((await h.wait(10000)).status, 'validating', name); assert.equal(h.tier, 'single', name); + } + const big = path.join(dir, 'big.wav'); fs.writeFileSync(big, Buffer.from(attachmentWavHeader(2, 24000, 100))); const fd = fs.openSync(big, 'r+'); fs.ftruncateSync(fd, 60 * MiB); fs.closeSync(fd); + const h = f.manager.attachOwnedRecording(identity, scope.source('big.wav', await scope.identify('big.wav')), { mimeType: 'audio/wav', direction: 'mixed', wavTracks: stereo }); + assert.equal((await h.wait(10000)).status, 'too_large'); assert.equal(h.tier, 'denied'); + await scope.close(); await f.manager.shutdown(); +}); +test('T-LS-01 file changed during transfer -> source_unavailable and abort; T-LS-04 the app file position is unchanged', async () => { + const dir = root(); const file = path.join(dir, 'grow.ogg'); fs.writeFileSync(file, pattern(MiB)); + const appFd = fs.openSync(file, 'r'); const probe = Buffer.alloc(10); fs.readSync(appFd, probe, 0, 10, null); + const f = await managerV1(base => ({ putContent: async (s, u, src) => { + for (let o = 0; o < src.size; o += 262144) { await src.check(); await src.read(o, 262144); if (o === 0) fs.appendFileSync(file, 'x'); } + await src.check(); + } })); + const scope = await createRecordingFileScope(dir, { enabled: true }); + const h = f.manager.attachOwnedRecording(identity, scope.source('grow.ogg', await scope.identify('grow.ogg')), { mimeType: 'audio/ogg', direction: 'mixed' }); + assert.equal((await h.wait(10000)).status, 'source_unavailable'); + assert.equal(f.ops().filter(op => op === 'abort').length, 1); + const next = Buffer.alloc(1); fs.readSync(appFd, next, 0, 1, null); assert.equal(next[0], pattern(11)[10]); + fs.closeSync(appFd); await scope.close(); await f.manager.shutdown(); +}); +test('T-LS-05 intermediate directory swapped to a symlink after identify: identity bind fails closed (documented residual race)', async () => { + const dir = root(); fs.mkdirSync(path.join(dir, 'session')); fs.writeFileSync(path.join(dir, 'session', 'r.ogg'), pattern(1000)); + fs.mkdirSync(path.join(dir, 'other')); fs.writeFileSync(path.join(dir, 'other', 'r.ogg'), pattern(1000, 9)); + const scope = await createRecordingFileScope(dir, { enabled: true }); + const id = await scope.identify('session/r.ogg'); const src = scope.source('session/r.ogg', id); + fs.renameSync(path.join(dir, 'session'), path.join(dir, 'session-real')); fs.symlinkSync(path.join(dir, 'other'), path.join(dir, 'session')); + const f = await managerV1(); + const h = f.manager.attachOwnedRecording(identity, src, { mimeType: 'audio/ogg', direction: 'mixed' }); + assert.equal((await h.wait(10000)).status, 'source_unavailable'); assert.equal(f.ops().includes('initialize'), false); + // Same inode reached through a symlinked directory is still refused (lstat of every component). + fs.unlinkSync(path.join(dir, 'session')); fs.symlinkSync(path.join(dir, 'session-real'), path.join(dir, 'session')); + const h2 = f.manager.attachOwnedRecording(identity, src, { mimeType: 'audio/ogg', direction: 'mixed' }); + assert.equal((await h2.wait(10000)).status, 'source_unavailable'); + await scope.close(); await f.manager.shutdown(); +}); +test('T-LS-10 finalization deadline with a blocked transport: incomplete within 1.25 s; lease and scope closed', async () => { + const dir = root(); fs.writeFileSync(path.join(dir, 'r.ogg'), pattern(600000)); + const f = await managerV1(() => ({ putContent: (_s, _u, _src, signal) => new Promise((_, reject) => signal.addEventListener('abort', () => reject(new AttachmentTransportError('incomplete')), { once: true })) })); + const scope = await createRecordingFileScope(dir, { enabled: true }); + const h = f.manager.attachOwnedRecording(identity, scope.source('r.ogg', await scope.identify('r.ogg')), { mimeType: 'audio/ogg', direction: 'mixed' }); + const start = Date.now(); const result = await h.wait(1000); const elapsed = Date.now() - start; + assert.equal(result.status, 'incomplete'); assert.ok(elapsed <= 1250, `wait took ${elapsed} ms`); + const closing = Date.now(); await scope.close(); assert.ok(Date.now() - closing < 500); + await f.manager.shutdown(500); assert.equal(f.manager.status().ownedBytes, 0); +}); diff --git a/typescript/packages/fi-core/test/attachments-fixtures.cjs b/typescript/packages/fi-core/test/attachments-fixtures.cjs index 2dfb96b4f..98666639b 100644 --- a/typescript/packages/fi-core/test/attachments-fixtures.cjs +++ b/typescript/packages/fi-core/test/attachments-fixtures.cjs @@ -8,7 +8,8 @@ const limits = { max_upload_bytes: 52428800, segment_bytes: 8388608, segment_dur transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }; const options = { mimeType: 'audio/wav', direction: 'mixed' }; const source = () => ({ kind: 'Bytes', data: new Uint8Array([82, 73, 70, 70, 1, 2, 3, 4]) }); -const expiry = () => new Date(Date.now() + 60000).toISOString(); +// One hour: well outside the renewal window, so call-count assertions see no session renewal. +const expiry = () => new Date(Date.now() + 3600000).toISOString(); function fake(overrides = {}) { const calls = []; let body; const transport = { diff --git a/typescript/packages/fi-core/test/attachments-v1.cjs b/typescript/packages/fi-core/test/attachments-v1.cjs new file mode 100644 index 000000000..5320e96dc --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-v1.cjs @@ -0,0 +1,93 @@ +// Synthetic doubles for the versioned wire surface (design section 9). No network, no real credentials. +const path = require('node:path'); +const { createRequire } = require('node:module'); +const packageJson = process.env.FI_CORE_PACKAGE_JSON || path.resolve(__dirname, '../package.json'); +const load = createRequire(packageJson); +const core = load('@traceai/fi-core/attachments'); +const nodeCore = load('@traceai/fi-core/attachments/node'); +const { ids, policy, limits, identity } = require('./attachments-fixtures.cjs'); + +async function readAll(source) { + if (source instanceof Uint8Array) return Buffer.from(source); + if (typeof Blob !== 'undefined' && source instanceof Blob) return Buffer.from(await source.arrayBuffer()); + const out = Buffer.alloc(source.size); + for (let offset = 0; offset < source.size; offset += 262144) { + await source.check(); + const part = await source.read(offset, Math.min(262144, source.size - offset)); + Buffer.from(part.buffer, part.byteOffset, part.byteLength).copy(out, offset); + } + await source.check(); + return out; +} +function clock(start = 1_700_000_000_000) { + let t = 0; + return { now: () => t, wall: () => start + t, advance(ms) { t += ms; }, set(ms) { t = ms; } }; +} +/** A complete in-memory AttachmentTransportV1. `overrides(base)` may replace any method. */ +function v1(overrides = {}, options = {}) { + const calls = []; let n = 0; + const id = () => `30000000-0000-4000-8000-${(++n).toString(16).padStart(12, '0')}`; + const cap = () => `synthetic-capability-${(++n).toString(36)}-abcdefgh`; + const wall = options.clock ? options.clock.wall : () => Date.now(); + const contents = new Map(); const manifests = []; const imports = []; const requests = []; const completed = new Map(); + const session = sid => ({ schema_version: 1, scope: { organization_id: ids[0], workspace_id: ids[1], project_id: ids[2] }, capture_session_id: sid, + policy: { ...policy, allowed_sources: ['Bytes', 'Blob', 'OwnedRecordingFile', 'ProviderFile', 'RemoteUrl'], record_audio: true, ...(options.policy || {}) }, + limits: { ...limits, ...(options.limits || {}) }, expires_at: new Date(wall() + (options.lifetimeMs ?? 3600000)).toISOString(), transfer_modes: ['authenticated_stream'] }); + const base = { + async bootstrap(body, key) { calls.push({ op: 'bootstrap', key, body }); return session(body.capture_session_id); }, + async renewSession(current, sequence) { calls.push({ op: 'renewSession', sequence }); return session(current.capture_session_id); }, + async initialize(_s, body, key) { + const request = JSON.parse(body); calls.push({ op: 'initialize', key, request, body }); requests.push(request); + return { upload_id: id(), attachment_id: request.attachment_id, occurrence_id: request.occurrence_id, state: 'pending_upload', reserved_bytes: request.declared_bytes, + expires_at: new Date(wall() + (options.uploadLifetimeMs ?? 600000)).toISOString(), transfer_mode: 'authenticated_stream', write_capability: cap() }; + }, + async renewUpload(_s, upload, body) { + calls.push({ op: 'renewUpload', body }); + return { ...upload, write_capability: cap(), expires_at: new Date(Date.parse(upload.expires_at) + 600000).toISOString() }; + }, + async putContent(_s, upload, source) { + const bytes = await readAll(source); + calls.push({ op: 'putContent', upload: upload.upload_id, occurrence: upload.occurrence_id, size: bytes.length, capability: upload.write_capability }); + contents.set(upload.occurrence_id, bytes); + }, + async complete(_s, upload, key) { calls.push({ op: 'complete', key, capability: upload.write_capability, occurrence: upload.occurrence_id }); completed.set(upload.occurrence_id, upload.write_capability); return 'validating'; }, + async abort(_s, upload, key) { calls.push({ op: 'abort', key, occurrence: upload.occurrence_id }); }, + async uploadStatus() { calls.push({ op: 'uploadStatus' }); return { state: 'validating', reason: 'binding_pending' }; }, + async putManifest(_s, capability, recordingId, body, key) { calls.push({ op: 'putManifest', key, capability, recordingId }); manifests.push({ capability, key, raw: body, body: JSON.parse(body) }); }, + async submitImport(_s, body, key) { const parsed = JSON.parse(body); calls.push({ op: 'submitImport', key, body: parsed }); imports.push({ raw: body, body: parsed, key }); return { jobId: id(), recordingId: parsed.recording.recording_id, state: 'importing' }; }, + async importStatus(_s, jobId) { calls.push({ op: 'importStatus', jobId }); return { state: 'importing', reason: 'binding_pending' }; }, + }; + const transport = { ...base, ...(typeof overrides === 'function' ? overrides(base) : overrides) }; + return { transport, base, calls, contents, manifests, imports, requests, completed, ops: () => calls.map(c => c.op) }; +} +async function managerV1(overrides, options = {}, managerOptions = {}) { + const double = v1(overrides, options); + const manager = new core.AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: double.transport, + recordAudio: true, ...(options.clock ? { clock: options.clock } : {}), ...managerOptions }); + if (options.ready !== false) { if (!await manager.ready()) throw Error('ready'); } + return { manager, ...double }; +} +function span(context = identity) { + const attrs = {}; let recording = true; let writes = 0; + return { attrs, get writes() { return writes; }, end() { recording = false; }, + spanContext: () => context, isRecording: () => recording, setAttribute(k, v) { writes++; attrs[k] = v; return this; } }; +} +/** Deterministic s16le PCM: `frames` frames of `channels` channels, offset by `seed`. */ +function pcm(frames, channels = 1, seed = 0) { + const out = new Uint8Array(frames * channels * 2); + for (let i = 0; i < out.length; i++) out[i] = (i * 31 + seed * 7 + (i >> 9)) & 0xff; + return out; +} +/** Write `seconds` of audio in `chunkMs` chunks; awaits `manager.flush()` every `flushEvery` seconds so uploads progress. */ +async function writeSeconds(manager, writer, rate, seconds, { channels = 1, track = 0, chunkMs = 20, flushEvery = 10, seed = 0 } = {}) { + const frames = rate * chunkMs / 1000; let accepted = 0; let rejected = 0; let offset = 0; + const chunks = Math.round(seconds * 1000 / chunkMs); + for (let i = 0; i < chunks; i++) { + const data = pcm(frames, channels, seed + offset); offset += frames; + if (writer.tryWrite(data, track)) accepted++; else rejected++; + if (flushEvery && (i + 1) % Math.round(flushEvery * 1000 / chunkMs) === 0) await manager.flush(10000); + } + return { accepted, rejected }; +} +const mono = rate => ({ format: { sampleRateHz: rate }, layout: 'mixed', tracks: [{ channels: [0], direction: 'mixed', role: 'mixed' }], operation: 'observe_consumed' }); +module.exports = { core, nodeCore, v1, managerV1, span, pcm, writeSeconds, readAll, clock, mono }; diff --git a/typescript/packages/fi-core/test/attachments-wire.test.cjs b/typescript/packages/fi-core/test/attachments-wire.test.cjs new file mode 100644 index 000000000..e23476bed --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-wire.test.cjs @@ -0,0 +1,32 @@ +/* T-TR-01: every F01 wire fixture for the shapes the SDK sends or parses, reproduced by the hand validators. */ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { core } = require('./attachments-v1.cjs'); +const directory = process.env.FI_ATTACHMENTS_SPEC_DIR; +const SDK = new Set(core.ATTACHMENT_WIRE_SHAPES); +test('T-TR-01 F01 wire fixtures agree with the core validators', { skip: !directory && 'FI_ATTACHMENTS_SPEC_DIR not set' }, () => { + const cases = JSON.parse(fs.readFileSync(path.join(directory, 'wire-fixtures.json'))).cases; + let checked = 0; const skipped = new Set(); + for (const fixture of cases) { + let schema = fixture.schema; let value = fixture.value; + if (schema === 'ImportRequest' && value && value.mode === 'completed_recording') schema = 'RecordingImportRequest'; + else if (schema === 'ImportRequest') { skipped.add('ImportRequest(single)'); continue; } + if (!SDK.has(schema)) { skipped.add(schema); continue; } + assert.equal(core.validAttachmentWire(schema, value), fixture.valid, fixture.id); checked++; + } + const imports = JSON.parse(fs.readFileSync(path.join(directory, 'recording-import-fixtures.json'))); + for (const fixture of imports.cases ?? []) { + if (!fixture.request || fixture.schema && fixture.schema !== 'RecordingImportRequest') continue; + assert.equal(core.validAttachmentWire('RecordingImportRequest', fixture.request), fixture.valid ?? true, fixture.id); checked++; + } + console.log(`# T-TR-01 checked=${checked} not-sdk-shapes=${[...skipped].sort().join(',')}`); + assert.ok(checked >= 80, `only ${checked} fixtures checked`); +}); +test('T-TR-08 lifecycle fixtures: complete is validating, never ready, in both orders', { skip: !directory && 'FI_ATTACHMENTS_SPEC_DIR not set' }, () => { + const lifecycle = JSON.parse(fs.readFileSync(path.join(directory, 'lifecycle-fixtures.json'))); + const text = JSON.stringify(lifecycle); + assert.match(text, /late-root-releases-ready-bytes/); assert.match(text, /root-first-still-needs-validation/); + assert.equal(lifecycle.initial?.generation ?? 0, 0, 'CQ-5: new occurrences start at deletion generation 0'); +}); diff --git a/typescript/packages/fi-core/test/attachments.test.cjs b/typescript/packages/fi-core/test/attachments.test.cjs index 20467abc6..8d0a1cf23 100644 --- a/typescript/packages/fi-core/test/attachments.test.cjs +++ b/typescript/packages/fi-core/test/attachments.test.cjs @@ -177,7 +177,8 @@ test('HTTP v1 loopback upload: stable identity, bounded response, no source URLs test('transport blocks redirects and invalid origins', async () => { for (const endpoint of ['http://example.com', 'https://user:secret@example.com', 'https://example.com/path', 'https://example.com/?token=secret']) assert.throws(() => createAttachmentHttpTransport({ endpoint, headers: {} }), /endpoint_unconfigured/); const transport = createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: {}, client: async () => ({ status: 302, body: Buffer.from('synthetic-secret') }) }); - await assert.rejects(transport.bootstrap({}, new AbortController().signal), error => error.message === 'failed' && !error.retryable); + // Redirects are never followed; a 3xx is an unmapped status: incomplete, not retried (design 0.5). + await assert.rejects(transport.bootstrap({}, new AbortController().signal), error => error.message === 'incomplete' && !error.retryable); }); test('CJS and ESM leaf export the public manager API', async () => { @@ -197,7 +198,8 @@ test('late completion uses the stamped writer carrier and never mutates an ended const f = await manager(); const attributes = {}; let recording = true; let writes = 0; const span = { spanContext: () => identity, isRecording: () => recording, setAttribute(k, v) { attributes[k] = v; writes++; } }; const correlation = f.value.correlation(span); - assert.match(attributes['fi.attachment.capture_session_id'], /^[0-9a-f-]{36}$/); + // Design 0.8: a correlation snapshot alone stamps nothing; only an admitted capture on a live span does. + assert.equal(writes, 0); recording = false; const before = writes; const h = f.value.attach(correlation, source(), { ...options, conversationId: 'conversation-1', turnId: 'turn-1', toolCallId: 'call-1' }); assert.equal((await h.completion).status, 'validating'); assert.equal(writes, before); await f.value.shutdown(); @@ -216,14 +218,14 @@ test('untrusted metadata keys and malformed IDs are not retained or projected', test('upload response binds exact IDs, reservation and capability', async () => { for (const field of ['attachment_id', 'occurrence_id', 'reserved_bytes', 'write_capability']) { const t = fake(); const f = await manager({}, { initialize: async (s, r) => ({ ...await t.transport.initialize(s, r), [field]: field === 'reserved_bytes' ? 999 : 'wrong' }) }); - const h = f.value.attach(identity, source(), options); assert.equal((await h.completion).status, 'failed'); + const h = f.value.attach(identity, source(), options); assert.equal((await h.completion).status, 'unsupported_version'); assert.equal(f.calls.includes('upload'), false); await f.value.shutdown(); } }); test('successful response reader rejects oversized control data', async () => { const transport = createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: {}, client: async () => ({ status: 200, body: new Uint8Array(65537) }) }); - await assert.rejects(transport.bootstrap({}, new AbortController().signal), error => error.message === 'failed' && !error.retryable); + await assert.rejects(transport.bootstrap({}, new AbortController().signal), error => error.message === 'unsupported_version' && !error.retryable); }); test('provider trace options cannot weaken attachment-level privacy restrictions', async () => { From 031178f6054a86384c874edb871dad71c8e6833b Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 22:14:55 -0800 Subject: [PATCH 5/9] fix(core): attachment design qualification fixes Type FinalizationRegistry locally so consumers compiling fi-core source with an ES2020 lib still build. Hold a fallback-order segment's complete until a later segment is sealed or the recording finishes, so the last capability can PUT the final manifest (BR-7). Add T-ST-16 fallback, T-CS-05 forced-GC, T-TR-04 proxy-canary and T-PF-03 preflight replay coverage. Co-Authored-By: Claude Opus 5.5 --- .../fi-core/src/attachments/manager.ts | 10 ++-- .../fi-core/src/attachments/recording.ts | 31 ++++++++++-- .../fi-core/test/attachments-contract.cjs | 15 +++++- .../fi-core/test/attachments-design.test.cjs | 48 ++++++++++++++++++- .../fi-core/test/attachments-files.test.cjs | 2 +- .../fi-core/test/attachments-tls.test.cjs | 9 +++- 6 files changed, 103 insertions(+), 12 deletions(-) diff --git a/typescript/packages/fi-core/src/attachments/manager.ts b/typescript/packages/fi-core/src/attachments/manager.ts index eb24dc9ad..8b7c78a68 100644 --- a/typescript/packages/fi-core/src/attachments/manager.ts +++ b/typescript/packages/fi-core/src/attachments/manager.ts @@ -85,6 +85,9 @@ class TokenBucket implements AttachmentPacer { if (this.tokens < 0) await pause(-this.tokens / rate * 1000, signal); } } +/** ES2021 FinalizationRegistry, typed locally so consumers compiling this source with an older `lib` still build. */ +interface Finalizer { register(target: object, held: () => void, token?: object): void; unregister(token: object): boolean } +type FinalizerConstructor = new (cleanup: (held: () => void) => void) => Finalizer; class Lane { readonly jobs: CoreJob[] = []; busy = false; constructor(readonly id: string) {} } interface Reservation { bytes: number; expiresAt: number } interface StreamExtra { facts: AdmissionFacts; fields: AttachmentFields; correlation: AttachmentCorrelation; purpose?: AttachmentOptions["purpose"] } @@ -128,7 +131,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost private renewing = false; private nextRenewalAt = 0; private renewalFailures = 0; - private registry?: FinalizationRegistry<() => void>; + private registry?: Finalizer; private readonly pacer: TokenBucket; private readonly maxBytes: number; private readonly maxDescriptors: number; @@ -285,8 +288,9 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost if (options.purpose !== undefined && !PURPOSES.includes(options.purpose)) return false; return this.validFields(options); } - private finalizer(): FinalizationRegistry<() => void> | undefined { - if (!this.registry && typeof FinalizationRegistry !== "undefined") this.registry = new FinalizationRegistry(release => { try { release(); } catch { /* bounded */ } }); + private finalizer(): Finalizer | undefined { + const Registry = (globalThis as unknown as { FinalizationRegistry?: FinalizerConstructor }).FinalizationRegistry; + if (!this.registry && typeof Registry === "function") this.registry = new Registry((release: () => void) => { try { release(); } catch { /* bounded */ } }); return this.registry; } /** Lazy sweep at every entry point: no timers are armed for tickets or streams (design 1.5, 6.3). */ diff --git a/typescript/packages/fi-core/src/attachments/recording.ts b/typescript/packages/fi-core/src/attachments/recording.ts index 36500ed1a..e46beeffd 100644 --- a/typescript/packages/fi-core/src/attachments/recording.ts +++ b/typescript/packages/fi-core/src/attachments/recording.ts @@ -234,6 +234,8 @@ export class AttachmentRecording implements AttachmentRecordingWriter { private closed = false; private finalEnd?: number; private stamped = false; + private finalPublished = false; + private sealWaiters: (() => void)[] = []; constructor(init: RecordingInit) { this.init = init; this.host = init.host; this.options = init.options; this.identity = init.identity; @@ -276,8 +278,9 @@ export class AttachmentRecording implements AttachmentRecordingWriter { } private terminal(reason: AttachmentAdmission | AttachmentCompletion["status"]): void { if (this.mode === "terminal") return; - this.mode = "terminal"; this.terminalReason = reason; + this.mode = "terminal"; this.terminalReason = reason; this.wakeSeal(); } + private wakeSeal(): void { const waiters = this.sealWaiters; this.sealWaiters = []; for (const wake of waiters) wake(); } private closeOnce(): void { if (!this.closed) { this.closed = true; try { this.init.close(); } catch { /* bounded */ } this.host.recordingClosed(this.recordingId); } } // --- block timeline ----------------------------------------------------------------------------- @@ -318,6 +321,7 @@ export class AttachmentRecording implements AttachmentRecordingWriter { this.totalBytes += dataBytes + ATTACHMENT_WAV_HEADER_BYTES; track.pieces.push({ kind: "segment", track: track.ordinal, start: track.runStart, end: track.runStart + track.runFrames, source: attachmentChunkedSource(header, used), bytes: dataBytes, state: "sealed", occurrenceId: uuid(), attachmentId: uuid() }); + this.wakeSeal(); } track.runFrames = 0; track.runStart = track.pos; this.release(); @@ -516,6 +520,8 @@ export class AttachmentRecording implements AttachmentRecordingWriter { } private async uploadSegment(piece: Piece, signal: AbortSignal): Promise { if (piece.state !== "queued") return; + // Both manifest orders failed: the recording cannot publish, so stop spending uploads (design 2.8). + if (this.manifestFallback && this.manifestDisabled) { this.segmentSettled(piece, "incomplete"); return; } const reason = this.segmentDenied(piece); if (reason) { this.segmentSettled(piece, reason); return; } piece.state = "uploading"; @@ -663,12 +669,28 @@ export class AttachmentRecording implements AttachmentRecordingWriter { } private async fallbackManifest(piece: Piece, outcome: UploadOutcome, signal: AbortSignal): Promise { piece.outcome = outcome; + // Hold this segment's complete until a later segment is sealed or the recording finishes, so the LAST + // segment's capability can still PUT the final manifest (no completed capability can do it afterwards). + const later = () => this.finalEnd !== undefined || this.mode === "terminal" || signal.aborted || + this.tracks.some(track => track.pieces.some(p => p !== piece && p.kind === "segment" && p.state !== "done" && p.state !== "failed")); + if (!later()) { + await new Promise(resolve => { + const done = () => { clearTimeout(timer); signal.removeEventListener("abort", done); resolve(); }; + const timer = setTimeout(done, MAX_SEGMENT_MS + 5000); (timer as unknown as { unref?: () => void }).unref?.(); + this.sealWaiters.push(done); signal.addEventListener("abort", done, { once: true }); + }); + } // Prospectively include this segment; its own capability authorizes the PUT. const savedState = piece.state; piece.state = "done"; piece.index = this.completed; const end = this.settledEnd(); + let pending = 0; + for (const track of this.tracks) for (const p of track.pieces) if (p !== piece && p.kind === "segment" && p.state !== "done" && p.state !== "failed") pending++; piece.state = savedState; + // The last segment of a finishing recording carries the FINAL manifest (no completed capability can PUT it later). + const final = this.finalEnd !== undefined && pending === 0 && end === this.finalEnd; let ok = false; - try { ok = await this.putManifest(end, false, signal, piece, outcome); } catch { ok = false; } + try { ok = await this.putManifest(end, final, signal, piece, outcome); } catch { ok = false; } + if (ok && final) this.finalPublished = true; piece.index = undefined; if (!ok) { this.manifestDisabled = true; throw new Error("manifest"); } } @@ -718,6 +740,7 @@ export class AttachmentRecording implements AttachmentRecordingWriter { const end = Math.max(0, ...this.tracks.map(track => track.pos)); for (const track of this.tracks) if (track.pos < end) { this.appendGap(track, track.denied ? "policy_denied" : "source_unavailable", end - track.pos); track.pos = end; } this.finalEnd = end; + this.wakeSeal(); } private idle(deadline: number): Promise { if (this.outstanding === 0) return Promise.resolve(); @@ -756,7 +779,7 @@ export class AttachmentRecording implements AttachmentRecordingWriter { piece.state = "failed"; piece.failure = "incomplete"; piece.source = undefined; } } - if (this.latestCapability() && !this.manifestDisabled) { + if (this.latestCapability() && !this.manifestDisabled && !this.finalPublished) { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), Math.max(1, deadline - this.host.clock.now())); try { await this.raceDeadline(this.putManifest(this.finalEnd ?? 0, true, controller.signal), deadline); } @@ -812,7 +835,7 @@ export class AttachmentRecording implements AttachmentRecordingWriter { if (signal.aborted) { this.discardAll(); this.terminal("incomplete"); this.closeOnce(); return "incomplete"; } let failed: AttachmentCompletion["status"] | undefined; for (const track of this.tracks) for (const piece of track.pieces) if (piece.state === "failed") failed = failed ?? piece.failure; - if (this.latestCapability()) await this.putManifest(this.finalEnd ?? 0, true, signal); + if (this.latestCapability() && !this.finalPublished && !this.manifestDisabled) await this.putManifest(this.finalEnd ?? 0, true, signal); this.terminal(failed ?? "incomplete"); this.closeOnce(); return this.revision === 0 ? (failed ?? "incomplete") : failed ?? (this.publishedComplete ? "validating" : "incomplete"); } diff --git a/typescript/packages/fi-core/test/attachments-contract.cjs b/typescript/packages/fi-core/test/attachments-contract.cjs index 3b34b23c9..9fbee960d 100644 --- a/typescript/packages/fi-core/test/attachments-contract.cjs +++ b/typescript/packages/fi-core/test/attachments-contract.cjs @@ -11,7 +11,7 @@ const privacy = JSON.parse(fs.readFileSync(path.join(directory, 'privacy-fixture const identities = JSON.parse(fs.readFileSync(path.join(directory, 'identity-fixtures.json'))).cases; const keepalive = setInterval(() => {}, 1000); (async () => { - const results = []; + const results = []; const preflightResults = []; for (const fixture of privacy) { const c = fixture.context; const traceConfig = Object.fromEntries(['hide_inputs', 'hide_outputs', 'hide_input_messages', 'hide_output_messages', 'hide_input_images', 'hide_input_text', 'hide_output_text', 'hide_error_details', 'pii_redaction'].map(key => [key.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()), c[key]])); @@ -50,6 +50,18 @@ const keepalive = setInterval(() => {}, 1000); assert.notEqual(result.admission, 'queued', fixture.id); assert.equal(requests.includes(result.occurrenceId), false, fixture.id); } else assert.equal(result.admission, 'queued', fixture.id); + // T-PF-03: the same fixture through preflight. A denial never reads the source; a ticket whose denial + // depends on the source itself (availability, budget) is refused at commit without being uploaded. + let preflightRead = false; + const ticket = manager.preflight(context, { mimeType, direction: c.direction, maxBytes: 8, sourceKind: actualSource.kind, + operation: c.operation === 'create_recording' ? 'create_recording' : c.operation === 'import_existing' ? 'import_existing' : 'observe_consumed', capture: options.capture }); + let preflightOutcome = ticket.reason; + if (ticket.admitted && (fixture.expected.decision === 'deny' || unsupported)) { + preflightOutcome = ticket.attach({ kind: actualSource.kind, get data() { preflightRead = true; return actualSource.data; } }).admission; + assert.notEqual(preflightOutcome, 'queued', fixture.id); + } else if (!ticket.admitted) assert.equal(preflightRead, false, fixture.id); + preflightResults.push({ id: fixture.id, expected: fixture.expected.reason, actual: preflightOutcome }); + ticket.release(); results.push({ id: fixture.id, expected: fixture.expected.reason, actual: result.admission, qualification: fixture.expected.decision === 'allow' && unsupported ? 'unsupported_fail_closed' : 'safety_pass' }); await manager.shutdown(10); @@ -61,6 +73,7 @@ const keepalive = setInterval(() => {}, 1000); } console.log(JSON.stringify({ privacyFixtures: results.length, identityFixtures: identities.length, exactReasons: results.filter(result => result.expected === result.actual).length, + preflightFixtures: preflightResults.length, preflightExactReasons: preflightResults.filter(r => r.expected === r.actual).length, unsupportedPositive: results.filter(result => result.qualification === 'unsupported_fail_closed'), conservativeReasonDifferences: results.filter(result => result.expected !== result.actual && result.qualification !== 'unsupported_fail_closed') }, null, 2)); })().catch(error => { console.error(error); process.exitCode = 1; }).finally(() => clearInterval(keepalive)); diff --git a/typescript/packages/fi-core/test/attachments-design.test.cjs b/typescript/packages/fi-core/test/attachments-design.test.cjs index 5b3b8a1f2..9ccf560fe 100644 --- a/typescript/packages/fi-core/test/attachments-design.test.cjs +++ b/typescript/packages/fi-core/test/attachments-design.test.cjs @@ -587,9 +587,13 @@ test('T-CH-01 UploadRequest never carries role or channel; T-CH-02 the live refe await f.manager.shutdown(); }); test('T-CH-03 span ended before attach: no projection, the upload still proceeds; T-CH-05 carrier stamp only on admitted targets', async () => { - const f = await managerV1(); const ended = span(); ended.end(); + const f = await managerV1(); const ended = span(); + f.manager.correlation(ended); // snapshot taken while the span was live (adapters bind at span start) + ended.end(); const h = f.manager.attach(ended, { kind: 'Bytes', data: new Uint8Array([1]) }, options); assert.equal((await h.completion).status, 'validating'); assert.equal(ended.writes, 0); + const never = span(); never.end(); // never observed live: not recording at admission (design 1.4 step 10) + assert.equal(f.manager.attach(never, { kind: 'Bytes', data: new Uint8Array([1]) }, options).admission, 'unsampled'); const bystander = span(); f.manager.correlation(bystander); assert.equal(bystander.writes, 0, 'non-target spans are never stamped'); const denied = span(); f.manager.attach(denied, { kind: 'Bytes', data: new Uint8Array([1]) }, { ...options, capture: { captureContent: false } }); assert.equal(denied.writes, 0, 'a denied capture stamps nothing'); @@ -727,3 +731,45 @@ test('legacy four-method transports keep working through the adapter (stacked la assert.equal(r.state, 'unpublished', 'legacy transports cannot PUT manifests: fail closed'); await m.shutdown(); }); +test('T-ST-16 BR-7 fallback: capability rejected after complete -> init, content, manifest, complete for the remaining segments', async () => { + const f = await managerV1(base => ({ putManifest: async (s, capability, rid, body, key, signal) => { + if ([...f.completed.values()].includes(capability)) throw new AttachmentTransportError('policy_denied', false, 0, 403); + return base.putManifest(s, capability, rid, body, key, signal); + } }), { limits: { segment_duration_ms: 1000 } }); + const w = f.manager.startRecording(identity, mono(8000)); + await writeSeconds(f.manager, w, 8000, 8, { flushEvery: 1 }); + const result = await w.finish(10000); + assert.ok(f.manifests.length >= 2, `manifests ${f.manifests.length}`); + for (const m of f.manifests) { + const top = m.body.segments.at(-1); + assert.equal(f.completed.has(top.occurrence_id), true); + const order = f.calls.map(c => c.op + ':' + (c.capability || '')); + assert.ok(order.indexOf('putManifest:' + m.capability) < order.indexOf('complete:' + m.capability), 'manifest must precede that segment complete'); + } + assert.equal(f.manifests.at(-1).body.capture_complete, true); assert.equal(result.state, 'published'); + for (let i = 1; i < f.manifests.length; i++) assert.ok(f.manifests[i].body.segments.length >= f.manifests[i - 1].body.segments.length); + await f.manager.shutdown(); + // Fallback also rejected: nothing published. + const g = await managerV1(() => ({ putManifest: async () => { throw new AttachmentTransportError('policy_denied', false, 0, 403); } }), { limits: { segment_duration_ms: 1000 } }); + const w2 = g.manager.startRecording(identity, mono(8000)); + await writeSeconds(g.manager, w2, 8000, 3, { flushEvery: 1 }); + assert.equal((await w2.finish(10000)).state, 'unpublished'); + await g.manager.shutdown(); +}); +test('T-CS-05 garbage-collected abandoned stream and ticket release their budget (best effort, forced GC)', () => { + const { spawnSync } = require('node:child_process'); + const script = ` + const { managerV1 } = require(${JSON.stringify(require.resolve('./attachments-v1.cjs'))}); + const identity = { traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }; + (async () => { + const f = await managerV1(); + (() => { const s = f.manager.startAttachmentStream(identity, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 4096 }); s.tryWrite(new Uint8Array(10)); + f.manager.preflight(identity, { mimeType: 'image/png', direction: 'input', maxBytes: 2048 }); })(); + for (let i = 0; i < 50 && f.manager.status().ownedBytes; i++) { global.gc(); await new Promise(r => setTimeout(r, 20)); } + const st = f.manager.status(); console.log(JSON.stringify({ owned: st.ownedBytes, streams: st.openStreams, tickets: st.openTickets })); + await f.manager.shutdown(100); + })();`; + const r = spawnSync(process.execPath, ['--expose-gc', '-e', script], { env: process.env, encoding: 'utf8', timeout: 20000 }); + assert.equal(r.status, 0, r.stderr); + assert.deepEqual(JSON.parse(r.stdout.trim().split('\n').pop()), { owned: 0, streams: 0, tickets: 0 }); +}); diff --git a/typescript/packages/fi-core/test/attachments-files.test.cjs b/typescript/packages/fi-core/test/attachments-files.test.cjs index 94ff0bd3b..2b7f0c4ee 100644 --- a/typescript/packages/fi-core/test/attachments-files.test.cjs +++ b/typescript/packages/fi-core/test/attachments-files.test.cjs @@ -42,7 +42,7 @@ test('T-LS-06 12 MiB OGG: tier single, streamed from the lease, owned memory nev const f = await managerV1(base => ({ putContent: async (s, u, src, sig) => { const out = Buffer.alloc(src.size); for (let o = 0; o < src.size; o += 262144) { await src.check(); const p = await src.read(o, Math.min(262144, src.size - o)); Buffer.from(p).copy(out, o); peak = Math.max(peak, manager.status().ownedBytes); } - base.contents.set ? null : null; f.contents.set(u.occurrence_id, out); + f.contents.set(u.occurrence_id, out); } })); manager = f.manager; const scope = await createRecordingFileScope(dir, { enabled: true }); diff --git a/typescript/packages/fi-core/test/attachments-tls.test.cjs b/typescript/packages/fi-core/test/attachments-tls.test.cjs index 0d6ea7b5b..48f1733d8 100644 --- a/typescript/packages/fi-core/test/attachments-tls.test.cjs +++ b/typescript/packages/fi-core/test/attachments-tls.test.cjs @@ -26,6 +26,10 @@ test('Node TLS transfer uses upload capability, bounds readers and does not keep } else { res.statusCode = 202; res.end(JSON.stringify({ upload_id: ids[0], state: 'validating', reason: 'binding_pending' })); } }); server.listen(0, '127.0.0.1'); await once(server, 'listening'); + // T-TR-04: ambient proxy settings are a canary; the attachment client must never connect to them. + let proxied = 0; const canary = require('node:net').createServer(socket => { proxied++; socket.destroy(); }); + canary.listen(0, '127.0.0.1'); await once(canary, 'listening'); + const proxy = `http://127.0.0.1:${canary.address().port}`; const modulePath = path.resolve(path.dirname(process.env.FI_CORE_PACKAGE_JSON || path.resolve(__dirname, '../package.json')), 'dist/src/attachments/node.js'); async function child(stall) { const script = ` @@ -42,7 +46,7 @@ test('Node TLS transfer uses upload capability, bounds readers and does not keep if (live) clearTimeout(live); })().catch(() => process.exitCode = 1); `; - const processChild = spawn(process.execPath, ['-e', script], { env: { ...process.env, NODE_EXTRA_CA_CERTS: cert }, stdio: ['ignore', 'pipe', 'pipe'] }); + const processChild = spawn(process.execPath, ['-e', script], { env: { ...process.env, NODE_EXTRA_CA_CERTS: cert, HTTPS_PROXY: proxy, https_proxy: proxy, HTTP_PROXY: proxy, NODE_USE_ENV_PROXY: '1' }, stdio: ['ignore', 'pipe', 'pipe'] }); let stdout = ''; let stderr = ''; processChild.stdout.on('data', c => { stdout += c; }); processChild.stderr.on('data', c => { stderr += c; }); const deadline = setTimeout(() => processChild.kill('SIGKILL'), 5000); const [code, signal] = await once(processChild, 'exit'); clearTimeout(deadline); @@ -56,5 +60,6 @@ test('Node TLS transfer uses upload capability, bounds readers and does not keep assert.equal(headers[3].authorization, 'Bearer synthetic-write-capability'); stalled = true; assert.match(await child(true), /TRANSFER_STARTED/); assert.equal(bodies.length, 2); - } finally { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); fs.rmSync(dir, { recursive: true, force: true }); } + assert.equal(proxied, 0, 'environment proxy was used'); + } finally { canary.close(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); fs.rmSync(dir, { recursive: true, force: true }); } }); From f054a68eef290bbb1eea50867eea133b4bde5c9c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 10 Nov 2026 23:04:20 -0800 Subject: [PATCH 6/9] fix(core): renew attachment sessions from recording writes, flush, status and worker wake A recording-only workload (tryWrite + flush, as Pipecat-style adapters drive it) never reached maybeRenew(), which ran only from the five admission entry points. A recording that outlived the server session lost its tail as `incomplete` gaps. Per design 3.1, renewal is now also checked from: - pump() (worker wake), - flush() and status() (enter() instead of sweep()), - AttachmentRecording.writeFrames, through a deduped host touch() that defers maybeRenew to a microtask so tryWrite does no I/O on the caller's stack. Regression tests: T-RN-09 (120 s session, 150 s recording, tryWrite + flush only: renewal, no rejected writes, 15 segments, published, no gaps) and T-RN-10 (each trigger alone renews: recording write, flush, status, worker wake). Co-Authored-By: Claude Opus 5.5 --- .../fi-core/src/attachments/internal.ts | 2 + .../fi-core/src/attachments/manager.ts | 13 ++++- .../fi-core/src/attachments/recording.ts | 1 + .../fi-core/test/attachments-design.test.cjs | 57 +++++++++++++++++++ 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/typescript/packages/fi-core/src/attachments/internal.ts b/typescript/packages/fi-core/src/attachments/internal.ts index 5a152da51..76d703dd3 100644 --- a/typescript/packages/fi-core/src/attachments/internal.ts +++ b/typescript/packages/fi-core/src/attachments/internal.ts @@ -60,6 +60,8 @@ export interface AttachmentHost { renewUpload(outcome: UploadOutcome, signal: AbortSignal): Promise; putManifest(capability: string, recordingId: string, body: string, key: string, signal: AbortSignal, replay: boolean): Promise; recordingClosed(id: string): void; + /** Schedules a deferred session-renewal check (no I/O on the caller's stack); recording writes call it (design 3.1). */ + touch(): void; countDrop(): void; notify(): void; } diff --git a/typescript/packages/fi-core/src/attachments/manager.ts b/typescript/packages/fi-core/src/attachments/manager.ts index 8b7c78a68..69b3289cb 100644 --- a/typescript/packages/fi-core/src/attachments/manager.ts +++ b/typescript/packages/fi-core/src/attachments/manager.ts @@ -131,6 +131,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost private renewing = false; private nextRenewalAt = 0; private renewalFailures = 0; + private renewCheckPending = false; private registry?: Finalizer; private readonly pacer: TokenBucket; private readonly maxBytes: number; @@ -327,6 +328,12 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost })(); } private enter(): void { this.sweep(); this.maybeRenew(); } + /** @internal Deduped deferred renewal check for hot paths such as recording `tryWrite` (design 3.1). */ + touch(): void { + if (this.renewCheckPending || !this.enabled) return; + this.renewCheckPending = true; + void Promise.resolve().then(() => { this.renewCheckPending = false; this.maybeRenew(); }); + } private preflightFacts(options: AttachmentPreflightOptions, importSource = false): AdmissionFacts { return { mimeType: options.mimeType, direction: options.direction, purpose: options.purpose, capture: options.capture ? { ...options.capture } : undefined, operation: options.operation ?? "observe_consumed", @@ -938,6 +945,8 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost job.settled = true; this.releaseBytes(job.bytes); job.bytes = 0; } private pump(): void { + // Worker wake is a renewal trigger (design 3.1): recording-only workloads never pass through preflight. + this.maybeRenew(); while (this.active.size < this.workers) { const lane = this.nextLane(); if (!lane) break; @@ -1101,7 +1110,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost } async flush(timeoutMs = this.shutdownTimeout): Promise { if (!this.enabled) return Object.freeze({ unfinished: 0, dropped: 0 }); - this.sweep(); + this.enter(); this.cancelDenied(); if (this.queued || this.active.size) { await new Promise(resolve => { @@ -1141,7 +1150,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost } /** Aggregate diagnostics only; no per-source metadata, names, hashes, capabilities or exception text. */ status(): AttachmentStatus { - if (this.enabled) this.sweep(); + if (this.enabled) this.enter(); return Object.freeze({ state: this.runtimeState(), queued: this.queued, active: this.active.size, openTickets: this.reservations.size, openStreams: this.streams.size, openRecordings: this.recordings.size, ownedBytes: this.bytes, dropped: this.dropped }); } diff --git a/typescript/packages/fi-core/src/attachments/recording.ts b/typescript/packages/fi-core/src/attachments/recording.ts index e46beeffd..432def5ca 100644 --- a/typescript/packages/fi-core/src/attachments/recording.ts +++ b/typescript/packages/fi-core/src/attachments/recording.ts @@ -387,6 +387,7 @@ export class AttachmentRecording implements AttachmentRecordingWriter { } private writeFrames(t: Track, data: Uint8Array): boolean { if (this.mode === "terminal" || this.mode === "finishing") return false; + this.host.touch(); // deferred renewal check; tryWrite itself does no I/O (design 3.1) const whole = data.byteLength; const frames = whole / t.frameBytes; if (this.mode === "pending") { const state = this.host.runtimeState(); diff --git a/typescript/packages/fi-core/test/attachments-design.test.cjs b/typescript/packages/fi-core/test/attachments-design.test.cjs index 9ccf560fe..3f28820be 100644 --- a/typescript/packages/fi-core/test/attachments-design.test.cjs +++ b/typescript/packages/fi-core/test/attachments-design.test.cjs @@ -478,6 +478,63 @@ test('T-RN-08 the write capability never appears in status, handles, span attrib for (const cap of caps) assert.equal(haystack.includes(cap), false); await f.manager.shutdown(); }); +test('T-RN-09 recording-only workload (tryWrite + flush, never preflight) renews before expiry and keeps its tail', async () => { + // Pipecat shape: one long recording, 20 ms writes, a flush every second, no other entry point (design 3.1). + const c = clock(Date.now()); + const f = await managerV1({}, { clock: c, lifetimeMs: 120000, limits: { segment_duration_ms: 10000 } }); + const w = f.manager.startRecording(identity, mono(8000)); + assert.equal(w.reason, 'queued'); + let rejectedAfter = 0; const stateAt = {}; + for (let ms = 0; ms < 150000; ms += 20) { + const ok = w.tryWrite(pcm(160, 1, ms)); + if (ms >= 120000 && !ok) rejectedAfter++; + c.advance(20); + if (ms % 1000 === 0) { await f.manager.flush(5000); for (let i = 0; i < 5; i++) await tick(); } + if (ms === 100000 || ms === 116000 || ms === 130000) stateAt[ms] = f.manager.status().state; + } + const result = await w.finish(10000); + const manifest = f.manifests.at(-1).body; + assert.ok(f.calls.filter(x => x.op === 'renewSession').length >= 1); + assert.deepEqual(stateAt, { 100000: 'ready', 116000: 'ready', 130000: 'ready' }); + assert.equal(rejectedAfter, 0); + assert.equal(result.state, 'published'); assert.equal(result.captureComplete, true); + assert.equal(manifest.segments.length, 15); assert.deepEqual(manifest.gaps, []); + await f.manager.shutdown(1000); +}); +test('T-RN-10 each non-admission entry point is a renewal trigger on its own: recording write, flush, status, worker wake', async () => { + const renewals = f => f.calls.filter(x => x.op === 'renewSession').length; + const settle = async f => { for (let i = 0; i < 20 && !renewals(f); i++) await tick(); }; + const fresh = async (overrides = {}) => { const c = clock(Date.now()); return { c, f: await managerV1(overrides, { clock: c, lifetimeMs: 120000, limits: { segment_duration_ms: 10000 } }) }; }; + // Recording write: a sub-segment write seals nothing and enqueues nothing; only the deferred host check can renew. + { const { c, f } = await fresh(); + const w = f.manager.startRecording(identity, mono(8000)); + assert.equal(w.tryWrite(pcm(160)), true); + for (let i = 0; i < 5; i++) await tick(); + assert.equal(renewals(f), 0); + c.advance(70000); + const before = f.calls.length; + assert.equal(w.tryWrite(pcm(160, 1, 1)), true); + assert.equal(f.calls.length, before, 'tryWrite performs no transport call on the caller stack'); + await settle(f); assert.equal(renewals(f), 1, 'recording write'); + w.abort(); await f.manager.shutdown(100); } + // flush() and status() with no other work. + for (const [name, call] of [['flush', m => m.flush(100)], ['status', m => m.status()]]) { + const { c, f } = await fresh(); + c.advance(70000); assert.equal(renewals(f), 0); + await call(f.manager); await settle(f); + assert.equal(renewals(f), 1, name); + await f.manager.shutdown(100); + } + // Worker wake: an upload admitted before the window finishes after it; its job end wakes the pump. + { let unblock; const block = new Promise(resolve => { unblock = resolve; }); + const { c, f } = await fresh(base => ({ initialize: async (...a) => { await block; return base.initialize(...a); } })); + const h = f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array(4) }, options); + for (let i = 0; i < 5; i++) await tick(); + c.advance(70000); assert.equal(renewals(f), 0); + unblock(); assert.equal((await h.completion).status, 'validating'); + await settle(f); assert.equal(renewals(f), 1, 'worker wake'); + await f.manager.shutdown(100); } +}); // ---- 10.5 Completed import ------------------------------------------------------------------------------- const providerResolved = { source: { kind: 'ProviderFile', connectionId: '80000000-0000-4000-8000-000000000001', registeredOutputId: '80000000-0000-4000-8000-000000000002' }, From f0a56b37964ad85fce336d10ef44fb5ebed95a45 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:29:03 -0800 Subject: [PATCH 7/9] fix(core): make stacked-lane attachment API breakage loud Changes that used to stop capture silently now fail loudly: - A legacy four-method transport (including a wrapper that drops ATTACHMENT_TRANSPORT_V1) emits one DeprecationWarning FI_ATTACHMENTS_LEGACY_TRANSPORT naming what it cannot do and the fix. startRecording, WAV-tier attachOwnedRecording and importCompletedRecording are denied at the call with unsupported_version instead of buffering and uploading segments that can never be bound. Renewal is no longer retried on a wire that has no renewal route. - A response failing its F01 shape check still fails closed as unsupported_version, and now warns once (FI_ATTACHMENTS_WIRE_REJECTED) with the shape name only. - `failed` leaves AttachmentCompletion, so a leftover check is TS2367 like the retired `queued` ready state. A tsc test pins both. - ATTACHMENTS-MIGRATION.md lists every change, how it fails, and the fix. Co-Authored-By: Claude Opus 5.5 --- .../packages/fi-core/ATTACHMENTS-MIGRATION.md | 16 ++ typescript/packages/fi-core/package.json | 3 +- .../packages/fi-core/src/attachments/index.ts | 1 + .../fi-core/src/attachments/manager.ts | 19 ++- .../fi-core/src/attachments/transport-v1.ts | 39 +++-- .../packages/fi-core/src/attachments/types.ts | 12 +- .../packages/fi-core/src/attachments/warn.ts | 40 +++++ .../fi-core/test/attachments-compat.test.cjs | 151 ++++++++++++++++++ .../fi-core/test/attachments-design.test.cjs | 9 +- .../fi-core/test/types/attachments-compat.ts | 32 ++++ .../packages/fi-core/test/types/tsconfig.json | 13 ++ 11 files changed, 308 insertions(+), 27 deletions(-) create mode 100644 typescript/packages/fi-core/ATTACHMENTS-MIGRATION.md create mode 100644 typescript/packages/fi-core/src/attachments/warn.ts create mode 100644 typescript/packages/fi-core/test/attachments-compat.test.cjs create mode 100644 typescript/packages/fi-core/test/types/attachments-compat.ts create mode 100644 typescript/packages/fi-core/test/types/tsconfig.json diff --git a/typescript/packages/fi-core/ATTACHMENTS-MIGRATION.md b/typescript/packages/fi-core/ATTACHMENTS-MIGRATION.md new file mode 100644 index 000000000..0ef441666 --- /dev/null +++ b/typescript/packages/fi-core/ATTACHMENTS-MIGRATION.md @@ -0,0 +1,16 @@ +# Attachments API migration (`@traceai/fi-core/attachments`) + +The streaming, recording and import work changed a few behaviours that integrations built against the first attachment runtime depend on. Earlier, some of these changes broke callers silently: capture just stopped. Each one now fails loudly, as a compile error, a one-time process warning with a stable code, or an explicit reason at the call. This page lists each change and what to do about it. + +| Change | Old | New | How it fails now | What to change | +|---|---|---|---|---| +| Ready state | `status().state === "queued"` | `status().state === "ready"` | TypeScript: TS2367 on any comparison with `"queued"`. JavaScript: the comparison is always false, so check for `"ready"`. | Compare with `"ready"`. Admission results (`handle.admission`, `ticket.reason`, `writer.reason`) still use `"queued"` for "admitted". | +| Completion `failed` | `completion.status === "failed"` | `"incomplete"` (transport or unknown) or `"unsupported_version"` (invalid response shape) | TypeScript: TS2367. `failed` is no longer in `AttachmentCompletion`. | Treat any status other than `validating` / `importing` as not delivered. A legacy transport that still throws `AttachmentTransportError("failed")` is mapped to `incomplete`. | +| Four-method transport (`bootstrap` / `initialize` / `upload` / `complete`) | Full support | Adapted, with reduced capability | Warns once per process: `DeprecationWarning` code `FI_ATTACHMENTS_LEGACY_TRANSPORT`. `startRecording`, WAV-tier `attachOwnedRecording` and `importCompletedRecording` return `unsupported_version` at the call, so no bytes are buffered or uploaded. Sessions cannot renew, so capture reports `expired` after the first session lifetime. | Pass `createAttachmentHttpTransportV1(...)` or implement `AttachmentTransportV1`. A wrapper around `createAttachmentHttpTransport(...)` must forward its `[ATTACHMENT_TRANSPORT_V1]` member, or wrap the V1 surface directly. Otherwise the wrapper silently becomes legacy. | +| Strict F01 response shapes | Lenient | Every response is checked against F01 (for example, status bodies need `reason`) | The affected attachment reports `unsupported_version`, and a warning with code `FI_ATTACHMENTS_WIRE_REJECTED` names the shape once, such as `UploadStatus` or `AttachmentSession`. The warning contains no values. | Make the server or test double match `spec/attachments/v1`. | +| Session renewal triggers | Only admission calls (`preflight`, `attach`, ...) | Also `status()`, `flush()`, worker wake and recording writes (design 3.1) | Not an error. HTTP doubles that mint short sessions see an extra `POST /tracer/v1/attachment-sessions` with `Idempotency-Key: bs-{session}-{n}` near expiry. | Mint sessions of one hour or more in doubles, or expect the renewal request. | +| `manager.correlation(span)` | Stamped `fi.attachment.*` on the span | Returns a snapshot only. Stamping happens when a capture is admitted (design 0.8). | Not an error: a span without an admitted capture carries no attachment attributes, which is intended. | Pass the correlation (or the span) to `attach` / `preflight` / `startRecording`. Do not rely on `correlation()` alone to mark a span. | + +## Warnings + +Both warning codes are exported as `ATTACHMENT_WARNING_LEGACY_TRANSPORT` and `ATTACHMENT_WARNING_WIRE_REJECTED`. On Node they go through `process.emitWarning`, so `--no-deprecation` and `--no-warnings` apply. In browsers they go to `console.warn`. Each is emitted at most once per process, per code and shape. diff --git a/typescript/packages/fi-core/package.json b/typescript/packages/fi-core/package.json index c0b9be5fc..e9ac05c53 100644 --- a/typescript/packages/fi-core/package.json +++ b/typescript/packages/fi-core/package.json @@ -35,7 +35,7 @@ "postbuild": "node scripts/postbuild.cjs", "test": "jest", "test:provider": "node --test test/provider-lifecycle.test.cjs", - "test:attachments": "node --test test/attachments.test.cjs test/attachments-tls.test.cjs test/attachments-design.test.cjs test/attachments-files.test.cjs test/attachments-exit.test.cjs test/attachments-wire.test.cjs", + "test:attachments": "node --test test/attachments.test.cjs test/attachments-tls.test.cjs test/attachments-design.test.cjs test/attachments-files.test.cjs test/attachments-exit.test.cjs test/attachments-wire.test.cjs test/attachments-compat.test.cjs", "test:watch": "jest --watch", "test:coverage": "jest --coverage", "lint": "eslint . --ext .ts", @@ -44,6 +44,7 @@ "files": [ "dist", "README.md", + "ATTACHMENTS-MIGRATION.md", "src" ], "repository": { diff --git a/typescript/packages/fi-core/src/attachments/index.ts b/typescript/packages/fi-core/src/attachments/index.ts index 6fc134ada..d52ec8903 100644 --- a/typescript/packages/fi-core/src/attachments/index.ts +++ b/typescript/packages/fi-core/src/attachments/index.ts @@ -7,6 +7,7 @@ export { AttachmentConsumedStream } from "./stream"; export { AttachmentRecording, attachmentBoundaryStep, attachmentSegmentFrames, attachmentFramesToMs, attachmentWavHeader, validAttachmentManifest } from "./recording"; export { AttachmentUnavailableResolver } from "./imports"; export { validAttachmentWire, ATTACHMENT_WIRE_SHAPES } from "./wire"; +export { ATTACHMENT_WARNING_LEGACY_TRANSPORT, ATTACHMENT_WARNING_WIRE_REJECTED } from "./warn"; export { AttachmentCorrelation, canonicalAttachmentTraceId, attachmentTraceUuid, attachmentMessageUri } from "./correlation"; export { evaluateAttachmentPolicy, attachmentModality } from "./policy"; export { diff --git a/typescript/packages/fi-core/src/attachments/manager.ts b/typescript/packages/fi-core/src/attachments/manager.ts index 69b3289cb..b0f1359f8 100644 --- a/typescript/packages/fi-core/src/attachments/manager.ts +++ b/typescript/packages/fi-core/src/attachments/manager.ts @@ -13,7 +13,8 @@ import { type RecordingFileSegment, } from "./recording"; import { AttachmentConsumedStream, type StreamHost, type StreamState } from "./stream"; -import { AttachmentTransportError, attachmentIdempotencyKey, attachmentWireTransport, validAttachmentSession, validAttachmentUpload } from "./transport-v1"; +import { AttachmentTransportError, attachmentIdempotencyKey, attachmentIsLegacyWire, attachmentWireTransport, validAttachmentSession, validAttachmentUpload } from "./transport-v1"; +import { attachmentWireRejected } from "./warn"; import type { AttachmentAdmission, AttachmentAdmissionTicket, AttachmentClock, AttachmentCompletedRecordingRequest, AttachmentCompletion, AttachmentContentSource, AttachmentFields, AttachmentFlushResult, AttachmentHandle, AttachmentImportHandle, AttachmentImportOptions, AttachmentManagerOptions, AttachmentOptions, @@ -108,6 +109,8 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost private readonly bootController?: AbortController; private readonly boot: Promise; private readonly wire?: AttachmentTransportV1; + /** A four-method transport: no renewal, manifests or imports (warned once at construction). */ + private readonly legacyWire: boolean = false; private readonly lanes = new Map(); private order: Lane[] = []; private cursor = 0; @@ -164,6 +167,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost if (!this.enabled) { this.bootStatus = "disabled"; this.boot = Promise.resolve(false); return; } this.bootController = new AbortController(); this.wire = attachmentWireTransport(options.transport); + this.legacyWire = attachmentIsLegacyWire(this.wire); this.boot = this.bootstrap(); } private env(): Readonly> { @@ -181,7 +185,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost ...(this.config.workspaceId ? { workspace_id: this.config.workspaceId } : {}) }; const key = attachmentIdempotencyKey("bootstrap", this.sessionId, 0); const session = await this.attempt(signal => this.wire!.bootstrap(request, key, signal), this.bootController!.signal); - if (!validAttachmentSession(session, this.sessionId, this.clock.wall())) throw new AttachmentTransportError("unsupported_version"); + if (!validAttachmentSession(session, this.sessionId, this.clock.wall())) throw attachmentWireRejected("AttachmentSession", new AttachmentTransportError("unsupported_version")); if (this.config.workspaceId && session.scope.workspace_id !== this.config.workspaceId) throw new AttachmentTransportError("scope_unresolved"); if (this.closed) return false; // Detached bounded wire object: a caller/transport cannot later loosen policy through mutation. @@ -302,7 +306,8 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost } private maybeRenew(): void { const session = this.session; - if (!session || this.renewing || this.closed || this.stoppingFlag || this.bootStatus !== "ready" || !this.wire) return; + // A legacy wire has no renewal route: retrying it would only spin; the session runs to `expired` (warned at construction). + if (!session || this.renewing || this.closed || this.stoppingFlag || this.bootStatus !== "ready" || !this.wire || this.legacyWire) return; const now = this.clock.wall(); const expires = Date.parse(session.expires_at); if (expires - now > Math.max(60000, 0.2 * this.sessionLifetime)) return; if (this.clock.now() < this.nextRenewalAt) return; @@ -311,7 +316,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost void (async () => { try { const next = await this.attempt(signal => this.wire!.renewSession(session, sequence, signal), this.bootController!.signal, () => undefined, 3); - if (!validAttachmentSession(next, this.sessionId, this.clock.wall())) throw new AttachmentTransportError("unsupported_version"); + if (!validAttachmentSession(next, this.sessionId, this.clock.wall())) throw attachmentWireRejected("AttachmentSession", new AttachmentTransportError("unsupported_version")); const scope = (s: AttachmentSession) => `${s.scope.organization_id}/${s.scope.workspace_id}/${s.scope.project_id}`; if (scope(next) !== scope(session)) { this.invalidate("scope_unresolved" as "deleted"); return; } if (this.session !== session || this.closed) return; @@ -642,6 +647,8 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost if (evaluated.reason) return inert(evaluated.reason); trackDenials = evaluated.tracks; } + // Fail at start, not at the manifest: a legacy wire cannot publish, so no bytes are retained or uploaded. + if (this.legacyWire) return inert("unsupported_version"); const correlation = this.resolveCorrelation(target); const identity = correlation?.snapshot(); if (!correlation || !identity) return inert("correlation_unavailable"); @@ -689,6 +696,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost if (!Number.isInteger(size) || size <= 0 || (!wav && size > singleCap)) return denied("too_large"); if (this.descriptors() >= this.maxDescriptors || this.bytes + CHUNK > this.maxBytes) return denied("queue_full"); if (wav && options.wavTracks && !this.validWavTracks(options.wavTracks)) return denied("policy_denied"); + if (wav && this.legacyWire) return denied("unsupported_version"); this.bytes += CHUNK; return this.ownedTransfer(record, facts, correlation!, identity!, options, wav, singleCap); } catch { return denied("source_unavailable"); } @@ -854,6 +862,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost const { reason, correlation, identity } = this.admitTarget(target, facts); // Denied preflight never calls the resolver. if (reason) return finished(reason); + if (this.legacyWire) return finished("unsupported_version"); if (!validAttachmentImportRequest(request) || !options?.resolver || typeof options.resolver.resolve !== "function") return finished("policy_denied"); if (this.descriptors() >= this.maxDescriptors || this.bytes + 1 > this.maxBytes) return finished("queue_full"); const id = ++this.nextReservation; this.bytes += 1; this.reservations.set(id, { bytes: 1, expiresAt: Infinity }); @@ -891,7 +900,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost const done = handle("queued"); this.reservations.delete(id); // the descriptor becomes the job this.enqueue({ lane: recordingId, bytes: 1, controller: new AbortController(), denied: () => this.recheck(finalFacts), - fail: reason => { state = reason === "failed" ? "incomplete" : reason; done.finish(state); }, + fail: reason => { state = reason; done.finish(state); }, run: async signal => { try { const job = await this.attempt(s => this.wire!.submitImport(this.session!, body, attachmentIdempotencyKey("import", recordingId), s), signal, diff --git a/typescript/packages/fi-core/src/attachments/transport-v1.ts b/typescript/packages/fi-core/src/attachments/transport-v1.ts index 662f7d660..8dad0faef 100644 --- a/typescript/packages/fi-core/src/attachments/transport-v1.ts +++ b/typescript/packages/fi-core/src/attachments/transport-v1.ts @@ -1,6 +1,7 @@ import { context } from "@opentelemetry/api"; import { suppressTracing } from "@opentelemetry/core"; import { validAttachmentUuid } from "./correlation"; +import { attachmentWarnLegacyTransport, attachmentWireRejected } from "./warn"; import type { AttachmentAdmission, AttachmentBootstrap, AttachmentCompletionStatus, AttachmentContentSource, AttachmentImportJob, AttachmentPacer, AttachmentSession, AttachmentTransport, AttachmentTransportV1, AttachmentUpload, AttachmentUploadRequest, @@ -154,7 +155,7 @@ export const attachmentFetchClient: AttachmentHttpClient = async request => { const ok = response.status >= 200 && response.status < 300; const data = await readBounded(response, ok ? SUCCESS_BODY : ERROR_BODY); if (!data) { - if (ok) throw new AttachmentTransportError("unsupported_version"); + if (ok) throw attachmentWireRejected("response size", new AttachmentTransportError("unsupported_version")); return { status: response.status, body: new Uint8Array(), retryAfter: response.headers.get("retry-after") ?? undefined }; } return { status: response.status, body: data, retryAfter: response.headers.get("retry-after") ?? undefined }; @@ -235,13 +236,13 @@ export function createAttachmentHttpTransportV1(options: AttachmentHttpTransport throw new AttachmentTransportError("incomplete", !signal.aborted); } if (response.status !== expected) { - if (response.status >= 200 && response.status < 300) throw new AttachmentTransportError("unsupported_version"); + if (response.status >= 200 && response.status < 300) throw attachmentWireRejected("HTTP success status", new AttachmentTransportError("unsupported_version")); throw attachmentHttpError(response.status, response.body, response.retryAfter); } - if (response.body.byteLength > SUCCESS_BODY) throw new AttachmentTransportError("unsupported_version"); + if (response.body.byteLength > SUCCESS_BODY) throw attachmentWireRejected("response size", new AttachmentTransportError("unsupported_version")); if (expected === 204) return undefined; try { return attachmentStrictJson(new TextDecoder("utf-8", { fatal: true }).decode(response.body)); } - catch { throw new AttachmentTransportError("unsupported_version"); } + catch { throw attachmentWireRejected("strict JSON body", new AttachmentTransportError("unsupported_version")); } } const base = (session: AttachmentSession) => { if (!validAttachmentUuid(session.scope.project_id)) throw new AttachmentTransportError("scope_unresolved"); @@ -258,7 +259,7 @@ export function createAttachmentHttpTransportV1(options: AttachmentHttpTransport const v1: AttachmentTransportV1 = { async bootstrap(body, key, signal) { const result = await send("POST", "/tracer/v1/attachment-sessions", JSON.stringify(body), signal, 200, { key }); - if (!validAttachmentSession(result, body.capture_session_id)) throw new AttachmentTransportError("unsupported_version"); + if (!validAttachmentSession(result, body.capture_session_id)) throw attachmentWireRejected("AttachmentSession", new AttachmentTransportError("unsupported_version")); bootstraps.set(body.capture_session_id, { ...body }); return result; }, @@ -268,20 +269,20 @@ export function createAttachmentHttpTransportV1(options: AttachmentHttpTransport if (!body) throw new AttachmentTransportError("unsupported_version"); const result = await send("POST", "/tracer/v1/attachment-sessions", JSON.stringify(body), signal, 200, { key: attachmentIdempotencyKey("bootstrap", current.capture_session_id, sequence) }); - if (!validAttachmentSession(result, body.capture_session_id)) throw new AttachmentTransportError("unsupported_version"); + if (!validAttachmentSession(result, body.capture_session_id)) throw attachmentWireRejected("AttachmentSession", new AttachmentTransportError("unsupported_version")); return result; }, async initialize(session, body, key, signal) { const request = parseUploadBody(body); const result = await send("POST", `${base(session)}/uploads`, body, signal, 201, { key }); - if (!validAttachmentUpload(result, request)) throw new AttachmentTransportError("unsupported_version"); + if (!validAttachmentUpload(result, request)) throw attachmentWireRejected("UploadResponse", new AttachmentTransportError("unsupported_version")); return result; }, async renewUpload(session, upload, body, signal) { // Replay the byte-identical initialize under the same key with the ORIGINAL identity (BR-1). const request = parseUploadBody(body); const result = await send("POST", `${base(session)}/uploads`, body, signal, 201, { key: attachmentIdempotencyKey("upload", request.occurrence_id) }); - if (!validAttachmentUpload(result, request)) throw new AttachmentTransportError("unsupported_version"); + if (!validAttachmentUpload(result, request)) throw attachmentWireRejected("UploadResponse", new AttachmentTransportError("unsupported_version")); if (result.upload_id !== upload.upload_id || result.write_capability === upload.write_capability || Date.parse(result.expires_at) <= Date.parse(upload.expires_at)) throw new AttachmentTransportError("expired"); return result; @@ -290,21 +291,21 @@ export function createAttachmentHttpTransportV1(options: AttachmentHttpTransport const size = source instanceof Uint8Array ? source.byteLength : (source as { size: number }).size; if (size !== upload.reserved_bytes) throw new AttachmentTransportError("too_large"); const result = await send("PUT", `${uploadPath(session, upload)}/content`, source, signal, 200, { capability: upload.write_capability, binary: true, pacer }); - if (!validStatus(result, upload.upload_id, "upload_id", "pending_upload")) throw new AttachmentTransportError("unsupported_version"); + if (!validStatus(result, upload.upload_id, "upload_id", "pending_upload")) throw attachmentWireRejected("UploadStatus", new AttachmentTransportError("unsupported_version")); }, async complete(session, upload, key, signal) { const result = await send("POST", `${uploadPath(session, upload)}/complete`, completeBody(session), signal, 202, { key, capability: upload.write_capability }); // validating, never ready: late-root and root-first orders both still need validation (F01 lifecycle fixtures). - if (!validStatus(result, upload.upload_id, "upload_id", "validating")) throw new AttachmentTransportError("unsupported_version"); + if (!validStatus(result, upload.upload_id, "upload_id", "validating")) throw attachmentWireRejected("UploadStatus", new AttachmentTransportError("unsupported_version")); return "validating"; }, async abort(session, upload, key, signal) { const result = await send("POST", `${uploadPath(session, upload)}/abort`, completeBody(session), signal, 200, { key, capability: upload.write_capability }); - if (!validStatus(result, upload.upload_id, "upload_id")) throw new AttachmentTransportError("unsupported_version"); + if (!validStatus(result, upload.upload_id, "upload_id")) throw attachmentWireRejected("UploadStatus", new AttachmentTransportError("unsupported_version")); }, async uploadStatus(session, upload, signal) { const result = await send("GET", `${uploadPath(session, upload)}/status`, undefined, signal, 200, { capability: upload.write_capability }); - if (!validStatus(result, upload.upload_id, "upload_id")) throw new AttachmentTransportError("unsupported_version"); + if (!validStatus(result, upload.upload_id, "upload_id")) throw attachmentWireRejected("UploadStatus", new AttachmentTransportError("unsupported_version")); return { state: result.state, reason: result.reason }; }, async putManifest(session, capability, recordingId, body, key, signal) { @@ -318,13 +319,13 @@ export function createAttachmentHttpTransportV1(options: AttachmentHttpTransport catch { throw new AttachmentTransportError("unsupported_version"); } const result = await send("POST", `${base(session)}/imports`, body, signal, 202, { key }); if (!object(result) || Object.keys(result).length !== 4 || result.mode !== "completed_recording" || result.recording_id !== recordingId || - result.state !== "importing" || !validAttachmentUuid(result.job_id)) throw new AttachmentTransportError("unsupported_version"); + result.state !== "importing" || !validAttachmentUuid(result.job_id)) throw attachmentWireRejected("RecordingImportResponse", new AttachmentTransportError("unsupported_version")); return { jobId: result.job_id, recordingId, state: "importing" } as AttachmentImportJob; }, async importStatus(session, jobId, signal) { if (!validAttachmentUuid(jobId)) throw new AttachmentTransportError("unsupported_version"); const result = await send("GET", `${base(session)}/imports/${jobId}/status`, undefined, signal, 200); - if (!validStatus(result, jobId, "job_id")) throw new AttachmentTransportError("unsupported_version"); + if (!validStatus(result, jobId, "job_id")) throw attachmentWireRejected("ImportStatus", new AttachmentTransportError("unsupported_version")); return { state: result.state, reason: result.reason }; }, }; @@ -384,11 +385,17 @@ export function attachmentLegacyTransport(transport: AttachmentTransport): Attac importStatus: unsupported, }; } -/** Pick the versioned surface of whatever transport shape the caller provided. */ +const LEGACY_WIRES = new WeakSet(); +/** @internal True for the adapter around a four-method transport, which cannot renew, manifest or import. */ +export function attachmentIsLegacyWire(wire: AttachmentTransportV1 | undefined): boolean { return !!wire && LEGACY_WIRES.has(wire); } +/** Pick the versioned surface of whatever transport shape the caller provided. A four-method shape warns once (loudly). */ export function attachmentWireTransport(transport: AttachmentTransport | AttachmentTransportV1 | undefined): AttachmentTransportV1 | undefined { if (!transport) return undefined; const carried = (transport as { [ATTACHMENT_TRANSPORT_V1]?: AttachmentTransportV1 })[ATTACHMENT_TRANSPORT_V1]; if (carried && typeof carried.putContent === "function") return carried; if (typeof (transport as AttachmentTransportV1).putContent === "function") return transport as AttachmentTransportV1; - return attachmentLegacyTransport(transport as AttachmentTransport); + attachmentWarnLegacyTransport(); + const legacy = attachmentLegacyTransport(transport as AttachmentTransport); + LEGACY_WIRES.add(legacy); + return legacy; } diff --git a/typescript/packages/fi-core/src/attachments/types.ts b/typescript/packages/fi-core/src/attachments/types.ts index f4cedb3b0..3cdadf77a 100644 --- a/typescript/packages/fi-core/src/attachments/types.ts +++ b/typescript/packages/fi-core/src/attachments/types.ts @@ -18,8 +18,11 @@ export type AttachmentAdmission = "queued" | "disabled" | "bootstrap_pending" | /** Terminal transfer outcomes (design 0.3). `validating` means transferred, NOT readable. `importing` is an accepted import job. */ export type AttachmentCompletionStatus = "validating" | "importing" | "incomplete" | "expired" | "deleted" | "policy_denied" | "too_large" | "throttled" | "conflict" | "not_found" | "credential_scope_unsupported" | "validation_failed" | "unsupported_version" | "source_unavailable" | "redaction_unavailable"; -/** `failed` is retired (design 0.3): it is never produced and stays in the type only for source compatibility. */ -export type AttachmentCompletion = Readonly<{ status: AttachmentAdmission | AttachmentCompletionStatus | "failed" }>; +/** + * `failed` is retired (design 0.3) and is no longer a member, so a leftover `status === "failed"` check is a compile error + * (TS2367) instead of a branch that silently never runs. Failures are `incomplete` or `unsupported_version`. + */ +export type AttachmentCompletion = Readonly<{ status: AttachmentAdmission | AttachmentCompletionStatus }>; export interface AttachmentHandle { readonly attachmentId: string; readonly occurrenceId: string; @@ -179,7 +182,10 @@ export type AttachmentSpan = Span | SpanContext; // Design sections 0-9 (sdk-streaming-design.md). Additive; earlier names are unchanged. // --------------------------------------------------------------------------- export interface AttachmentClock { now(): number; wall(): number } -/** Runtime state (design 0.2). `deleted` and `policy_denied` remain only for an explicit `invalidate()`. */ +/** + * Runtime state (design 0.2). `deleted` and `policy_denied` remain only for an explicit `invalidate()`. + * The old ready value `"queued"` is retired for `"ready"`; it is deliberately absent, so `state !== "queued"` is a compile error (TS2367). + */ export type AttachmentRuntimeState = "disabled" | "endpoint_unconfigured" | "credential_scope_unsupported" | "scope_unresolved" | "bootstrap_pending" | "ready" | "expired" | "unsupported_version" | "incomplete" | "deleted" | "policy_denied" | "source_unavailable"; export interface AttachmentStatus { diff --git a/typescript/packages/fi-core/src/attachments/warn.ts b/typescript/packages/fi-core/src/attachments/warn.ts new file mode 100644 index 000000000..7a483ac1b --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/warn.ts @@ -0,0 +1,40 @@ +/** + * One-time, process-visible diagnostics for integration mistakes that would otherwise only show up as attachments + * quietly not arriving. Browser-safe: no `node:*` import. Messages carry fixed text and shape names only, never values, + * URLs, capabilities or credentials. + */ +export const ATTACHMENT_WARNING_LEGACY_TRANSPORT = "FI_ATTACHMENTS_LEGACY_TRANSPORT"; +export const ATTACHMENT_WARNING_WIRE_REJECTED = "FI_ATTACHMENTS_WIRE_REJECTED"; +const MIGRATION = "See ATTACHMENTS-MIGRATION.md in @traceai/fi-core."; +const seen = new Set(); + +type Emitter = (message: string, options: { type: string; code: string }) => void; +/** Emits once per key per process: `process.emitWarning` on Node (honours --no-deprecation/--no-warnings), else `console.warn`. */ +export function attachmentWarnOnce(key: string, code: string, message: string, deprecation = false): void { + if (seen.has(key)) return; + seen.add(key); + try { + const proc = (globalThis as { process?: { emitWarning?: Emitter } }).process; + if (proc && typeof proc.emitWarning === "function") { proc.emitWarning(`${message} ${MIGRATION}`, { type: deprecation ? "DeprecationWarning" : "Warning", code }); return; } + const out = (globalThis as { console?: { warn?: (...args: unknown[]) => void } }).console; + if (out && typeof out.warn === "function") out.warn(`[${code}] ${message} ${MIGRATION}`); + } catch { /* Diagnostics never break application code (e.g. --throw-deprecation). */ } +} + +/** A four-method `AttachmentTransport` was adapted: say exactly what it cannot do, once. */ +export function attachmentWarnLegacyTransport(): void { + attachmentWarnOnce("legacy-transport", ATTACHMENT_WARNING_LEGACY_TRANSPORT, + "A legacy four-method AttachmentTransport (bootstrap/initialize/upload/complete) was passed to AttachmentManager. " + + "It cannot renew sessions, PUT recording manifests, submit imports or read upload status: attachments stop with `expired` " + + "after the first session lifetime, and startRecording, WAV-tier attachOwnedRecording and importCompletedRecording are denied with " + + "`unsupported_version`. Pass createAttachmentHttpTransportV1(...) or an AttachmentTransportV1; a wrapper must forward the " + + "ATTACHMENT_TRANSPORT_V1 member of the transport it wraps.", true); +} + +/** A response failed its F01 shape check. Fails closed as `unsupported_version` and names the shape once. */ +export function attachmentWireRejected(shape: string, error: E): E { + attachmentWarnOnce(`wire:${shape}`, ATTACHMENT_WARNING_WIRE_REJECTED, + `An attachment API response failed the F01 ${shape} check, so the affected attachments report \`unsupported_version\`. ` + + "Check the server, or the test double, against spec/attachments/v1 (every status body carries `reason`)."); + return error; +} diff --git a/typescript/packages/fi-core/test/attachments-compat.test.cjs b/typescript/packages/fi-core/test/attachments-compat.test.cjs new file mode 100644 index 000000000..840c6be76 --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-compat.test.cjs @@ -0,0 +1,151 @@ +/* + * Stacked-lane compatibility: API changes that used to break callers SILENTLY (capture just stopped) must now be loud: + * a compile error, a one-time process warning with a stable code, or an explicit reason at the call. Synthetic data only. + * See ATTACHMENTS-MIGRATION.md. + */ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { core, clock, mono, pcm } = require('./attachments-v1.cjs'); +const { identity, ids, policy, limits, fake } = require('./attachments-fixtures.cjs'); +const { AttachmentManager } = core; +const tick = () => new Promise(resolve => setImmediate(resolve)); +const helpers = { v1: require.resolve('./attachments-v1.cjs'), fixtures: require.resolve('./attachments-fixtures.cjs') }; + +/** Runs `body` in a fresh Node process (warnings are once per process) and returns its stderr and last stdout line. */ +function isolated(body) { + const script = ` + const { core, managerV1, clock, mono, pcm } = require(${JSON.stringify(helpers.v1)}); + const { identity, ids, policy, limits, fake } = require(${JSON.stringify(helpers.fixtures)}); + const out = {}; const warnings = []; + process.on('warning', w => warnings.push({ name: w.name, code: w.code })); + (async () => { ${body} + await new Promise(r => setImmediate(r)); + console.log(JSON.stringify({ out, warnings })); + })().catch(e => { console.error(e && e.stack); process.exit(3); });`; + const r = spawnSync(process.execPath, ['-e', script], { env: process.env, encoding: 'utf8', timeout: 20000 }); + assert.equal(r.status, 0, r.stderr); + return { stderr: r.stderr, ...JSON.parse(r.stdout.trim().split('\n').pop()) }; +} +const legacySession = sessionPolicy => ({ + async bootstrap(r) { + return { schema_version: 1, scope: { organization_id: ids[0], workspace_id: ids[1], project_id: ids[2] }, capture_session_id: r.capture_session_id, + policy: { ...policy, ...sessionPolicy }, limits, expires_at: new Date(Date.now() + 3600000).toISOString(), transfer_modes: ['authenticated_stream'] }; + }, +}); + +test('C-01 a legacy four-method transport warns ONCE per process (DeprecationWarning FI_ATTACHMENTS_LEGACY_TRANSPORT) and names the fix', () => { + const r = isolated(` + for (let i = 0; i < 3; i++) { + const m = new core.AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: fake().transport }); + out['ready' + i] = await m.ready(); await m.shutdown(100); + }`); + assert.deepEqual(r.warnings, [{ name: 'DeprecationWarning', code: 'FI_ATTACHMENTS_LEGACY_TRANSPORT' }]); + assert.match(r.stderr, /FI_ATTACHMENTS_LEGACY_TRANSPORT/); + assert.match(r.stderr, /createAttachmentHttpTransportV1/); assert.match(r.stderr, /ATTACHMENT_TRANSPORT_V1/); assert.match(r.stderr, /ATTACHMENTS-MIGRATION\.md/); + assert.deepEqual(r.out, { ready0: true, ready1: true, ready2: true }, 'the legacy shape still works for single uploads'); +}); + +test('C-02 no warning for an AttachmentTransportV1, for createAttachmentHttpTransport (carries V1), or when attachments are OFF', () => { + const r = isolated(` + const http = core.createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: {}, client: async () => ({ status: 503, body: new Uint8Array() }) }); + const f = await managerV1(); await f.manager.shutdown(100); + const h = new core.AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: http }); await h.shutdown(100); + const off = new core.AttachmentManager({ transport: fake().transport }); await off.shutdown(100);`); + assert.deepEqual(r.warnings, []); + assert.doesNotMatch(r.stderr, /FI_ATTACHMENTS_/); +}); + +test('C-03 a wrapper that drops ATTACHMENT_TRANSPORT_V1 is detected as legacy (the coding-lane guard shape) and warns', () => { + const r = isolated(` + const http = core.createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: {}, client: async () => ({ status: 503, body: new Uint8Array() }) }); + const guarded = { bootstrap: (...a) => http.bootstrap(...a), initialize: (...a) => http.initialize(...a), upload: (...a) => http.upload(...a), complete: (...a) => http.complete(...a) }; + const m = new core.AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: guarded }); await m.shutdown(100); + const forwarded = { ...guarded, [core.ATTACHMENT_TRANSPORT_V1]: http[core.ATTACHMENT_TRANSPORT_V1] }; + out.forwardedIsV1 = typeof forwarded[core.ATTACHMENT_TRANSPORT_V1].putContent === 'function';`); + assert.deepEqual(r.warnings, [{ name: 'DeprecationWarning', code: 'FI_ATTACHMENTS_LEGACY_TRANSPORT' }]); + assert.equal(r.out.forwardedIsV1, true); +}); + +test('C-04 legacy wire: recording, WAV import and completed-recording import are denied up front with unsupported_version; single uploads still work', async () => { + const f = fake(legacySession({ allowed_sources: ['Bytes', 'Blob', 'OwnedRecordingFile', 'ProviderFile', 'RemoteUrl'], record_audio: true })); + const m = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: f.transport, recordAudio: true }); + assert.equal(await m.ready(), true); + const w = m.startRecording(identity, mono(8000)); + assert.equal(w.reason, 'unsupported_version'); assert.equal(w.tryWrite(pcm(160)), false); + assert.equal((await w.finish(100)).state, 'denied'); + let resolverCalls = 0; + const imp = await m.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_synthetic' }, { resolver: { async resolve() { resolverCalls++; return 'source_unavailable'; } } }); + assert.equal(imp.reason, 'unsupported_version'); assert.equal(resolverCalls, 0, 'the resolver is never called'); + assert.equal((await imp.completion).status, 'unsupported_version'); + assert.equal((await m.attach(identity, { kind: 'Bytes', data: new Uint8Array([1, 2]) }, { mimeType: 'image/png', direction: 'input' }).completion).status, 'validating'); + assert.deepEqual(f.calls, ['initialize', 'upload', 'complete'], 'only the single upload reached the transport (bootstrap is the override)'); + await m.flush(1000); // the finished job releases its bytes just after its completion settles + assert.equal(m.status().ownedBytes, 0); assert.equal(m.status().openRecordings, 0); + await m.shutdown(100); +}); + +test('C-05 deny-wins: on a legacy wire a policy denial still beats unsupported_version', async () => { + const f = fake(legacySession({ record_audio: false })); + const m = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: f.transport, traceConfig: { hideInputs: true } }); + assert.equal(await m.ready(), true); + assert.equal(m.startRecording(identity, mono(8000)).reason, 'policy_denied'); + assert.ok(['policy_denied', 'recording_disabled'].includes(m.startRecording(identity, { ...mono(8000), operation: 'create_recording' }).reason)); + await m.shutdown(100); + const off = new AttachmentManager({ transport: fake().transport }); + assert.equal(off.startRecording(identity, mono(8000)).reason, 'disabled'); +}); + +test('C-06 legacy wire: no renewal reaches the transport; the session runs to `expired` and capture says so', async () => { + const c = clock(Date.now()); let boots = 0; + const f = fake({ async bootstrap(r) { boots++; return { ...(await legacySession({}).bootstrap(r)), expires_at: new Date(c.wall() + 120000).toISOString() }; } }); + const m = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: f.transport, clock: c }); + assert.equal(await m.ready(), true); + c.advance(100000); m.status(); await m.flush(100); for (let i = 0; i < 10; i++) await tick(); + assert.equal(m.status().state, 'ready'); assert.equal(boots, 1); + c.advance(30000); + assert.equal(m.status().state, 'expired'); + assert.equal(m.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, { mimeType: 'image/png', direction: 'input' }).admission, 'expired'); + await m.shutdown(100); +}); + +test('C-07 a response failing its F01 shape (status body without `reason`) is unsupported_version AND warns once with the shape name', () => { + const r = isolated(` + let n = 0; + const session = sid => ({ schema_version: 1, scope: { organization_id: ids[0], workspace_id: ids[1], project_id: ids[2] }, capture_session_id: sid, + policy, limits, expires_at: new Date(Date.now() + 3600000).toISOString(), transfer_modes: ['authenticated_stream'] }); + const client = async req => { + const p = new URL(req.url).pathname; const json = (status, body) => ({ status, body: Buffer.from(JSON.stringify(body)) }); + if (p.endsWith('/attachment-sessions')) return json(200, session(JSON.parse(req.body).capture_session_id)); + if (p.endsWith('/uploads')) { const b = JSON.parse(req.body); return json(201, { upload_id: '50000000-0000-4000-8000-00000000000' + (++n), attachment_id: b.attachment_id, + occurrence_id: b.occurrence_id, state: 'pending_upload', reserved_bytes: b.declared_bytes, expires_at: new Date(Date.now() + 600000).toISOString(), + transfer_mode: 'authenticated_stream', write_capability: 'synthetic-capability-' + n + '-abcdef' }); } + if (p.endsWith('/content')) return json(200, { upload_id: p.split('/')[7], state: 'pending_upload' }); // no reason: the pre-F01 double shape + if (p.endsWith('/abort')) return json(200, { upload_id: p.split('/')[7], state: 'failed', reason: 'incomplete' }); + return { status: 404, body: new Uint8Array() }; + }; + const transport = core.createAttachmentHttpTransportV1({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic-original' }, client }); + const m = new core.AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport }); + out.ready = await m.ready(); + const a = m.attach(identity, { kind: 'Bytes', data: new Uint8Array([1, 2]) }, { mimeType: 'image/png', direction: 'input' }); + const b = m.attach(identity, { kind: 'Bytes', data: new Uint8Array([3, 4]) }, { mimeType: 'image/png', direction: 'input' }); + out.a = (await a.completion).status; out.b = (await b.completion).status; + await m.shutdown(100);`); + assert.equal(r.out.ready, true); + assert.equal(r.out.a, 'unsupported_version'); assert.equal(r.out.b, 'unsupported_version'); + assert.deepEqual(r.warnings, [{ name: 'Warning', code: 'FI_ATTACHMENTS_WIRE_REJECTED' }]); + assert.match(r.stderr, /F01 UploadStatus check/); + // Shape names only: no capability, credential, URL or body value reaches the warning. + assert.doesNotMatch(r.stderr, /synthetic-capability|synthetic-original|attachments\.example|50000000-/); +}); + +test('C-08 type-level: retired values are compile errors, not silent never-true checks (tsc --noEmit on test/types)', { timeout: 120000 }, t => { + let tsc; + try { tsc = require.resolve('typescript/bin/tsc', { paths: [path.resolve(__dirname, '..')] }); } + catch { t.skip('typescript is not installed next to this fi-core (packed-install run)'); return; } + const project = path.join(__dirname, 'types', 'tsconfig.json'); + const r = spawnSync(process.execPath, [tsc, '--noEmit', '-p', project], { encoding: 'utf8', timeout: 110000 }); + // Every `@ts-expect-error` in the file must be consumed: an unused one (TS2578) means the retired value compiles again. + assert.equal(r.status, 0, `${r.stdout}\n${r.stderr}`); +}); diff --git a/typescript/packages/fi-core/test/attachments-design.test.cjs b/typescript/packages/fi-core/test/attachments-design.test.cjs index 3f28820be..0f2b162cf 100644 --- a/typescript/packages/fi-core/test/attachments-design.test.cjs +++ b/typescript/packages/fi-core/test/attachments-design.test.cjs @@ -784,8 +784,13 @@ test('legacy four-method transports keep working through the adapter (stacked la assert.equal(await m.ready(), true); assert.equal(m.status().state, 'ready'); assert.equal((await m.attach(identity, source(), { mimeType: 'audio/wav', direction: 'mixed' }).completion).status, 'validating'); assert.deepEqual(f.calls, ['bootstrap', 'initialize', 'upload', 'complete']); - const w = m.startRecording(identity, mono(8000)); w.tryWrite(pcm(800)); const r = await w.finish(2000); - assert.equal(r.state, 'unpublished', 'legacy transports cannot PUT manifests: fail closed'); + // Legacy transports cannot PUT manifests: the recording is denied at start (loud), not buffered and uploaded unbindable. + const w = m.startRecording(identity, mono(8000)); + assert.equal(w.reason, 'unsupported_version'); assert.equal(w.tryWrite(pcm(800)), false); + const r = await w.finish(2000); + assert.equal(r.state, 'denied'); assert.equal(r.segmentsPublished, 0); + assert.deepEqual(f.calls, ['bootstrap', 'initialize', 'upload', 'complete'], 'no segment was uploaded'); + assert.equal(m.status().ownedBytes, 0); await m.shutdown(); }); test('T-ST-16 BR-7 fallback: capability rejected after complete -> init, content, manifest, complete for the remaining segments', async () => { diff --git a/typescript/packages/fi-core/test/types/attachments-compat.ts b/typescript/packages/fi-core/test/types/attachments-compat.ts new file mode 100644 index 000000000..cabf1a2d3 --- /dev/null +++ b/typescript/packages/fi-core/test/types/attachments-compat.ts @@ -0,0 +1,32 @@ +/* + * Type-level guard for ATTACHMENTS-MIGRATION.md, compiled against the BUILT declarations (what consumers see). + * Each `@ts-expect-error` must be consumed; if a retired value becomes assignable again, tsc fails with TS2578. + */ +import type { + AttachmentCompletion, AttachmentManager, AttachmentManagerOptions, AttachmentStatus, AttachmentTransport, AttachmentTransportV1, +} from "../../dist/src/attachments/index"; + +declare const manager: AttachmentManager; +declare const status: AttachmentStatus; +declare const completion: AttachmentCompletion; +declare const legacy: AttachmentTransport; +declare const wire: AttachmentTransportV1; + +// Ready state: "queued" -> "ready" (design 0.2). +// @ts-expect-error TS2367: "queued" is no longer a runtime state; compare with "ready". +export const oldReady = manager.status().state !== "queued"; +// @ts-expect-error TS2367: same check through a stored status object. +export const oldReadyStored = status.state === "queued"; +export const newReady: boolean = manager.status().state === "ready"; + +// Completion: "failed" retired for "incomplete" / "unsupported_version" (design 0.3). +// @ts-expect-error TS2367: "failed" is never produced. +export const oldFailed = completion.status === "failed"; +export const newFailed: boolean = completion.status === "incomplete" || completion.status === "unsupported_version"; + +// Admission results keep "queued": that is the admitted reason, not a runtime state. +export const admitted: boolean = manager.attach(undefined, { kind: "Bytes", data: new Uint8Array(1) }, { mimeType: "image/png", direction: "input" }).admission === "queued"; + +// Both transport shapes still type-check; the legacy one is warned about at runtime. +export const legacyOptions: AttachmentManagerOptions = { transport: legacy }; +export const v1Options: AttachmentManagerOptions = { transport: wire }; diff --git a/typescript/packages/fi-core/test/types/tsconfig.json b/typescript/packages/fi-core/test/types/tsconfig.json new file mode 100644 index 000000000..24bcc060c --- /dev/null +++ b/typescript/packages/fi-core/test/types/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "strict": true, + "noEmit": true, + "target": "es2021", + "lib": ["es2021", "dom"], + "module": "commonjs", + "moduleResolution": "node", + "skipLibCheck": true, + "types": [] + }, + "files": ["attachments-compat.ts"] +} From 01b3327a6d42f3a5c6cb1a64e66ad315356d6be2 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 07:17:56 -0800 Subject: [PATCH 8/9] feat(core): deferred attachment targets for late-bound spans Adds manager.deferredCorrelation(): Bytes, Blob and consumed-stream captures can be admitted before their span exists (wire spans that are emitted only after the body completes). Committed bytes stay in the owned budget, capped at half of it, and nothing is sent until target.bind(span), which re-evaluates policy and sampling. Timeout, cancel() and shutdown fail pending captures with correlation_unavailable. Owned files, recordings and imports never defer, and a span that already has a correlation keeps one reference list. startAttachmentStream now reserves min(maxBytes, negotiated single-upload cap), so servers below 4 MiB still admit consumed streams. shutdown() keeps an earlier invalidate() reason instead of overwriting it with incomplete. Co-Authored-By: Claude Opus 5.5 --- .../workflows/fi-core-attachments-tests.yml | 2 +- .../packages/fi-core/ATTACHMENTS-MIGRATION.md | 8 + typescript/packages/fi-core/package.json | 2 +- .../fi-core/src/attachments/correlation.ts | 36 +++ .../fi-core/src/attachments/disabled.ts | 6 +- .../packages/fi-core/src/attachments/index.ts | 2 +- .../fi-core/src/attachments/manager.ts | 170 +++++++++-- .../fi-core/src/attachments/stream.ts | 2 +- .../packages/fi-core/src/attachments/types.ts | 5 + .../test/attachments-deferred.test.cjs | 286 ++++++++++++++++++ 10 files changed, 494 insertions(+), 25 deletions(-) create mode 100644 typescript/packages/fi-core/test/attachments-deferred.test.cjs diff --git a/.github/workflows/fi-core-attachments-tests.yml b/.github/workflows/fi-core-attachments-tests.yml index 3b5e51409..aa9b7fc4c 100644 --- a/.github/workflows/fi-core-attachments-tests.yml +++ b/.github/workflows/fi-core-attachments-tests.yml @@ -48,7 +48,7 @@ jobs: pnpm --dir packages/fi-semantic-conventions pack --pack-destination "$RUNNER_TEMP/attachment-packs" for version in 2.0.1 2.12.1; do node packages/fi-core/test/install-sdk-fixture.cjs "$version" "$RUNNER_TEMP/attachment-sdk-$version" "$RUNNER_TEMP"/attachment-packs/traceai-fi-core-*.tgz "$RUNNER_TEMP"/attachment-packs/traceai-fi-semantic-conventions-*.tgz - FI_CORE_PACKAGE_JSON="$RUNNER_TEMP/attachment-sdk-$version/node_modules/@traceai/fi-core/package.json" FI_ATTACHMENTS_SPEC_DIR="$GITHUB_WORKSPACE/attachment-contract/spec/attachments/v1" node --test packages/fi-core/test/attachments.test.cjs packages/fi-core/test/attachments-tls.test.cjs packages/fi-core/test/attachments-design.test.cjs packages/fi-core/test/attachments-files.test.cjs packages/fi-core/test/attachments-exit.test.cjs packages/fi-core/test/attachments-wire.test.cjs packages/fi-core/test/provider-lifecycle.test.cjs + FI_CORE_PACKAGE_JSON="$RUNNER_TEMP/attachment-sdk-$version/node_modules/@traceai/fi-core/package.json" FI_ATTACHMENTS_SPEC_DIR="$GITHUB_WORKSPACE/attachment-contract/spec/attachments/v1" node --test packages/fi-core/test/attachments.test.cjs packages/fi-core/test/attachments-tls.test.cjs packages/fi-core/test/attachments-design.test.cjs packages/fi-core/test/attachments-files.test.cjs packages/fi-core/test/attachments-exit.test.cjs packages/fi-core/test/attachments-wire.test.cjs packages/fi-core/test/attachments-deferred.test.cjs packages/fi-core/test/provider-lifecycle.test.cjs done - name: Replay privacy and identity fixture safety run: node typescript/packages/fi-core/test/attachments-contract.cjs attachment-contract/spec/attachments/v1 diff --git a/typescript/packages/fi-core/ATTACHMENTS-MIGRATION.md b/typescript/packages/fi-core/ATTACHMENTS-MIGRATION.md index 0ef441666..7d1f1f3d1 100644 --- a/typescript/packages/fi-core/ATTACHMENTS-MIGRATION.md +++ b/typescript/packages/fi-core/ATTACHMENTS-MIGRATION.md @@ -14,3 +14,11 @@ The streaming, recording and import work changed a few behaviours that integrati ## Warnings Both warning codes are exported as `ATTACHMENT_WARNING_LEGACY_TRANSPORT` and `ATTACHMENT_WARNING_WIRE_REJECTED`. On Node they go through `process.emitWarning`, so `--no-deprecation` and `--no-warnings` apply. In browsers they go to `console.warn`. Each is emitted at most once per process, per code and shape. + +## Additions (non-breaking) + +| Addition | Behaviour | +|---|---| +| `manager.deferredCorrelation({ bindTimeoutMs })` returns an `AttachmentDeferredCorrelation` | A capture target for a span that does not exist yet, such as a wire span emitted only after the response body completes. `preflight`, `attach` and `startAttachmentStream` admit `Bytes`, `Blob` and consumed-stream captures against it using the full deny-wins check. Committed bytes stay in the owned budget, capped at half of it, and nothing is sent. `target.bind(span)` checks policy and sampling again, then uploads against that span and projects the references onto it. If the bind timeout passes (default 60 s, max 300 s, checked lazily without a timer), `target.cancel()` is called or the manager shuts down, pending captures end `correlation_unavailable` and upload nothing. Owned files, recordings and imports never defer. If a span already has a correlation, the bound target forwards to it, so the span keeps one reference list. | +| `startAttachmentStream` reservation | The reservation is `min(maxBytes, negotiated single-upload cap)`. A server whose `max_upload_bytes` is below an adapter's default (for example 4 MiB) still admits the stream, and a body larger than the effective cap is discarded whole (`too_large`). `preflight` keeps its exact semantics: a `maxBytes` over the cap is `too_large`. | +| `shutdown()` after `invalidate(reason)` | The state stays `reason` (`policy_denied`, `deleted`, `expired`); before this change, `shutdown()` overwrote it with `incomplete`. | diff --git a/typescript/packages/fi-core/package.json b/typescript/packages/fi-core/package.json index e9ac05c53..ac37844ac 100644 --- a/typescript/packages/fi-core/package.json +++ b/typescript/packages/fi-core/package.json @@ -35,7 +35,7 @@ "postbuild": "node scripts/postbuild.cjs", "test": "jest", "test:provider": "node --test test/provider-lifecycle.test.cjs", - "test:attachments": "node --test test/attachments.test.cjs test/attachments-tls.test.cjs test/attachments-design.test.cjs test/attachments-files.test.cjs test/attachments-exit.test.cjs test/attachments-wire.test.cjs test/attachments-compat.test.cjs", + "test:attachments": "node --test test/attachments.test.cjs test/attachments-tls.test.cjs test/attachments-design.test.cjs test/attachments-files.test.cjs test/attachments-exit.test.cjs test/attachments-wire.test.cjs test/attachments-compat.test.cjs test/attachments-deferred.test.cjs", "test:watch": "jest --watch", "test:coverage": "jest --coverage", "lint": "eslint . --ext .ts", diff --git a/typescript/packages/fi-core/src/attachments/correlation.ts b/typescript/packages/fi-core/src/attachments/correlation.ts index 7174a7761..e341c196e 100644 --- a/typescript/packages/fi-core/src/attachments/correlation.ts +++ b/typescript/packages/fi-core/src/attachments/correlation.ts @@ -61,6 +61,42 @@ export class AttachmentCorrelation { } references(): readonly AttachmentReference[] { return this.refs.map(ref => Object.freeze({ ...ref })); } } +/** @internal The manager side of a deferred target: told once when the target binds or is cancelled. */ +export interface AttachmentDeferredHost { + deferredBound(target: AttachmentDeferredCorrelation, spanOrContext: AttachmentSpan): void; + deferredCancelled(target: AttachmentDeferredCorrelation): void; +} +/** + * A capture target whose span does not exist yet (late-bound wire spans). Captures admitted against it keep bounded + * bytes until `bind(span)`, where policy is evaluated again; on timeout, `cancel()` or manager shutdown they end + * `correlation_unavailable` and upload nothing. Create it with `manager.deferredCorrelation()`. + */ +export class AttachmentDeferredCorrelation extends AttachmentCorrelation { + /** @internal */ canonical?: AttachmentCorrelation; + /** @internal */ cancelled = false; + /** @internal */ + constructor(/** @internal */ readonly host: AttachmentDeferredHost | undefined, /** @internal */ readonly expiresAt: number) { super(); } + /** True while the target is unbound and still accepts captures (no timeout check; the manager checks that). */ + get pending(): boolean { return !this.cancelled && !this.snapshot(); } + /** Bind the emitted span. Pending captures are re-evaluated now and upload against it, or end with a reason. Never throws. */ + bind(spanOrContext: AttachmentSpan): boolean { + if (this.cancelled) return false; + const bound = super.bind(spanOrContext); + if (bound) { try { this.host?.deferredBound(this, spanOrContext); } catch { /* host failures never reach the caller */ } } + return bound; + } + /** No span will come: pending captures end `correlation_unavailable`, and later captures are refused. Never throws. */ + cancel(): void { + if (this.cancelled || this.snapshot()) return; + this.cancelled = true; + try { this.host?.deferredCancelled(this); } catch { /* bounded */ } + } + prepare(sessionId: string, nonce: string): void { if (this.canonical) this.canonical.prepare(sessionId, nonce); else super.prepare(sessionId, nonce); } + project(reference: AttachmentReference, sessionId: string, nonce: string): void { + if (this.canonical) this.canonical.project(reference, sessionId, nonce); else super.project(reference, sessionId, nonce); + } + references(): readonly AttachmentReference[] { return this.canonical ? this.canonical.references() : super.references(); } +} export function attachmentMessageUri(reference: AttachmentReference): Readonly<{ type: "uri"; modality: AttachmentReference["modality"]; mime_type: string; uri: string }> { return Object.freeze({ type: "uri", modality: reference.modality, mime_type: reference.mime_type, uri: `fi-attachment:${reference.occurrence_id}` }); } diff --git a/typescript/packages/fi-core/src/attachments/disabled.ts b/typescript/packages/fi-core/src/attachments/disabled.ts index d49a804e3..10d294ea2 100644 --- a/typescript/packages/fi-core/src/attachments/disabled.ts +++ b/typescript/packages/fi-core/src/attachments/disabled.ts @@ -1,8 +1,8 @@ import { AttachmentTicket } from "./admission"; -import { AttachmentCorrelation } from "./correlation"; +import { AttachmentCorrelation, AttachmentDeferredCorrelation } from "./correlation"; import { AttachmentManager } from "./manager"; import type { - AttachmentAdmissionTicket, AttachmentCompletedRecordingRequest, AttachmentCompletion, AttachmentFlushResult, AttachmentHandle, AttachmentImportHandle, + AttachmentAdmissionTicket, AttachmentCompletedRecordingRequest, AttachmentCompletion, AttachmentDeferredCorrelationOptions, AttachmentFlushResult, AttachmentHandle, AttachmentImportHandle, AttachmentImportOptions, AttachmentManagerOptions, AttachmentOptions, AttachmentOwnedFileRef, AttachmentOwnedRecordingHandle, AttachmentOwnedRecordingOptions, AttachmentPreflightOptions, AttachmentRecordingOptions, AttachmentRecordingWriter, AttachmentSource, AttachmentSpan, AttachmentStatus, AttachmentStream, AttachmentStreamOptions, AttachmentTarget, @@ -31,6 +31,8 @@ export class DisabledAttachmentManager extends AttachmentManager { constructor(_options: AttachmentManagerOptions = {}) { super({ enabled: false, environment: () => ({}) }); } async ready(): Promise { return false; } correlation(spanOrContext?: AttachmentSpan): AttachmentCorrelation { return new AttachmentCorrelation(spanOrContext); } + /** Inert: no host, so binding it schedules nothing; every capture against it is `disabled`. */ + deferredCorrelation(_options?: AttachmentDeferredCorrelationOptions): AttachmentDeferredCorrelation { return new AttachmentDeferredCorrelation(undefined, 0); } preflight(_target: AttachmentTarget, _options: AttachmentPreflightOptions): AttachmentAdmissionTicket { return new AttachmentTicket({ commitTicket: () => HANDLE, commitStream: () => STREAM, releaseReservation: () => undefined, deniedHandle: () => HANDLE, deniedStream: () => STREAM }, "disabled", 0, undefined, undefined, -1, 0); diff --git a/typescript/packages/fi-core/src/attachments/index.ts b/typescript/packages/fi-core/src/attachments/index.ts index d52ec8903..cb4adbcba 100644 --- a/typescript/packages/fi-core/src/attachments/index.ts +++ b/typescript/packages/fi-core/src/attachments/index.ts @@ -8,7 +8,7 @@ export { AttachmentRecording, attachmentBoundaryStep, attachmentSegmentFrames, a export { AttachmentUnavailableResolver } from "./imports"; export { validAttachmentWire, ATTACHMENT_WIRE_SHAPES } from "./wire"; export { ATTACHMENT_WARNING_LEGACY_TRANSPORT, ATTACHMENT_WARNING_WIRE_REJECTED } from "./warn"; -export { AttachmentCorrelation, canonicalAttachmentTraceId, attachmentTraceUuid, attachmentMessageUri } from "./correlation"; +export { AttachmentCorrelation, AttachmentDeferredCorrelation, canonicalAttachmentTraceId, attachmentTraceUuid, attachmentMessageUri } from "./correlation"; export { evaluateAttachmentPolicy, attachmentModality } from "./policy"; export { createAttachmentHttpTransport, createAttachmentHttpTransportV1, attachmentFetchClient, AttachmentTransportError, attachmentHttpError, diff --git a/typescript/packages/fi-core/src/attachments/manager.ts b/typescript/packages/fi-core/src/attachments/manager.ts index b0f1359f8..152471963 100644 --- a/typescript/packages/fi-core/src/attachments/manager.ts +++ b/typescript/packages/fi-core/src/attachments/manager.ts @@ -2,7 +2,7 @@ import { context, trace } from "@opentelemetry/api"; import { isTracingSuppressed } from "@opentelemetry/core"; import { v4 as uuid } from "uuid"; import { AttachmentTicket, type TicketHost } from "./admission"; -import { AttachmentCorrelation, attachmentTraceUuid, validAttachmentUuid } from "./correlation"; +import { AttachmentCorrelation, AttachmentDeferredCorrelation, attachmentTraceUuid, validAttachmentUuid, type AttachmentDeferredHost } from "./correlation"; import { attachmentImportDirection, attachmentResolvedRecordingDenial, validAttachmentImportRequest } from "./imports"; import type { AdmissionFacts, AttachmentHost, CoreJob, UploadOutcome, UploadSpec } from "./internal"; import { validAttachmentWire } from "./wire"; @@ -17,7 +17,7 @@ import { AttachmentTransportError, attachmentIdempotencyKey, attachmentIsLegacyW import { attachmentWireRejected } from "./warn"; import type { AttachmentAdmission, AttachmentAdmissionTicket, AttachmentClock, AttachmentCompletedRecordingRequest, AttachmentCompletion, AttachmentContentSource, - AttachmentFields, AttachmentFlushResult, AttachmentHandle, AttachmentImportHandle, AttachmentImportOptions, AttachmentManagerOptions, AttachmentOptions, + AttachmentDeferredCorrelationOptions, AttachmentFields, AttachmentFlushResult, AttachmentHandle, AttachmentImportHandle, AttachmentImportOptions, AttachmentManagerOptions, AttachmentOptions, AttachmentOwnedFileRef, AttachmentOwnedRecordingHandle, AttachmentOwnedRecordingOptions, AttachmentPacer, AttachmentPreflightOptions, AttachmentRecordingOptions, AttachmentRecordingWriter, AttachmentReference, AttachmentRuntimeState, AttachmentSession, AttachmentSource, AttachmentSourceKind, AttachmentSpan, AttachmentStatus, AttachmentStream, AttachmentStreamOptions, AttachmentTarget, AttachmentTransportV1, @@ -90,11 +90,20 @@ class TokenBucket implements AttachmentPacer { interface Finalizer { register(target: object, held: () => void, token?: object): void; unregister(token: object): boolean } type FinalizerConstructor = new (cleanup: (held: () => void) => void) => Finalizer; class Lane { readonly jobs: CoreJob[] = []; busy = false; constructor(readonly id: string) {} } -interface Reservation { bytes: number; expiresAt: number } +/** `deferred`: the ticket was admitted against an unbound deferred target, so its bytes count toward the pre-bind cap. */ +interface Reservation { bytes: number; expiresAt: number; deferred?: AttachmentDeferredCorrelation } +/** A capture admitted and committed against a deferred target that has not bound yet: bounded bytes held, nothing sent. */ +interface DeferredCapture { + readonly target: AttachmentDeferredCorrelation; + readonly facts: AdmissionFacts; + readonly bytes: number; + start(bound: AttachmentCorrelation): void; + fail(reason: AttachmentAdmission): void; +} interface StreamExtra { facts: AdmissionFacts; fields: AttachmentFields; correlation: AttachmentCorrelation; purpose?: AttachmentOptions["purpose"] } /** One per provider/runtime; explicit attachment APIs and adapters use this same instance. */ -export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost { +export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost, AttachmentDeferredHost { /** @internal */ readonly clock: AttachmentClock; /** @internal */ readonly sessionId = uuid(); /** @internal */ readonly nonce = uuid(); @@ -121,6 +130,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost private readonly reservations = new Map(); private nextReservation = 0; private readonly streams = new Set(); + private readonly deferredCaptures = new Set(); private readonly recordings = new Map(); private recordingSlots = 0; private bytes = 0; @@ -213,6 +223,17 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost if (!result) { result = new AttachmentCorrelation(spanOrContext); this.correlations.set(spanOrContext, result); } return result; } + /** + * A target for a span that does not exist yet (for example a wire span emitted only after the body completes). + * `preflight` / `attach` / `startAttachmentStream` admit Bytes, Blob and consumed-stream captures against it with the + * full deny-wins lattice; committed bytes stay in the owned budget (at most half of it pre-bind) until + * `target.bind(span)`, where policy and sampling are evaluated again before anything is sent. Timeout (`bindTimeoutMs`), + * `target.cancel()` and manager shutdown end pending captures `correlation_unavailable`, uploading nothing. + */ + deferredCorrelation(options: AttachmentDeferredCorrelationOptions = {}): AttachmentDeferredCorrelation { + if (this.enabled) this.enter(); + return new AttachmentDeferredCorrelation(this.enabled ? this : undefined, this.clock.now() + bounded(options?.bindTimeoutMs, 60000, 300000)); + } // --- state and admission ------------------------------------------------------------------------ /** @internal */ @@ -253,6 +274,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost } private resolveCorrelation(target: AttachmentTarget): AttachmentCorrelation | undefined { try { + if (target instanceof AttachmentDeferredCorrelation) return target.canonical ?? target; if (target instanceof AttachmentCorrelation) return target; if (target === undefined) { // The current span is read ONCE at call time. @@ -262,7 +284,23 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost return this.correlation(target); } catch { return undefined; } } - private descriptors(): number { return this.queued + this.active.size + this.reservations.size + this.streams.size + this.recordings.size; } + private descriptors(): number { return this.queued + this.active.size + this.reservations.size + this.streams.size + this.recordings.size + this.deferredCaptures.size; } + /** Owned by this manager, unbound, not cancelled and within its bind deadline. */ + private deferredOpen(target: AttachmentDeferredCorrelation): boolean { + return target.host === this && target.pending && this.clock.now() < target.expiresAt; + } + private static pendingTarget(correlation: AttachmentCorrelation | undefined): AttachmentDeferredCorrelation | undefined { + return correlation instanceof AttachmentDeferredCorrelation && correlation.pending ? correlation : undefined; + } + /** Bytes held for targets that have not bound yet. Capped at half the owned budget so bound captures keep headroom. */ + private deferredBytes(): number { + let total = 0; + for (const reservation of this.reservations.values()) if (reservation.deferred?.pending) total += reservation.bytes; + for (const state of this.streams) if (AttachmentManager.pendingTarget(state.correlation)) total += state.reserved; + for (const capture of this.deferredCaptures) total += capture.bytes; + return total; + } + private deferredCap(): number { return Math.floor(this.maxBytes / 2); } private uploadCap(mimeType: string, sourceKind: AttachmentSourceKind): number { const limit = Math.min(this.session?.limits.max_upload_bytes ?? 0, 50 * MiB); const kind = attachmentModality(mimeType) === "image" ? Math.min(limit, 20 * MiB) : limit; @@ -303,6 +341,13 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost const now = this.clock.now(); for (const [id, reservation] of this.reservations) if (reservation.expiresAt <= now) this.releaseReservation(id); for (const state of this.streams) if (state.open && now - state.lastWrite > state.idleTtl) this.streamAbort(state, "incomplete"); + // Deferred targets past their bind deadline: fail closed, discard retained bytes (no timer; design 1.5 style). + for (const [id, reservation] of this.reservations) if (reservation.deferred?.pending && now >= reservation.deferred.expiresAt) this.releaseReservation(id); + for (const state of [...this.streams]) { + const target = AttachmentManager.pendingTarget(state.correlation); + if (state.open && target && now >= target.expiresAt) this.streamAbort(state, "correlation_unavailable"); + } + for (const capture of [...this.deferredCaptures]) if (now >= capture.target.expiresAt) capture.fail("correlation_unavailable"); } private maybeRenew(): void { const session = this.session; @@ -344,7 +389,8 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost capture: options.capture ? { ...options.capture } : undefined, operation: options.operation ?? "observe_consumed", sourceKind: options.sourceKind ?? "Bytes", importSource }; } - private admitTarget(target: AttachmentTarget, facts: AdmissionFacts): { reason?: AttachmentAdmission; correlation?: AttachmentCorrelation; identity?: { traceId: string; spanId: string } } { + private admitTarget(target: AttachmentTarget, facts: AdmissionFacts, allowDeferred = false): + { reason?: AttachmentAdmission; correlation?: AttachmentCorrelation; identity?: { traceId: string; spanId: string }; deferred?: AttachmentDeferredCorrelation } { if (typeof facts.mimeType !== "string" || !["input", "output", "mixed"].includes(facts.direction) || !["observe_consumed", "import_existing", "create_recording"].includes(facts.operation)) { const first = this.recheck(facts, { admission: true }); @@ -353,6 +399,10 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost const reason = this.recheck(facts, { admission: true }); if (reason) return { reason }; const correlation = this.resolveCorrelation(target); + // An unbound deferred target is admitted on the lattice alone; sampling and policy are evaluated again at bind. + if (correlation instanceof AttachmentDeferredCorrelation && !correlation.snapshot()) { + return allowDeferred && this.deferredOpen(correlation) ? { correlation, deferred: correlation } : { reason: "correlation_unavailable" }; + } const identity = correlation?.snapshot(); if (!correlation || !identity) return { reason: "correlation_unavailable" }; if (!identity.sampled) return { reason: "unsampled" }; @@ -375,15 +425,16 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost try { this.enter(); const facts = this.preflightFacts(options ?? ({} as AttachmentPreflightOptions)); - const { reason, correlation } = this.admitTarget(target, facts); + const { reason, correlation, deferred } = this.admitTarget(target, facts, facts.sourceKind === "Bytes" || facts.sourceKind === "Blob"); if (reason) return denied(reason); const maxBytes = options.maxBytes; if (!Number.isInteger(maxBytes) || maxBytes <= 0 || maxBytes > this.uploadCap(facts.mimeType, facts.sourceKind)) return denied("too_large"); const reserve = facts.sourceKind === "OwnedRecordingFile" ? CHUNK : maxBytes; if (this.descriptors() >= this.maxDescriptors || this.bytes + reserve > this.maxBytes) return denied("queue_full"); + if (deferred && this.deferredBytes() + reserve > this.deferredCap()) return denied("queue_full"); const ttl = bounded(options.ttlMs, this.ticketTtl, 30000); const id = ++this.nextReservation; const expiresAt = this.clock.now() + ttl; - this.bytes += reserve; this.reservations.set(id, { bytes: reserve, expiresAt }); + this.bytes += reserve; this.reservations.set(id, { bytes: reserve, expiresAt, ...(deferred ? { deferred } : {}) }); const ticket = new AttachmentTicket(this, "queued", maxBytes, correlation, facts, id, expiresAt); this.finalizer()?.register(ticket, () => this.releaseReservation(id), ticket); return ticket; @@ -399,6 +450,10 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost ticket.used = true; try { this.registry?.unregister(ticket); } catch { /* bounded */ } this.sweep(); + const target = ticket.correlation; + if (target instanceof AttachmentDeferredCorrelation && !target.snapshot() && !this.deferredOpen(target)) { + this.releaseReservation(ticket.reservation); return { reason: "correlation_unavailable" }; + } const reservation = this.reservations.get(ticket.reservation); if (!reservation || this.clock.now() >= ticket.expiresAt) { this.releaseReservation(ticket.reservation); return { reason: "expired" }; } // Commit re-check against the CURRENT policy epoch; tightening denies without reading the source. @@ -469,9 +524,33 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost ...(fields.turnId ? { turn_id: fields.turnId } : {}), ...(fields.toolCallId ? { tool_call_id: fields.toolCallId } : {}) }; } + /** The correlation to upload against now, or undefined while a deferred target is unbound. */ + private boundTarget(correlation: AttachmentCorrelation): AttachmentCorrelation | undefined { + if (!(correlation instanceof AttachmentDeferredCorrelation)) return correlation; + return correlation.snapshot() ? correlation.canonical ?? correlation : undefined; + } + /** Holds a committed capture (its bytes stay reserved) until its deferred target binds, fails or times out. */ + private park(target: AttachmentDeferredCorrelation, facts: AdmissionFacts, bytes: number, result: HandleControl, + start: (bound: AttachmentCorrelation) => void, discard: () => void): void { + const capture: DeferredCapture = { target, facts, bytes, + start: bound => { if (this.deferredCaptures.delete(capture)) start(bound); }, + fail: reason => { + if (!this.deferredCaptures.delete(capture)) return; + discard(); this.releaseBytes(bytes); this.dropped++; result.finish(reason); this.notify(); + } }; + this.deferredCaptures.add(capture); + if (!this.deferredOpen(target)) capture.fail("correlation_unavailable"); + } private enqueueUpload(facts: AdmissionFacts, fields: AttachmentFields, correlation: AttachmentCorrelation, size: number, - content: AttachmentContentSource | undefined, bytes: number, owned?: AttachmentOwnedFileRecord, project = true): AttachmentHandle { - const result = handle("queued"); + content: AttachmentContentSource | undefined, bytes: number, owned?: AttachmentOwnedFileRecord, project = true, result: HandleControl = handle("queued")): AttachmentHandle { + const bound = this.boundTarget(correlation); + if (!bound) { + // Retained, bounded and not sent: the request (parent, policy version) is built only once the span is known. + this.park(correlation as AttachmentDeferredCorrelation, facts, bytes, result, + target => { this.enqueueUpload(facts, fields, target, size, content, bytes, owned, project, result); }, () => { content = undefined; }); + return result.value; + } + correlation = bound; const identity = correlation.snapshot()!; const request = this.uploadRequest(facts, fields, identity, result.value, size); const body = JSON.stringify(request); @@ -510,7 +589,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost const kind = source.kind; const facts: AdmissionFacts = { mimeType: options.mimeType, direction: options.direction, purpose: options.purpose, capture: options.capture ? { ...options.capture } : undefined, operation: options.capture?.newRecording ? "create_recording" : "observe_consumed", sourceKind: kind }; - const { reason, correlation } = this.admitTarget(spanOrContext, facts); + const { reason, correlation, deferred } = this.admitTarget(spanOrContext, facts, kind === "Bytes" || kind === "Blob"); if (reason) return handle(reason).value; if (!this.validOptions(options)) return handle("policy_denied").value; if (this.descriptors() >= this.maxDescriptors) return handle("queue_full").value; @@ -518,6 +597,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost if ("reason" in read) return handle(read.reason).value; const reserve = kind === "OwnedRecordingFile" ? CHUNK : read.size; if (this.descriptors() >= this.maxDescriptors || this.bytes + reserve > this.maxBytes) return handle("queue_full").value; + if (deferred && this.deferredBytes() + reserve > this.deferredCap()) return handle("queue_full").value; this.bytes += reserve; const fields: AttachmentFields = { messageIndex: options.messageIndex, partIndex: options.partIndex, role: options.role, channel: options.channel, conversationId: options.conversationId, turnId: options.turnId, toolCallId: options.toolCallId }; @@ -528,9 +608,13 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost // --- consumed streams ------------------------------------------------------------------------------ startAttachmentStream(target: AttachmentTarget, options: AttachmentStreamOptions): AttachmentStream { try { - const ticket = this.preflight(target, { mimeType: options.mimeType, direction: options.direction, maxBytes: options.maxBytes, + // The reservation is min(requested, negotiated single-upload cap) (design 0.7): a server whose max_upload_bytes is + // below the adapter default still admits the stream, and a body over the effective cap is discarded whole (too_large). + let maxBytes = options.maxBytes; + if (this.session && Number.isInteger(maxBytes) && maxBytes > 0) maxBytes = Math.min(maxBytes, this.uploadCap(options.mimeType, "Bytes")); + const ticket = this.preflight(target, { mimeType: options.mimeType, direction: options.direction, maxBytes, purpose: options.purpose, capture: options.capture, operation: "observe_consumed", sourceKind: "Bytes" }); - return ticket.startStream(options); + return ticket.startStream(maxBytes === options.maxBytes ? options : { ...options, maxBytes }); } catch { return this.deniedStream("source_unavailable"); } } /** @internal */ @@ -585,14 +669,23 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost s.handle = this.enqueueUploadWithIds(s, content, bytes); return s.handle; } - private enqueueUploadWithIds(s: StreamState & StreamExtra, content: AttachmentContentSource, bytes: number): AttachmentHandle { - const result = handle("queued"); + private enqueueUploadWithIds(s: StreamState & StreamExtra, content: AttachmentContentSource, bytes: number, + result: HandleControl = handle("queued"), parked?: AttachmentHandle): AttachmentHandle { const ids = { attachmentId: s.attachmentId!, occurrenceId: s.occurrenceId! }; + const bound = this.boundTarget(s.correlation); + if (!bound) { + const value = Object.freeze({ attachmentId: ids.attachmentId, occurrenceId: ids.occurrenceId, admission: "queued" as AttachmentAdmission, completion: result.value.completion }); + let data: AttachmentContentSource | undefined = content; + this.park(s.correlation as AttachmentDeferredCorrelation, s.facts, bytes, result, + () => { if (data) this.enqueueUploadWithIds(s, data, bytes, result, value); }, () => { data = undefined; }); + return value; + } + s.correlation = bound; const identity = s.correlation.snapshot()!; const request = this.uploadRequest(s.facts, s.fields, identity, ids, bytes); const body = JSON.stringify(request); let data: AttachmentContentSource | undefined = content; - const value = Object.freeze({ attachmentId: ids.attachmentId, occurrenceId: ids.occurrenceId, admission: "queued" as AttachmentAdmission, completion: result.value.completion }); + const value = parked ?? Object.freeze({ attachmentId: ids.attachmentId, occurrenceId: ids.occurrenceId, admission: "queued" as AttachmentAdmission, completion: result.value.completion }); this.enqueue({ lane: ids.occurrenceId, bytes, controller: new AbortController(), denied: () => this.recheck(s.facts), fail: reason => { data = undefined; result.finish(reason); }, run: async signal => { @@ -607,7 +700,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost return value; } /** @internal */ - streamAbort(state: StreamState, reason: "incomplete" | "policy_denied" | "too_large"): void { + streamAbort(state: StreamState, reason: "incomplete" | "policy_denied" | "too_large" | "correlation_unavailable"): void { if (!state.open) return; state.open = false; this.streams.delete(state as StreamState & StreamExtra); try { this.registry?.unregister(state); } catch { /* bounded */ } @@ -1078,6 +1171,40 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost } private drop(job: CoreJob, reason: AttachmentCompletion["status"]): void { this.dropped++; job.fail(reason); } + // --- deferred targets ------------------------------------------------------------------------------------ + /** @internal Called once by `AttachmentDeferredCorrelation.bind`. Synchronous; the transport runs only from the queue. */ + deferredBound(target: AttachmentDeferredCorrelation, spanOrContext: AttachmentSpan): void { + if (target.host !== this) return; + // One reference list per span: a span that already has a correlation keeps it, and the target forwards to it. + try { + const existing = this.correlations.get(spanOrContext); + if (existing && existing !== target) target.canonical = existing; + else this.correlations.set(spanOrContext, target); + } catch { /* identity is still bound; projection falls back to the target itself */ } + this.sweep(); // a capture past its deadline fails closed before anything else + const captures = [...this.deferredCaptures].filter(capture => capture.target === target); + if (!captures.length) return; + const bound = target.canonical ?? target; + const identity = bound.snapshot(); + for (const capture of captures) { + if (this.closed || this.stoppingFlag || this.shutdownPromise || this.clock.now() >= target.expiresAt) { capture.fail("correlation_unavailable"); continue; } + // Policy is evaluated again at bind against the CURRENT epoch, environment and session. + const reason = this.recheck(capture.facts); + if (reason) { capture.fail(reason); continue; } + if (!identity?.sampled) { capture.fail("unsampled"); continue; } + capture.start(bound); + } + this.notify(); + } + /** @internal Called once by `AttachmentDeferredCorrelation.cancel`. */ + deferredCancelled(target: AttachmentDeferredCorrelation): void { + if (target.host !== this) return; + for (const capture of [...this.deferredCaptures]) if (capture.target === target) capture.fail("correlation_unavailable"); + for (const state of [...this.streams]) if (state.correlation === target) this.streamAbort(state, "correlation_unavailable"); + for (const [id, reservation] of this.reservations) if (reservation.deferred === target) this.releaseReservation(id); + this.notify(); + } + // --- policy and lifecycle ------------------------------------------------------------------------------ /** Tightening discards all queued data and cancels transfers; re-enable never resurrects it or turns a disabled manager on. */ updatePolicy(update: Pick): void { @@ -1108,6 +1235,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost if (reason) { job.controller.abort(); this.drop(job, reason); } } for (const state of [...this.streams]) if (this.recheck(state.facts)) this.streamAbort(state, "policy_denied"); + for (const capture of [...this.deferredCaptures]) { const reason = this.recheck(capture.facts); if (reason) capture.fail(reason); } this.notify(); } private deadline(promise: Promise, timeout: number, fallback: T): Promise { @@ -1129,7 +1257,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost this.listeners.add(changed); changed(); }); } - return Object.freeze({ unfinished: this.queued + this.active.size + this.streams.size + this.recordings.size, dropped: this.dropped }); + return Object.freeze({ unfinished: this.queued + this.active.size + this.streams.size + this.recordings.size + this.deferredCaptures.size, dropped: this.dropped }); } shutdown(timeoutMs = this.shutdownTimeout): Promise { if (!this.shutdownPromise) { @@ -1142,9 +1270,13 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost const writers = [...this.recordings.values()]; if (writers.length) await this.deadline(Promise.all(writers.map(writer => writer.shutdown(Math.max(1, deadline - this.clock.now())))), Math.max(1, deadline - this.clock.now()), []); for (const state of [...this.streams]) this.streamAbort(state, "incomplete"); + // Captures still waiting for a span never upload after close. + for (const capture of [...this.deferredCaptures]) capture.fail("correlation_unavailable"); for (const id of [...this.reservations.keys()]) this.releaseReservation(id); const result = await this.flush(Math.max(1, Math.floor(deadline - this.clock.now()))); - this.closed = true; this.closedReason = "incomplete"; this.bootController?.abort(); + // An earlier invalidate() keeps its reason (policy_denied / deleted / expired / scope_unresolved). + if (!this.closed) this.closedReason = "incomplete"; + this.closed = true; this.bootController?.abort(); for (const lane of [...this.order]) { for (const job of lane.jobs.splice(0)) { this.queued--; this.endJob(job); this.drop(job, "incomplete"); } this.dropLane(lane); diff --git a/typescript/packages/fi-core/src/attachments/stream.ts b/typescript/packages/fi-core/src/attachments/stream.ts index 733c52f05..051d3f061 100644 --- a/typescript/packages/fi-core/src/attachments/stream.ts +++ b/typescript/packages/fi-core/src/attachments/stream.ts @@ -20,7 +20,7 @@ export interface StreamState { export interface StreamHost { streamWrite(state: StreamState, chunk: Uint8Array): boolean; streamFinish(state: StreamState): AttachmentHandle; - streamAbort(state: StreamState, reason: "incomplete" | "policy_denied" | "too_large"): void; + streamAbort(state: StreamState, reason: "incomplete" | "policy_denied" | "too_large" | "correlation_unavailable"): void; } /** * Single-object upload of chunks the application ALREADY consumed (design section 6). The core never pulls, clones diff --git a/typescript/packages/fi-core/src/attachments/types.ts b/typescript/packages/fi-core/src/attachments/types.ts index 3cdadf77a..97e07dd0d 100644 --- a/typescript/packages/fi-core/src/attachments/types.ts +++ b/typescript/packages/fi-core/src/attachments/types.ts @@ -209,6 +209,11 @@ export interface AttachmentPreflightOptions { capture?: AttachmentCapturePolicy; ttlMs?: number; } +/** `manager.deferredCorrelation(options)`: how long pre-bind captures may wait for their span. */ +export interface AttachmentDeferredCorrelationOptions { + /** Default 60 000 ms, max 300 000 ms. Enforced lazily at manager entry points and at bind; no timer is armed. */ + bindTimeoutMs?: number; +} export interface AttachmentFields { messageIndex?: number; partIndex?: number; role?: AttachmentRole; channel?: number; conversationId?: string; turnId?: string; toolCallId?: string; diff --git a/typescript/packages/fi-core/test/attachments-deferred.test.cjs b/typescript/packages/fi-core/test/attachments-deferred.test.cjs new file mode 100644 index 000000000..b2fc22c7e --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-deferred.test.cjs @@ -0,0 +1,286 @@ +/* + * Deferred-target preflight (late-bound wire spans), the consumed-stream reservation clamp and the shutdown reason. + * Synthetic data and in-memory doubles only (sdk-streaming-design sections 1, 6 and 8; rows T-DT, T-SR, T-SD). + */ +const { test, after } = require('node:test'); +const assert = require('node:assert/strict'); +const { core, managerV1, span, clock } = require('./attachments-v1.cjs'); +const { identity } = require('./attachments-fixtures.cjs'); +const { AttachmentManager, AttachmentDeferredCorrelation, AttachmentCorrelation } = core; +const live = setInterval(() => {}, 1000); // Explicit test owner, not an SDK keepalive. +after(() => clearInterval(live)); +const tick = () => new Promise(resolve => setImmediate(resolve)); +const MiB = 1024 * 1024; +const wav = { mimeType: 'audio/wav', direction: 'input', purpose: 'model_input' }; +const mp3 = { mimeType: 'audio/mpeg', direction: 'output', purpose: 'model_output' }; +const ctx2 = { traceId: 'fedcba9876543210fedcba9876543210', spanId: 'fedcba9876543210', traceFlags: 1 }; +const dashed = hex => `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +/** A Blob whose reads are counted: deferral must never read it before bind. */ +function spyBlob(bytes, type = 'audio/wav') { + const blob = new Blob([bytes], { type }); let reads = 0; + for (const name of ['arrayBuffer', 'stream', 'text', 'bytes']) { + const original = blob[name]; if (typeof original !== 'function') continue; + blob[name] = function (...args) { reads++; return original.apply(this, args); }; + } + const slice = blob.slice.bind(blob); blob.slice = (...args) => { reads++; return slice(...args); }; + return { blob, reads: () => reads }; +} + +// ---- T-DT deferred target --------------------------------------------------------------------------------------- +test('T-DT-01 Blob admitted before the span exists: nothing sent or read until bind; then uploaded with the bound parent and projected', async () => { + const f = await managerV1(); + const target = f.manager.deferredCorrelation(); + assert.ok(target instanceof AttachmentDeferredCorrelation); assert.ok(target instanceof AttachmentCorrelation); assert.equal(target.pending, true); + const { blob, reads } = spyBlob(new Uint8Array([1, 2, 3, 4, 5])); + const ticket = f.manager.preflight(target, { ...wav, maxBytes: blob.size, sourceKind: 'Blob' }); + assert.equal(ticket.reason, 'queued'); + const h = ticket.attach({ kind: 'Blob', data: blob }, { messageIndex: 0, partIndex: 1 }); + assert.equal(h.admission, 'queued'); + await f.manager.flush(50); await tick(); + assert.deepEqual(f.ops().filter(op => op !== 'bootstrap'), [], 'no request before bind'); + assert.equal(reads(), 0, 'the Blob is retained, never read, before bind'); + assert.equal(f.manager.status().ownedBytes, 5); + const s = span(ctx2); + assert.equal(target.bind(s), true); assert.equal(target.pending, false); + assert.equal((await h.completion).status, 'validating'); + assert.deepEqual([...f.contents.get(h.occurrenceId)], [1, 2, 3, 4, 5]); + const request = f.requests.find(r => r.occurrence_id === h.occurrenceId); + assert.equal(request.association.parent.trace_id, dashed(ctx2.traceId)); assert.equal(request.association.parent.span_id, ctx2.spanId); + const refs = JSON.parse(s.attrs['fi.attachment.refs']); + assert.equal(refs.length, 1); assert.equal(refs[0].occurrence_id, h.occurrenceId); assert.equal(refs[0].part_index, 1); + assert.equal(s.attrs['fi.attachment.schema_version'], 1); + assert.equal(f.manager.correlation(s), target, 'the bound target is the span\'s single correlation'); + await f.manager.flush(); assert.equal(f.manager.status().ownedBytes, 0); + await f.manager.shutdown(); +}); +test('T-DT-02 consumed stream finished before bind: bounded chunks held, uploaded at bind byte-identical', async () => { + const f = await managerV1(); + const target = f.manager.deferredCorrelation(); + const stream = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 64 }); + assert.equal(stream.reason, 'queued'); + const buffer = new Uint8Array([7, 8, 9]); assert.equal(stream.tryWrite(buffer), true); buffer.fill(0); assert.equal(stream.tryWrite(new Uint8Array([10])), true); + const h = stream.finish(); assert.equal(h.admission, 'queued'); assert.equal(h.occurrenceId, stream.occurrenceId); + await f.manager.flush(50); await tick(); + assert.equal(f.ops().includes('initialize'), false); + assert.equal(f.manager.status().ownedBytes, 4, 'the reservation shrank to the consumed length and is still held'); + assert.equal((await f.manager.flush(1)).unfinished, 1, 'a parked capture counts as unfinished; flush does not wait for it'); + const s = span(ctx2); target.bind(s); + assert.equal((await h.completion).status, 'validating'); + assert.deepEqual([...f.contents.get(h.occurrenceId)], [7, 8, 9, 10]); + assert.equal(JSON.parse(s.attrs['fi.attachment.refs'])[0].occurrence_id, h.occurrenceId); + await f.manager.shutdown(); +}); +test('T-DT-03 stream still open at bind finishes normally; an over-cap deferred stream uploads and projects nothing', async () => { + const f = await managerV1(); const target = f.manager.deferredCorrelation(); const s = span(ctx2); + const stream = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 4 }); + stream.tryWrite(new Uint8Array([1, 2])); target.bind(s); stream.tryWrite(new Uint8Array([3])); + const h = stream.finish(); assert.equal((await h.completion).status, 'validating'); assert.deepEqual([...f.contents.get(h.occurrenceId)], [1, 2, 3]); + const t2 = f.manager.deferredCorrelation(); const s2 = span(identity); + const over = f.manager.startAttachmentStream(t2, { ...mp3, maxBytes: 4 }); + over.tryWrite(new Uint8Array(3)); assert.equal(over.tryWrite(new Uint8Array(3)), false); assert.equal(over.reason, 'too_large'); + t2.bind(s2); assert.equal((await over.finish().completion).status, 'too_large'); + assert.equal(s2.attrs['fi.attachment.refs'], undefined); await f.manager.flush(); assert.equal(f.manager.status().ownedBytes, 0); + await f.manager.shutdown(); +}); +test('T-DT-04 bind timeout (fake clock): pending captures fail correlation_unavailable at the next entry point; no timer armed; nothing sent on a late bind', async () => { + const c = clock(); const f = await managerV1({}, { clock: c }); + const timers = []; const original = global.setTimeout; + global.setTimeout = (...args) => { timers.push(args[1]); return original(...args); }; + let target, h, stream, ticket; + try { + target = f.manager.deferredCorrelation({ bindTimeoutMs: 1000 }); + h = f.manager.preflight(target, { ...wav, maxBytes: 3 }).attach({ kind: 'Bytes', data: new Uint8Array([1, 2, 3]) }); + stream = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 64, idleTtlMs: 300000 }); stream.tryWrite(new Uint8Array(5)); + ticket = f.manager.preflight(target, { ...wav, maxBytes: 8, ttlMs: 30000 }); + } finally { global.setTimeout = original; } + assert.equal(timers.length, 0, 'deferral armed a timer'); + assert.equal(f.manager.status().ownedBytes, 3 + 64 + 8); + c.advance(1001); + const st = f.manager.status(); + assert.equal(st.ownedBytes, 0); assert.equal(st.openStreams, 0); assert.equal(st.openTickets, 0); + assert.equal((await h.completion).status, 'correlation_unavailable'); + assert.equal(stream.reason, 'correlation_unavailable'); + assert.equal(ticket.attach({ kind: 'Bytes', data: new Uint8Array(2) }).admission, 'correlation_unavailable'); + assert.equal(f.manager.preflight(target, { ...wav, maxBytes: 2 }).reason, 'correlation_unavailable', 'an expired unbound target admits nothing'); + const s = span(ctx2); target.bind(s); + await f.manager.flush(50); + assert.equal(f.ops().includes('initialize'), false); assert.equal(s.attrs['fi.attachment.refs'], undefined); + await f.manager.shutdown(1); +}); +test('T-DT-05 cancel(): pending captures and streams end correlation_unavailable, budget returned, later captures refused, bind refused', async () => { + const f = await managerV1(); const target = f.manager.deferredCorrelation(); + const h = f.manager.attach(target, { kind: 'Bytes', data: new Uint8Array([1]) }, wav); + const stream = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 16 }); stream.tryWrite(new Uint8Array(2)); + const ticket = f.manager.preflight(target, { ...wav, maxBytes: 4 }); + target.cancel(); + assert.equal((await h.completion).status, 'correlation_unavailable'); assert.equal(stream.reason, 'correlation_unavailable'); + assert.equal(f.manager.status().ownedBytes, 0); assert.equal(f.manager.status().openTickets, 0); + let read = false; assert.equal(ticket.attach({ kind: 'Bytes', get data() { read = true; return new Uint8Array(1); } }).admission, 'correlation_unavailable'); assert.equal(read, false); + assert.equal(f.manager.preflight(target, { ...wav, maxBytes: 1 }).reason, 'correlation_unavailable'); + assert.equal(target.bind(span(ctx2)), false); + await f.manager.flush(50); assert.equal(f.ops().includes('initialize'), false); + await f.manager.shutdown(); +}); +test('T-DT-06 manager shutdown before bind: correlation_unavailable, nothing uploaded, a later bind sends nothing', async () => { + const f = await managerV1(); const target = f.manager.deferredCorrelation(); + const h = f.manager.preflight(target, { ...wav, maxBytes: 2 }).attach({ kind: 'Bytes', data: new Uint8Array([1, 2]) }); + const s = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 8 }); s.tryWrite(new Uint8Array(1)); const sh = s.finish(); + const result = await f.manager.shutdown(200); + assert.equal((await h.completion).status, 'correlation_unavailable'); assert.equal((await sh.completion).status, 'correlation_unavailable'); + assert.equal(result.unfinished, 0); assert.equal(f.manager.status().ownedBytes, 0); + target.bind(span(ctx2)); await tick(); + assert.equal(f.ops().includes('initialize'), false); +}); +test('T-DT-07 policy is evaluated again at bind: environment kill switch, tightened trace config, invalidate', async () => { + // (a) A deny that arrives with no policy-epoch event (the environment is re-read): only the bind-time check sees it. + let env = {}; + const a = await managerV1({}, {}, { environment: () => env }); + const ta = a.manager.deferredCorrelation(); + const ha = a.manager.preflight(ta, { ...wav, maxBytes: 1 }).attach({ kind: 'Bytes', data: new Uint8Array([1]) }); + env = { FI_ATTACHMENTS_ENABLED: 'false' }; + const sa = span(ctx2); ta.bind(sa); + assert.equal((await ha.completion).status, 'disabled'); assert.equal(sa.attrs['fi.attachment.refs'], undefined); + await a.manager.flush(50); assert.equal(a.ops().includes('initialize'), false); assert.equal(a.manager.status().ownedBytes, 0); + // (b) Tightened trace config between commit and bind. + const b = await managerV1(); const tb = b.manager.deferredCorrelation(); + const hb = b.manager.startAttachmentStream(tb, { ...wav, maxBytes: 8 }); hb.tryWrite(new Uint8Array(1)); const hbh = hb.finish(); + b.manager.updatePolicy({ traceConfig: { hideInputs: true } }); + assert.equal((await hbh.completion).status, 'policy_denied'); tb.bind(span(ctx2)); await b.manager.flush(50); + assert.equal(b.ops().includes('initialize'), false); + // (c) Revocation. + const c = await managerV1(); const tc = c.manager.deferredCorrelation(); + const hc = c.manager.attach(tc, { kind: 'Bytes', data: new Uint8Array([1]) }, wav); + c.manager.invalidate('deleted'); assert.equal((await hc.completion).status, 'deleted'); + for (const f of [a, b, c]) await f.manager.shutdown(1); +}); +test('T-DT-08 unsampled or non-recording span at bind: unsampled, nothing uploaded', async () => { + const f = await managerV1(); const target = f.manager.deferredCorrelation(); + const h = f.manager.attach(target, { kind: 'Bytes', data: new Uint8Array([1]) }, wav); + target.bind({ ...ctx2, traceFlags: 0 }); + assert.equal((await h.completion).status, 'unsampled'); + const t2 = f.manager.deferredCorrelation(); const h2 = f.manager.attach(t2, { kind: 'Bytes', data: new Uint8Array([1]) }, wav); + const ended = span(ctx2); ended.end(); t2.bind(ended); + assert.equal((await h2.completion).status, 'unsampled'); + await f.manager.flush(50); assert.equal(f.ops().includes('initialize'), false); assert.equal(f.manager.status().ownedBytes, 0); + await f.manager.shutdown(); +}); +test('T-DT-09 pre-bind bytes are capped at half the owned budget; bound captures keep their headroom', async () => { + const f = await managerV1({}, {}, { maxOwnedBytes: 1000 }); + const target = f.manager.deferredCorrelation(); + const held = f.manager.preflight(target, { ...wav, maxBytes: 400 }); assert.equal(held.reason, 'queued'); + assert.equal(f.manager.preflight(target, { ...wav, maxBytes: 101 }).reason, 'queue_full', 'over the pre-bind cap'); + assert.equal(f.manager.startAttachmentStream(f.manager.deferredCorrelation(), { ...mp3, maxBytes: 101 }).reason, 'queue_full'); + assert.equal(f.manager.attach(target, { kind: 'Bytes', data: new Uint8Array(101) }, wav).admission, 'queue_full'); + assert.equal(f.manager.preflight(identity, { ...wav, maxBytes: 500 }).reason, 'queued', 'a bound capture still fits'); + held.release(); + assert.equal(f.manager.preflight(target, { ...wav, maxBytes: 100 }).reason, 'queued'); + await f.manager.shutdown(1); +}); +test('T-DT-10 one reference list per span: a span that already has a correlation keeps it and the target forwards to it', async () => { + const f = await managerV1(); const s = span(ctx2); + const first = f.manager.attach(s, { kind: 'Bytes', data: new Uint8Array([1]) }, { ...wav, partIndex: 0 }); + const target = f.manager.deferredCorrelation(); + const late = f.manager.attach(target, { kind: 'Bytes', data: new Uint8Array([2]) }, { ...wav, partIndex: 1 }); + target.bind(s); + await first.completion; assert.equal((await late.completion).status, 'validating'); + const refs = JSON.parse(s.attrs['fi.attachment.refs']); + assert.deepEqual(refs.map(r => r.occurrence_id).sort(), [first.occurrenceId, late.occurrenceId].sort()); + assert.equal(target.references().length, 2); + const after = f.manager.preflight(target, { ...wav, maxBytes: 1 }); assert.equal(after.reason, 'queued'); + after.attach({ kind: 'Bytes', data: new Uint8Array([3]) }, { partIndex: 2 }); + assert.equal(JSON.parse(s.attrs['fi.attachment.refs']).length, 3); + await f.manager.shutdown(); +}); +test('T-DT-11 scope: another manager\'s target, owned files, recordings and imports are not deferred (correlation_unavailable)', async () => { + const f = await managerV1(); const g = await managerV1(); + const foreign = g.manager.deferredCorrelation(); + assert.equal(f.manager.preflight(foreign, { ...wav, maxBytes: 1 }).reason, 'correlation_unavailable'); + const target = f.manager.deferredCorrelation(); + assert.equal(f.manager.preflight(target, { ...wav, maxBytes: 1, sourceKind: 'OwnedRecordingFile' }).reason, 'correlation_unavailable'); + assert.equal(f.manager.startRecording(target, { format: { sampleRateHz: 8000 }, layout: 'mixed', tracks: [{ channels: [0], direction: 'mixed', role: 'mixed' }], operation: 'observe_consumed' }).reason, 'correlation_unavailable'); + let resolved = false; + const imported = await f.manager.importCompletedRecording(target, { provider: 'livekit', providerRef: 'egress-1' }, { resolver: { resolve: async () => { resolved = true; } } }); + assert.equal(imported.reason, 'correlation_unavailable'); assert.equal(resolved, false); + assert.equal(f.manager.status().ownedBytes, 0); + await f.manager.shutdown(); await g.manager.shutdown(); +}); +test('T-DT-12 denials before admission never touch the source; a plain unbound correlation is still correlation_unavailable', async () => { + const f = await managerV1({}, {}, { traceConfig: { hideInputs: true } }); + const target = f.manager.deferredCorrelation(); let read = false; + const t = f.manager.preflight(target, { ...wav, maxBytes: 4 }); assert.equal(t.reason, 'policy_denied'); + assert.equal(t.attach({ kind: 'Bytes', get data() { read = true; return new Uint8Array(1); } }).admission, 'policy_denied'); assert.equal(read, false); + const g = await managerV1(); + assert.equal(g.manager.preflight(new AttachmentCorrelation(), { ...wav, maxBytes: 1 }).reason, 'correlation_unavailable'); + assert.equal(g.manager.preflight(undefined, { ...wav, maxBytes: 1 }).reason, 'correlation_unavailable'); + await f.manager.shutdown(1); await g.manager.shutdown(1); +}); +test('T-DT-13 disabled: inert deferred target, preflight disabled, bind schedules nothing, no timers/finalizers/listeners', async () => { + const saved = { st: global.setTimeout, fr: global.FinalizationRegistry }; let timers = 0; let registry = 0; + const listenersBefore = process.eventNames().map(n => process.listenerCount(n)).reduce((a, b) => a + b, 0); + try { + global.setTimeout = (...args) => { timers++; return saved.st(...args); }; + global.FinalizationRegistry = class extends saved.fr { constructor(...a) { super(...a); registry++; } }; + for (const m of [new AttachmentManager({ environment: () => ({}) }), new core.DisabledAttachmentManager()]) { + const target = m.deferredCorrelation({ bindTimeoutMs: 10 }); + assert.equal(m.preflight(target, { ...wav, maxBytes: 4 }).reason, 'disabled'); + assert.equal(m.startAttachmentStream(target, { ...mp3, maxBytes: 4 }).reason, 'disabled'); + const s = span(ctx2); assert.equal(target.bind(s), true); assert.equal(s.writes, 0); + target.cancel(); + } + } finally { global.setTimeout = saved.st; global.FinalizationRegistry = saved.fr; } + assert.equal(timers, 0); assert.equal(registry, 0); + assert.equal(process.eventNames().map(n => process.listenerCount(n)).reduce((a, b) => a + b, 0), listenersBefore); +}); +test('T-DT-14 bind is synchronous and never throws; the transport runs only from the queue', async () => { + let sync = true; let calledSync = false; + const f = await managerV1(base => ({ initialize: async (...args) => { if (sync) calledSync = true; return base.initialize(...args); } })); + const target = f.manager.deferredCorrelation(); + const h = f.manager.attach(target, { kind: 'Bytes', data: new Uint8Array([1]) }, wav); + const throwing = { spanContext: () => ctx2, isRecording: () => true, setAttribute() { throw Error('host'); } }; + assert.doesNotThrow(() => target.bind(throwing)); sync = false; + assert.equal(calledSync, false); assert.equal((await h.completion).status, 'validating'); + await f.manager.shutdown(); +}); + +// ---- T-SR consumed-stream reservation = min(requested, negotiated single-upload cap) --------------------------------- +test('T-SR-01 server max_upload_bytes below 4 MiB: a 4 MiB stream request is admitted at the server cap', async () => { + const f = await managerV1({}, { limits: { max_upload_bytes: 1 * MiB } }); + const stream = f.manager.startAttachmentStream(identity, { ...mp3, maxBytes: 4 * MiB }); + assert.equal(stream.reason, 'queued'); assert.equal(f.manager.status().ownedBytes, 1 * MiB, 'reservation = min(4 MiB, max_upload_bytes)'); + assert.equal(stream.tryWrite(new Uint8Array(512 * 1024)), true); + const h = stream.finish(); assert.equal((await h.completion).status, 'validating'); assert.equal(f.contents.get(h.occurrenceId).length, 512 * 1024); + const over = f.manager.startAttachmentStream(identity, { ...mp3, maxBytes: 4 * MiB }); + assert.equal(over.tryWrite(new Uint8Array(MiB)), true); assert.equal(over.tryWrite(new Uint8Array(1)), false); + assert.equal(over.reason, 'too_large', 'a body over the server cap is discarded whole'); + // The explicit ticket API keeps exact semantics: the caller asked for more than the cap. + assert.equal(f.manager.preflight(identity, { ...mp3, maxBytes: 4 * MiB }).reason, 'too_large'); + // A request below the cap is unchanged. + await f.manager.flush(); + const small = f.manager.startAttachmentStream(identity, { ...mp3, maxBytes: 1000 }); assert.equal(f.manager.status().ownedBytes, 1000); small.abort(); + await f.manager.shutdown(); +}); +test('T-SR-02 the clamp never admits what the lattice denies, and needs a session', async () => { + const pending = await managerV1({ bootstrap: () => new Promise(() => {}) }, { ready: false }); + assert.equal(pending.manager.startAttachmentStream(identity, { ...mp3, maxBytes: 4 * MiB }).reason, 'bootstrap_pending'); + const denied = await managerV1({}, { limits: { max_upload_bytes: MiB } }, { traceConfig: { hideOutputs: true } }); + assert.equal(denied.manager.startAttachmentStream(identity, { ...mp3, maxBytes: 4 * MiB }).reason, 'policy_denied'); + const bad = await managerV1({}, { limits: { max_upload_bytes: MiB } }); + assert.equal(bad.manager.startAttachmentStream(identity, { ...mp3, maxBytes: 0 }).reason, 'too_large'); + assert.equal(bad.manager.startAttachmentStream(identity, { ...mp3, maxBytes: 1.5 }).reason, 'too_large'); + for (const f of [pending, denied, bad]) await f.manager.shutdown(1); +}); + +// ---- T-SD shutdown keeps an earlier invalidate reason --------------------------------------------------------------- +test('T-SD-01 invalidate(policy_denied) then shutdown(): state and admissions stay policy_denied, not incomplete', async () => { + for (const reason of ['policy_denied', 'deleted', 'expired']) { + const f = await managerV1(); + f.manager.invalidate(reason); + assert.equal(f.manager.status().state, reason); + await f.manager.shutdown(100); + assert.equal(f.manager.status().state, reason, `shutdown overwrote ${reason}`); + assert.equal(f.manager.preflight(identity, { ...wav, maxBytes: 1 }).reason, reason); + assert.equal(f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array(1) }, wav).admission, 'incomplete', 'attach after shutdown is incomplete, as before'); + } + const plain = await managerV1(); await plain.manager.shutdown(100); + assert.equal(plain.manager.status().state, 'incomplete', 'a plain shutdown still reports incomplete'); +}); From 930876ea485df01e471ae716f7df8c4460ca53a4 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:05:11 -0800 Subject: [PATCH 9/9] fix(core): refuse unsampled and late binds for every deferred capture Bind-time sampling and the bind deadline now cover all captures on a deferred target, not only those already parked at bind: - deferredBound aborts streams still open on the target and releases its outstanding ticket reservations when the bound span is unsampled or not recording (`unsampled`), or when the bind is late (`correlation_unavailable`). - AttachmentDeferredCorrelation.bind asks the host whether the deadline has passed BEFORE binding the identity. A late bind cancels the target and returns false, so tickets and open streams can no longer upload after the deadline when no other manager entry point ran in between. - Backstops: ticket commit and stream finish re-check the bound span's sampled flag before anything is enqueued. Rows T-DT-15..18 cover a stream open at bind and a ticket committed after bind on an unsampled span, a late bind with no intervening entry point, and each backstop with the primary check bypassed. Co-Authored-By: Claude Opus 5.5 --- .../packages/fi-core/ATTACHMENTS-MIGRATION.md | 2 +- .../fi-core/src/attachments/correlation.ts | 12 ++- .../fi-core/src/attachments/manager.ts | 41 ++++++++-- .../fi-core/src/attachments/stream.ts | 2 +- .../test/attachments-deferred.test.cjs | 74 +++++++++++++++++++ 5 files changed, 123 insertions(+), 8 deletions(-) diff --git a/typescript/packages/fi-core/ATTACHMENTS-MIGRATION.md b/typescript/packages/fi-core/ATTACHMENTS-MIGRATION.md index 7d1f1f3d1..86db38831 100644 --- a/typescript/packages/fi-core/ATTACHMENTS-MIGRATION.md +++ b/typescript/packages/fi-core/ATTACHMENTS-MIGRATION.md @@ -19,6 +19,6 @@ Both warning codes are exported as `ATTACHMENT_WARNING_LEGACY_TRANSPORT` and `AT | Addition | Behaviour | |---|---| -| `manager.deferredCorrelation({ bindTimeoutMs })` returns an `AttachmentDeferredCorrelation` | A capture target for a span that does not exist yet, such as a wire span emitted only after the response body completes. `preflight`, `attach` and `startAttachmentStream` admit `Bytes`, `Blob` and consumed-stream captures against it using the full deny-wins check. Committed bytes stay in the owned budget, capped at half of it, and nothing is sent. `target.bind(span)` checks policy and sampling again, then uploads against that span and projects the references onto it. If the bind timeout passes (default 60 s, max 300 s, checked lazily without a timer), `target.cancel()` is called or the manager shuts down, pending captures end `correlation_unavailable` and upload nothing. Owned files, recordings and imports never defer. If a span already has a correlation, the bound target forwards to it, so the span keeps one reference list. | +| `manager.deferredCorrelation({ bindTimeoutMs })` returns an `AttachmentDeferredCorrelation` | A capture target for a span that does not exist yet, such as a wire span emitted only after the response body completes. `preflight`, `attach` and `startAttachmentStream` admit `Bytes`, `Blob` and consumed-stream captures against it using the full deny-wins check. Committed bytes stay in the owned budget, capped at half of it, and nothing is sent. `target.bind(span)` checks policy and sampling again, then uploads against that span and projects the references onto it. The sampling check covers every capture on the target: parked captures, streams still open at bind and tickets committed after bind all end `unsampled` on an unsampled or non-recording span. If the bind timeout passes (default 60 s, max 300 s, checked lazily without a timer), `target.cancel()` is called or the manager shuts down, pending captures end `correlation_unavailable` and upload nothing. A late `bind` (after the timeout) returns false, binds nothing and cancels the target, so open streams and outstanding tickets also end `correlation_unavailable`. Owned files, recordings and imports never defer. If a span already has a correlation, the bound target forwards to it, so the span keeps one reference list. | | `startAttachmentStream` reservation | The reservation is `min(maxBytes, negotiated single-upload cap)`. A server whose `max_upload_bytes` is below an adapter's default (for example 4 MiB) still admits the stream, and a body larger than the effective cap is discarded whole (`too_large`). `preflight` keeps its exact semantics: a `maxBytes` over the cap is `too_large`. | | `shutdown()` after `invalidate(reason)` | The state stays `reason` (`policy_denied`, `deleted`, `expired`); before this change, `shutdown()` overwrote it with `incomplete`. | diff --git a/typescript/packages/fi-core/src/attachments/correlation.ts b/typescript/packages/fi-core/src/attachments/correlation.ts index e341c196e..54be923a5 100644 --- a/typescript/packages/fi-core/src/attachments/correlation.ts +++ b/typescript/packages/fi-core/src/attachments/correlation.ts @@ -65,6 +65,8 @@ export class AttachmentCorrelation { export interface AttachmentDeferredHost { deferredBound(target: AttachmentDeferredCorrelation, spanOrContext: AttachmentSpan): void; deferredCancelled(target: AttachmentDeferredCorrelation): void; + /** True once the target's bind deadline has passed on the host's clock. Asked BEFORE the identity is bound. */ + deferredExpired?(target: AttachmentDeferredCorrelation): boolean; } /** * A capture target whose span does not exist yet (late-bound wire spans). Captures admitted against it keep bounded @@ -78,9 +80,17 @@ export class AttachmentDeferredCorrelation extends AttachmentCorrelation { constructor(/** @internal */ readonly host: AttachmentDeferredHost | undefined, /** @internal */ readonly expiresAt: number) { super(); } /** True while the target is unbound and still accepts captures (no timeout check; the manager checks that). */ get pending(): boolean { return !this.cancelled && !this.snapshot(); } - /** Bind the emitted span. Pending captures are re-evaluated now and upload against it, or end with a reason. Never throws. */ + /** + * Bind the emitted span. Pending captures are re-evaluated now and upload against it, or end with a reason. A bind + * after `bindTimeoutMs` binds nothing: the target is cancelled (captures end `correlation_unavailable`) and this + * returns false. Never throws. + */ bind(spanOrContext: AttachmentSpan): boolean { if (this.cancelled) return false; + // The deadline is enforced BEFORE the identity exists, so no ticket or open stream can outlive it through a late bind. + let expired = false; + try { expired = !this.snapshot() && this.host?.deferredExpired?.(this) === true; } catch { expired = true; } + if (expired) { this.cancel(); return false; } const bound = super.bind(spanOrContext); if (bound) { try { this.host?.deferredBound(this, spanOrContext); } catch { /* host failures never reach the caller */ } } return bound; diff --git a/typescript/packages/fi-core/src/attachments/manager.ts b/typescript/packages/fi-core/src/attachments/manager.ts index 152471963..0019c4eb2 100644 --- a/typescript/packages/fi-core/src/attachments/manager.ts +++ b/typescript/packages/fi-core/src/attachments/manager.ts @@ -289,6 +289,11 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost private deferredOpen(target: AttachmentDeferredCorrelation): boolean { return target.host === this && target.pending && this.clock.now() < target.expiresAt; } + /** The bound identity (the canonical correlation when the span already had one) is sampled and was recording at bind. */ + private static sampledTarget(correlation: AttachmentCorrelation): boolean { + const bound = correlation instanceof AttachmentDeferredCorrelation ? correlation.canonical ?? correlation : correlation; + return bound.snapshot()?.sampled === true; + } private static pendingTarget(correlation: AttachmentCorrelation | undefined): AttachmentDeferredCorrelation | undefined { return correlation instanceof AttachmentDeferredCorrelation && correlation.pending ? correlation : undefined; } @@ -454,6 +459,10 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost if (target instanceof AttachmentDeferredCorrelation && !target.snapshot() && !this.deferredOpen(target)) { this.releaseReservation(ticket.reservation); return { reason: "correlation_unavailable" }; } + // A ticket preflighted against a deferred target and committed after bind: the bound span must be sampled. + if (target instanceof AttachmentDeferredCorrelation && target.snapshot() && !AttachmentManager.sampledTarget(target)) { + this.releaseReservation(ticket.reservation); return { reason: "unsampled" }; + } const reservation = this.reservations.get(ticket.reservation); if (!reservation || this.clock.now() >= ticket.expiresAt) { this.releaseReservation(ticket.reservation); return { reason: "expired" }; } // Commit re-check against the CURRENT policy epoch; tightening denies without reading the source. @@ -550,6 +559,11 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost target => { this.enqueueUpload(facts, fields, target, size, content, bytes, owned, project, result); }, () => { content = undefined; }); return result.value; } + if (!AttachmentManager.sampledTarget(bound)) { + // Backstop for captures admitted against a deferred target: an unsampled bound span never receives media. + content = undefined; this.releaseBytes(bytes); this.dropped++; result.finish("unsampled"); this.notify(); + return result.value; + } correlation = bound; const identity = correlation.snapshot()!; const request = this.uploadRequest(facts, fields, identity, result.value, size); @@ -680,12 +694,17 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost () => { if (data) this.enqueueUploadWithIds(s, data, bytes, result, value); }, () => { data = undefined; }); return value; } + const value = parked ?? Object.freeze({ attachmentId: ids.attachmentId, occurrenceId: ids.occurrenceId, admission: "queued" as AttachmentAdmission, completion: result.value.completion }); + if (!AttachmentManager.sampledTarget(bound)) { + // Backstop: a stream opened against a deferred target that bound to an unsampled span uploads nothing. + this.releaseBytes(bytes); this.dropped++; result.finish("unsampled"); this.notify(); + return value; + } s.correlation = bound; const identity = s.correlation.snapshot()!; const request = this.uploadRequest(s.facts, s.fields, identity, ids, bytes); const body = JSON.stringify(request); let data: AttachmentContentSource | undefined = content; - const value = parked ?? Object.freeze({ attachmentId: ids.attachmentId, occurrenceId: ids.occurrenceId, admission: "queued" as AttachmentAdmission, completion: result.value.completion }); this.enqueue({ lane: ids.occurrenceId, bytes, controller: new AbortController(), denied: () => this.recheck(s.facts), fail: reason => { data = undefined; result.finish(reason); }, run: async signal => { @@ -700,7 +719,7 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost return value; } /** @internal */ - streamAbort(state: StreamState, reason: "incomplete" | "policy_denied" | "too_large" | "correlation_unavailable"): void { + streamAbort(state: StreamState, reason: "incomplete" | "policy_denied" | "too_large" | "correlation_unavailable" | "unsampled"): void { if (!state.open) return; state.open = false; this.streams.delete(state as StreamState & StreamExtra); try { this.registry?.unregister(state); } catch { /* bounded */ } @@ -1182,12 +1201,20 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost else this.correlations.set(spanOrContext, target); } catch { /* identity is still bound; projection falls back to the target itself */ } this.sweep(); // a capture past its deadline fails closed before anything else - const captures = [...this.deferredCaptures].filter(capture => capture.target === target); - if (!captures.length) return; const bound = target.canonical ?? target; const identity = bound.snapshot(); + const late = this.closed || this.stoppingFlag || !!this.shutdownPromise || this.clock.now() >= target.expiresAt; + // Open streams and outstanding tickets on this target are refused now, not only captures already parked: once the + // identity is bound they are no longer `pending`, so no later sweep would see them. + const refused = late ? "correlation_unavailable" : identity?.sampled ? undefined : "unsampled"; + if (refused) { + for (const state of [...this.streams]) if (state.correlation === target) this.streamAbort(state, refused); + for (const [id, reservation] of this.reservations) if (reservation.deferred === target) this.releaseReservation(id); + } + const captures = [...this.deferredCaptures].filter(capture => capture.target === target); + if (!captures.length) { if (refused) this.notify(); return; } for (const capture of captures) { - if (this.closed || this.stoppingFlag || this.shutdownPromise || this.clock.now() >= target.expiresAt) { capture.fail("correlation_unavailable"); continue; } + if (late) { capture.fail("correlation_unavailable"); continue; } // Policy is evaluated again at bind against the CURRENT epoch, environment and session. const reason = this.recheck(capture.facts); if (reason) { capture.fail(reason); continue; } @@ -1196,6 +1223,10 @@ export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost } this.notify(); } + /** @internal Asked by `AttachmentDeferredCorrelation.bind` before it binds: the deadline uses this manager's clock. */ + deferredExpired(target: AttachmentDeferredCorrelation): boolean { + return target.host === this && this.clock.now() >= target.expiresAt; + } /** @internal Called once by `AttachmentDeferredCorrelation.cancel`. */ deferredCancelled(target: AttachmentDeferredCorrelation): void { if (target.host !== this) return; diff --git a/typescript/packages/fi-core/src/attachments/stream.ts b/typescript/packages/fi-core/src/attachments/stream.ts index 051d3f061..47bf6cf63 100644 --- a/typescript/packages/fi-core/src/attachments/stream.ts +++ b/typescript/packages/fi-core/src/attachments/stream.ts @@ -20,7 +20,7 @@ export interface StreamState { export interface StreamHost { streamWrite(state: StreamState, chunk: Uint8Array): boolean; streamFinish(state: StreamState): AttachmentHandle; - streamAbort(state: StreamState, reason: "incomplete" | "policy_denied" | "too_large" | "correlation_unavailable"): void; + streamAbort(state: StreamState, reason: "incomplete" | "policy_denied" | "too_large" | "correlation_unavailable" | "unsampled"): void; } /** * Single-object upload of chunks the application ALREADY consumed (design section 6). The core never pulls, clones diff --git a/typescript/packages/fi-core/test/attachments-deferred.test.cjs b/typescript/packages/fi-core/test/attachments-deferred.test.cjs index b2fc22c7e..8adafc2b6 100644 --- a/typescript/packages/fi-core/test/attachments-deferred.test.cjs +++ b/typescript/packages/fi-core/test/attachments-deferred.test.cjs @@ -242,6 +242,80 @@ test('T-DT-14 bind is synchronous and never throws; the transport runs only from await f.manager.shutdown(); }); +// Sampling at bind covers every capture on the target, not only those parked before bind (design 1.4 step 10). +test('T-DT-15 stream open at bind, bound to an unsampled or non-recording span: unsampled, budget returned, nothing uploaded', async () => { + const f = await managerV1(); + for (const bindTo of [{ ...ctx2, traceFlags: 0 }, (() => { const ended = span(ctx2); ended.end(); return ended; })()]) { + const target = f.manager.deferredCorrelation(); + const stream = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 64 }); assert.equal(stream.tryWrite(new Uint8Array([1, 2, 3])), true); + assert.equal(target.bind(bindTo), true); + assert.equal(stream.reason, 'unsampled', 'the open stream is refused at bind'); assert.equal(f.manager.status().ownedBytes, 0); + assert.equal(stream.tryWrite(new Uint8Array([4])), false); + assert.equal((await stream.finish().completion).status, 'unsampled'); + } + await f.manager.flush(50); assert.equal(f.ops().includes('initialize'), false); assert.equal(f.manager.status().openStreams, 0); + await f.manager.shutdown(); +}); +test('T-DT-16 ticket preflighted before bind, committed after bind to an unsampled span: unsampled, source never read, nothing uploaded', async () => { + const f = await managerV1(); const target = f.manager.deferredCorrelation(); + const ticket = f.manager.preflight(target, { ...wav, maxBytes: 4 }); assert.equal(ticket.reason, 'queued'); + const blobTicket = f.manager.preflight(target, { ...wav, maxBytes: 5, sourceKind: 'Blob' }); assert.equal(blobTicket.reason, 'queued'); + target.bind({ ...ctx2, traceFlags: 0 }); + assert.equal(f.manager.status().openTickets, 0, 'reservations on the target are released at bind'); assert.equal(f.manager.status().ownedBytes, 0); + let read = false; + const h = ticket.attach({ kind: 'Bytes', get data() { read = true; return new Uint8Array([1, 2]); } }); + assert.equal(h.admission, 'unsampled'); assert.equal((await h.completion).status, 'unsampled'); assert.equal(read, false); + const { blob, reads } = spyBlob(new Uint8Array([1, 2, 3, 4, 5])); + assert.equal(blobTicket.attach({ kind: 'Blob', data: blob }).admission, 'unsampled'); assert.equal(reads(), 0); + await f.manager.flush(50); assert.equal(f.ops().includes('initialize'), false); + await f.manager.shutdown(); +}); +test('T-DT-17 late bind with no entry point since the deadline: bind returns false, open streams and tickets end correlation_unavailable', async () => { + const c = clock(); const f = await managerV1({}, { clock: c }); + const target = f.manager.deferredCorrelation({ bindTimeoutMs: 1000 }); + const stream = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 64, idleTtlMs: 300000 }); stream.tryWrite(new Uint8Array([1, 2])); + const ticket = f.manager.preflight(target, { ...wav, maxBytes: 4, ttlMs: 30000 }); + c.advance(5000); // nothing touches the manager between the deadline and the bind + const s = span(ctx2); + assert.equal(target.bind(s), false, 'a late bind binds nothing'); assert.equal(target.snapshot(), undefined); assert.equal(target.pending, false); + assert.equal(stream.reason, 'correlation_unavailable'); assert.equal(stream.tryWrite(new Uint8Array([3])), false); + assert.equal((await stream.finish().completion).status, 'correlation_unavailable'); + let read = false; + assert.equal(ticket.attach({ kind: 'Bytes', get data() { read = true; return new Uint8Array([9]); } }).admission, 'correlation_unavailable'); assert.equal(read, false); + assert.equal(f.manager.status().ownedBytes, 0); assert.equal(s.writes, 0); + await f.manager.flush(50); assert.equal(f.ops().includes('initialize'), false); + // Control: a bind inside the deadline still uploads a ticket committed after it and a stream finished after it. + const t2 = f.manager.deferredCorrelation({ bindTimeoutMs: 1000 }); + const s2 = f.manager.startAttachmentStream(t2, { ...mp3, maxBytes: 8, idleTtlMs: 300000 }); s2.tryWrite(new Uint8Array([5])); + const k2 = f.manager.preflight(t2, { ...wav, maxBytes: 4, ttlMs: 30000 }); + c.advance(999); assert.equal(t2.bind(span(ctx2)), true); + assert.equal((await s2.finish().completion).status, 'validating'); + assert.equal((await k2.attach({ kind: 'Bytes', data: new Uint8Array([6]) }).completion).status, 'validating'); + await f.manager.shutdown(1); +}); +test('T-DT-18 backstops: with the bind notification lost, commit and finish still refuse an unsampled span; deferredBound refuses a late target', async () => { + // (a) The sampled check also runs at ticket commit and stream finish, not only in deferredBound. + const f = await managerV1(); f.manager.deferredBound = () => undefined; + const target = f.manager.deferredCorrelation(); + const stream = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 8 }); stream.tryWrite(new Uint8Array([1])); + const ticket = f.manager.preflight(target, { ...wav, maxBytes: 4 }); + assert.equal(target.bind({ ...ctx2, traceFlags: 0 }), true); + assert.equal((await stream.finish().completion).status, 'unsampled'); + assert.equal(ticket.attach({ kind: 'Bytes', data: new Uint8Array([1]) }).admission, 'unsampled'); + await f.manager.flush(50); assert.equal(f.ops().includes('initialize'), false); assert.equal(f.manager.status().ownedBytes, 0); + await f.manager.shutdown(1); + // (b) deferredBound fails a late target's open streams and tickets even if the pre-bind deadline check is bypassed. + const c = clock(); const g = await managerV1({}, { clock: c }); g.manager.deferredExpired = () => false; + const late = g.manager.deferredCorrelation({ bindTimeoutMs: 1000 }); + const ls = g.manager.startAttachmentStream(late, { ...mp3, maxBytes: 8, idleTtlMs: 300000 }); ls.tryWrite(new Uint8Array([1])); + const lt = g.manager.preflight(late, { ...wav, maxBytes: 4, ttlMs: 30000 }); + c.advance(5000); late.bind(span(ctx2)); + assert.equal(ls.reason, 'correlation_unavailable'); + assert.equal((await lt.attach({ kind: 'Bytes', data: new Uint8Array([1]) }).completion).status, 'expired', 'its reservation was released at bind'); + await g.manager.flush(50); assert.equal(g.ops().includes('initialize'), false); assert.equal(g.manager.status().ownedBytes, 0); + await g.manager.shutdown(1); +}); + // ---- T-SR consumed-stream reservation = min(requested, negotiated single-upload cap) --------------------------------- test('T-SR-01 server max_upload_bytes below 4 MiB: a 4 MiB stream request is admitted at the server cap', async () => { const f = await managerV1({}, { limits: { max_upload_bytes: 1 * MiB } });