diff --git a/.github/workflows/fi-core-attachments-tests.yml b/.github/workflows/fi-core-attachments-tests.yml new file mode 100644 index 000000000..aa9b7fc4c --- /dev/null +++ b/.github/workflows/fi-core-attachments-tests.yml @@ -0,0 +1,62 @@ +name: TypeScript attachment core +on: + pull_request: + paths: + - 'typescript/packages/fi-core/**' + - 'typescript/packages/fi-semantic-conventions/**' + - '.github/workflows/fi-core-attachments-tests.yml' + workflow_dispatch: +permissions: + contents: read +jobs: + runtime: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + node: ['18', '24'] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - uses: pnpm/action-setup@v4 + with: + version: 10.30.1 + - name: Fetch pinned F01 contract fixtures + uses: actions/checkout@v4 + with: + ref: f3b0923c62d43b3a52bf4c16a2226c39c22b690d + path: attachment-contract + persist-credentials: false + - name: Build core and run synthetic attachment boundaries + working-directory: typescript + env: + FI_ATTACHMENTS_SPEC_DIR: ${{ github.workspace }}/attachment-contract/spec/attachments/v1 + run: | + pnpm --filter @traceai/fi-core... install --frozen-lockfile --ignore-scripts + pnpm --filter @traceai/fi-semantic-conventions run build + pnpm --filter @traceai/fi-core run build + pnpm --filter @traceai/fi-core run test:attachments + - name: Test installed OTel floor and latest qualified release + if: matrix.node == '24' + working-directory: typescript + run: | + mkdir -p "$RUNNER_TEMP/attachment-packs" + pnpm --dir packages/fi-core pack --pack-destination "$RUNNER_TEMP/attachment-packs" + pnpm --dir packages/fi-semantic-conventions pack --pack-destination "$RUNNER_TEMP/attachment-packs" + for version in 2.0.1 2.12.1; do + node packages/fi-core/test/install-sdk-fixture.cjs "$version" "$RUNNER_TEMP/attachment-sdk-$version" "$RUNNER_TEMP"/attachment-packs/traceai-fi-core-*.tgz "$RUNNER_TEMP"/attachment-packs/traceai-fi-semantic-conventions-*.tgz + FI_CORE_PACKAGE_JSON="$RUNNER_TEMP/attachment-sdk-$version/node_modules/@traceai/fi-core/package.json" FI_ATTACHMENTS_SPEC_DIR="$GITHUB_WORKSPACE/attachment-contract/spec/attachments/v1" node --test packages/fi-core/test/attachments.test.cjs packages/fi-core/test/attachments-tls.test.cjs packages/fi-core/test/attachments-design.test.cjs packages/fi-core/test/attachments-files.test.cjs packages/fi-core/test/attachments-exit.test.cjs packages/fi-core/test/attachments-wire.test.cjs packages/fi-core/test/attachments-deferred.test.cjs packages/fi-core/test/provider-lifecycle.test.cjs + done + - name: Replay privacy and identity fixture safety + run: node typescript/packages/fi-core/test/attachments-contract.cjs attachment-contract/spec/attachments/v1 + - name: Qualify browser-safe leaf + if: matrix.node == '24' + working-directory: typescript/packages/fi-core + run: | + npm install --prefix "$RUNNER_TEMP/attachment-tools" --no-audit --no-fund esbuild@0.25.12 playwright@1.56.1 + "$RUNNER_TEMP/attachment-tools/node_modules/.bin/playwright" install --with-deps chromium + "$RUNNER_TEMP/attachment-tools/node_modules/.bin/esbuild" dist/esm/attachments/index.js --bundle --platform=browser --format=iife --global-name=Attachments --outfile="$RUNNER_TEMP/attachments-browser.js" + node test/attachments-browser.cjs "$RUNNER_TEMP/attachment-tools/node_modules/playwright" "$RUNNER_TEMP/attachments-browser.js" diff --git a/typescript/packages/fi-core/ATTACHMENTS-MIGRATION.md b/typescript/packages/fi-core/ATTACHMENTS-MIGRATION.md new file mode 100644 index 000000000..86db38831 --- /dev/null +++ b/typescript/packages/fi-core/ATTACHMENTS-MIGRATION.md @@ -0,0 +1,24 @@ +# Attachments API migration (`@traceai/fi-core/attachments`) + +The streaming, recording and import work changed a few behaviours that integrations built against the first attachment runtime depend on. Earlier, some of these changes broke callers silently: capture just stopped. Each one now fails loudly, as a compile error, a one-time process warning with a stable code, or an explicit reason at the call. This page lists each change and what to do about it. + +| Change | Old | New | How it fails now | What to change | +|---|---|---|---|---| +| Ready state | `status().state === "queued"` | `status().state === "ready"` | TypeScript: TS2367 on any comparison with `"queued"`. JavaScript: the comparison is always false, so check for `"ready"`. | Compare with `"ready"`. Admission results (`handle.admission`, `ticket.reason`, `writer.reason`) still use `"queued"` for "admitted". | +| Completion `failed` | `completion.status === "failed"` | `"incomplete"` (transport or unknown) or `"unsupported_version"` (invalid response shape) | TypeScript: TS2367. `failed` is no longer in `AttachmentCompletion`. | Treat any status other than `validating` / `importing` as not delivered. A legacy transport that still throws `AttachmentTransportError("failed")` is mapped to `incomplete`. | +| Four-method transport (`bootstrap` / `initialize` / `upload` / `complete`) | Full support | Adapted, with reduced capability | Warns once per process: `DeprecationWarning` code `FI_ATTACHMENTS_LEGACY_TRANSPORT`. `startRecording`, WAV-tier `attachOwnedRecording` and `importCompletedRecording` return `unsupported_version` at the call, so no bytes are buffered or uploaded. Sessions cannot renew, so capture reports `expired` after the first session lifetime. | Pass `createAttachmentHttpTransportV1(...)` or implement `AttachmentTransportV1`. A wrapper around `createAttachmentHttpTransport(...)` must forward its `[ATTACHMENT_TRANSPORT_V1]` member, or wrap the V1 surface directly. Otherwise the wrapper silently becomes legacy. | +| Strict F01 response shapes | Lenient | Every response is checked against F01 (for example, status bodies need `reason`) | The affected attachment reports `unsupported_version`, and a warning with code `FI_ATTACHMENTS_WIRE_REJECTED` names the shape once, such as `UploadStatus` or `AttachmentSession`. The warning contains no values. | Make the server or test double match `spec/attachments/v1`. | +| Session renewal triggers | Only admission calls (`preflight`, `attach`, ...) | Also `status()`, `flush()`, worker wake and recording writes (design 3.1) | Not an error. HTTP doubles that mint short sessions see an extra `POST /tracer/v1/attachment-sessions` with `Idempotency-Key: bs-{session}-{n}` near expiry. | Mint sessions of one hour or more in doubles, or expect the renewal request. | +| `manager.correlation(span)` | Stamped `fi.attachment.*` on the span | Returns a snapshot only. Stamping happens when a capture is admitted (design 0.8). | Not an error: a span without an admitted capture carries no attachment attributes, which is intended. | Pass the correlation (or the span) to `attach` / `preflight` / `startRecording`. Do not rely on `correlation()` alone to mark a span. | + +## Warnings + +Both warning codes are exported as `ATTACHMENT_WARNING_LEGACY_TRANSPORT` and `ATTACHMENT_WARNING_WIRE_REJECTED`. On Node they go through `process.emitWarning`, so `--no-deprecation` and `--no-warnings` apply. In browsers they go to `console.warn`. Each is emitted at most once per process, per code and shape. + +## Additions (non-breaking) + +| Addition | Behaviour | +|---|---| +| `manager.deferredCorrelation({ bindTimeoutMs })` returns an `AttachmentDeferredCorrelation` | A capture target for a span that does not exist yet, such as a wire span emitted only after the response body completes. `preflight`, `attach` and `startAttachmentStream` admit `Bytes`, `Blob` and consumed-stream captures against it using the full deny-wins check. Committed bytes stay in the owned budget, capped at half of it, and nothing is sent. `target.bind(span)` checks policy and sampling again, then uploads against that span and projects the references onto it. The sampling check covers every capture on the target: parked captures, streams still open at bind and tickets committed after bind all end `unsampled` on an unsampled or non-recording span. If the bind timeout passes (default 60 s, max 300 s, checked lazily without a timer), `target.cancel()` is called or the manager shuts down, pending captures end `correlation_unavailable` and upload nothing. A late `bind` (after the timeout) returns false, binds nothing and cancels the target, so open streams and outstanding tickets also end `correlation_unavailable`. Owned files, recordings and imports never defer. If a span already has a correlation, the bound target forwards to it, so the span keeps one reference list. | +| `startAttachmentStream` reservation | The reservation is `min(maxBytes, negotiated single-upload cap)`. A server whose `max_upload_bytes` is below an adapter's default (for example 4 MiB) still admits the stream, and a body larger than the effective cap is discarded whole (`too_large`). `preflight` keeps its exact semantics: a `maxBytes` over the cap is `too_large`. | +| `shutdown()` after `invalidate(reason)` | The state stays `reason` (`policy_denied`, `deleted`, `expired`); before this change, `shutdown()` overwrote it with `incomplete`. | diff --git a/typescript/packages/fi-core/package.json b/typescript/packages/fi-core/package.json index a1dcb3102..ac37844ac 100644 --- a/typescript/packages/fi-core/package.json +++ b/typescript/packages/fi-core/package.json @@ -12,19 +12,30 @@ "import": "./dist/esm/index.js", "require": "./dist/src/index.js" }, + "./attachments": { + "types": "./dist/src/attachments/index.d.ts", + "import": "./dist/esm/attachments/index.js", + "require": "./dist/src/attachments/index.js" + }, + "./attachments/node": { + "types": "./dist/src/attachments/node.d.ts", + "import": "./dist/esm/attachments/node.js", + "require": "./dist/src/attachments/node.js" + }, "./privacy": { "types": "./dist/src/privacy-runtime.d.ts", "import": "./dist/esm/privacy.js", "require": "./dist/src/privacy-runtime.js" } }, - "typesVersions": { "*": { "privacy": ["dist/src/privacy-runtime.d.ts"] } }, + "typesVersions": { "*": { "privacy": ["dist/src/privacy-runtime.d.ts"], "attachments": ["dist/src/attachments/index.d.ts"], "attachments/node": ["dist/src/attachments/node.d.ts"] } }, "scripts": { "prebuild": "rimraf dist", "build": "tsc --build tsconfig.json tsconfig.esm.json tsconfig.esnext.json && tsc-alias -p tsconfig.esm.json && pnpm run postbuild", "postbuild": "node scripts/postbuild.cjs", "test": "jest", "test:provider": "node --test test/provider-lifecycle.test.cjs", + "test:attachments": "node --test test/attachments.test.cjs test/attachments-tls.test.cjs test/attachments-design.test.cjs test/attachments-files.test.cjs test/attachments-exit.test.cjs test/attachments-wire.test.cjs test/attachments-compat.test.cjs test/attachments-deferred.test.cjs", "test:watch": "jest --watch", "test:coverage": "jest --coverage", "lint": "eslint . --ext .ts", @@ -33,6 +44,7 @@ "files": [ "dist", "README.md", + "ATTACHMENTS-MIGRATION.md", "src" ], "repository": { diff --git a/typescript/packages/fi-core/scripts/postbuild.cjs b/typescript/packages/fi-core/scripts/postbuild.cjs index 146c4e624..09450b0fe 100644 --- a/typescript/packages/fi-core/scripts/postbuild.cjs +++ b/typescript/packages/fi-core/scripts/postbuild.cjs @@ -16,3 +16,13 @@ for (const dir of ['dist/esm', 'dist/esnext']) { fs.writeFileSync(`${dir}/${name}.js`, `export { ${runtimeExports.join(', ')} } from "../src/privacy-runtime.js";\n`); } } + +// A provider and an adapter may import different module formats. Keep correlation +// class identity (and its private span references) identical across those imports. +for (const name of ['index', 'manager', 'correlation', 'policy', 'transport-v1', 'node', 'disabled', 'admission', 'stream', 'recording', 'imports', 'owned', 'node-sources', 'wire']) { + const exports = Object.keys(require(`../dist/src/attachments/${name}.js`)); + if (!exports.every(key => /^[A-Za-z_$][\w$]*$/.test(key))) throw new Error('postbuild: unexpected attachment export'); + for (const dir of ['dist/esm', 'dist/esnext']) { + fs.writeFileSync(`${dir}/attachments/${name}.js`, `export { ${exports.join(', ')} } from "../../src/attachments/${name}.js";\n`); + } +} diff --git a/typescript/packages/fi-core/src/attachments/admission.ts b/typescript/packages/fi-core/src/attachments/admission.ts new file mode 100644 index 000000000..df34a7944 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/admission.ts @@ -0,0 +1,50 @@ +import type { AttachmentCorrelation } from "./correlation"; +import type { AdmissionFacts } from "./internal"; +import type { + AttachmentAdmission, AttachmentAdmissionTicket, AttachmentFields, AttachmentHandle, AttachmentSource, AttachmentStream, AttachmentStreamOptions, +} from "./types"; + +/** @internal */ +export interface TicketHost { + commitTicket(ticket: AttachmentTicket, source: Extract, fields: AttachmentFields): AttachmentHandle; + commitStream(ticket: AttachmentTicket, options: AttachmentStreamOptions): AttachmentStream; + releaseReservation(id: number): void; + deniedHandle(reason: AttachmentAdmission): AttachmentHandle; + deniedStream(reason: AttachmentAdmission): AttachmentStream; +} +/** + * Single-use preflight ticket (design 1.5). It holds a reservation id only, never the source; the manager tracks the + * reservation by id so an abandoned ticket can be collected and its budget returned by a finalizer. + */ +export class AttachmentTicket implements AttachmentAdmissionTicket { + /** @internal */ used = false; + constructor( + private readonly host: TicketHost, + readonly reason: AttachmentAdmission, + readonly maxBytes: number, + readonly correlation: AttachmentCorrelation | undefined, + /** @internal */ readonly facts: AdmissionFacts | undefined, + /** @internal */ readonly reservation: number, + /** @internal */ readonly expiresAt: number, + ) {} + get admitted(): boolean { return this.reason === "queued"; } + attach(source: Extract, fields: AttachmentFields = {}): AttachmentHandle { + try { + if (!this.admitted) return this.host.deniedHandle(this.reason); + if (this.used) return this.host.deniedHandle("conflict" as AttachmentAdmission); + return this.host.commitTicket(this, source, fields ?? {}); + } catch { return this.host.deniedHandle("source_unavailable"); } + } + startStream(options: AttachmentStreamOptions): AttachmentStream { + try { + if (!this.admitted) return this.host.deniedStream(this.reason); + if (this.used) return this.host.deniedStream("conflict" as AttachmentAdmission); + return this.host.commitStream(this, options); + } catch { return this.host.deniedStream("source_unavailable"); } + } + release(): void { + if (this.used || !this.admitted) return; + this.used = true; + this.host.releaseReservation(this.reservation); + } +} diff --git a/typescript/packages/fi-core/src/attachments/correlation.ts b/typescript/packages/fi-core/src/attachments/correlation.ts new file mode 100644 index 000000000..54be923a5 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/correlation.ts @@ -0,0 +1,112 @@ +import type { Span } from "@opentelemetry/api"; +import type { AttachmentReference, AttachmentSpan } from "./types"; + +export function canonicalAttachmentTraceId(value: string): string | undefined { + if (typeof value !== "string" || (value.length !== 32 && value.length !== 36) || !/^(?:[0-9a-f]{32}|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/.test(value)) return undefined; + const hex = value.replace(/-/g, ""); + return /^0+$/.test(hex) ? undefined : hex; +} +export function attachmentTraceUuid(hex: string): string { + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} +export function validAttachmentUuid(value: unknown): value is string { + return typeof value === "string" && value.length === 36 && canonicalAttachmentTraceId(value) !== undefined; +} +/** Snapshot while recording, or bind to the ACTUAL emitted context. Never consult ambient context later. */ +export class AttachmentCorrelation { + private identity?: Readonly<{ traceId: string; spanId: string; sampled: boolean }>; + private span?: Span; + private refs: AttachmentReference[] = []; + private prepared = false; + constructor(spanOrContext?: AttachmentSpan) { if (spanOrContext) this.bind(spanOrContext); } + bind(spanOrContext: AttachmentSpan): boolean { + if (this.identity) return false; + try { + const span = "spanContext" in spanOrContext ? spanOrContext : undefined; + const ctx = span ? span.spanContext() : spanOrContext as Exclude; + const traceId = canonicalAttachmentTraceId(ctx.traceId); + if (!traceId || ctx.spanId.length !== 16 || !/^[0-9a-f]{16}$/.test(ctx.spanId) || /^0+$/.test(ctx.spanId)) return false; + this.identity = Object.freeze({ traceId, spanId: ctx.spanId, sampled: (ctx.traceFlags & 1) === 1 && (!span || span.isRecording()) }); + this.span = span; + return true; + } catch { return false; } + } + /** Stamp the four F01 carrier attributes while the actual span is still recording. Called only for an ADMITTED capture. */ + prepare(sessionId: string, nonce: string): void { + try { + if (this.prepared || !this.span?.isRecording()) return; + this.span.setAttribute("fi.attachment.schema_version", 1); + this.span.setAttribute("fi.attachment.capture_session_id", sessionId); + this.span.setAttribute("fi.attachment.correlation_nonce", nonce); + // The carrier always carries the span's CURRENT projection (design 0.8), never a reset to "[]". + this.span.setAttribute("fi.attachment.refs", JSON.stringify(this.refs)); + this.prepared = true; + } catch { /* A foreign span must not break application code. */ } + } + snapshot(): Readonly<{ traceId: string; spanId: string; sampled: boolean }> | undefined { return this.identity; } + /** Called synchronously at admission only. Ended spans are NEVER mutated by transfer completion. */ + project(reference: AttachmentReference, sessionId: string, nonce: string): void { + try { + if (!this.span?.isRecording()) { this.span = undefined; return; } + const next = [...this.refs, Object.freeze({ ...reference })]; + if (next.length > 64) return; + const json = JSON.stringify(next); + if (new TextEncoder().encode(json).byteLength > 16384) return; + this.span.setAttribute("fi.attachment.schema_version", 1); + this.span.setAttribute("fi.attachment.capture_session_id", sessionId); + this.span.setAttribute("fi.attachment.correlation_nonce", nonce); + this.span.setAttribute("fi.attachment.refs", json); + this.refs = next; + } catch { /* Host instrumentation must never fail because of an attachment. */ } + } + references(): readonly AttachmentReference[] { return this.refs.map(ref => Object.freeze({ ...ref })); } +} +/** @internal The manager side of a deferred target: told once when the target binds or is cancelled. */ +export interface AttachmentDeferredHost { + deferredBound(target: AttachmentDeferredCorrelation, spanOrContext: AttachmentSpan): void; + deferredCancelled(target: AttachmentDeferredCorrelation): void; + /** True once the target's bind deadline has passed on the host's clock. Asked BEFORE the identity is bound. */ + deferredExpired?(target: AttachmentDeferredCorrelation): boolean; +} +/** + * A capture target whose span does not exist yet (late-bound wire spans). Captures admitted against it keep bounded + * bytes until `bind(span)`, where policy is evaluated again; on timeout, `cancel()` or manager shutdown they end + * `correlation_unavailable` and upload nothing. Create it with `manager.deferredCorrelation()`. + */ +export class AttachmentDeferredCorrelation extends AttachmentCorrelation { + /** @internal */ canonical?: AttachmentCorrelation; + /** @internal */ cancelled = false; + /** @internal */ + constructor(/** @internal */ readonly host: AttachmentDeferredHost | undefined, /** @internal */ readonly expiresAt: number) { super(); } + /** True while the target is unbound and still accepts captures (no timeout check; the manager checks that). */ + get pending(): boolean { return !this.cancelled && !this.snapshot(); } + /** + * Bind the emitted span. Pending captures are re-evaluated now and upload against it, or end with a reason. A bind + * after `bindTimeoutMs` binds nothing: the target is cancelled (captures end `correlation_unavailable`) and this + * returns false. Never throws. + */ + bind(spanOrContext: AttachmentSpan): boolean { + if (this.cancelled) return false; + // The deadline is enforced BEFORE the identity exists, so no ticket or open stream can outlive it through a late bind. + let expired = false; + try { expired = !this.snapshot() && this.host?.deferredExpired?.(this) === true; } catch { expired = true; } + if (expired) { this.cancel(); return false; } + const bound = super.bind(spanOrContext); + if (bound) { try { this.host?.deferredBound(this, spanOrContext); } catch { /* host failures never reach the caller */ } } + return bound; + } + /** No span will come: pending captures end `correlation_unavailable`, and later captures are refused. Never throws. */ + cancel(): void { + if (this.cancelled || this.snapshot()) return; + this.cancelled = true; + try { this.host?.deferredCancelled(this); } catch { /* bounded */ } + } + prepare(sessionId: string, nonce: string): void { if (this.canonical) this.canonical.prepare(sessionId, nonce); else super.prepare(sessionId, nonce); } + project(reference: AttachmentReference, sessionId: string, nonce: string): void { + if (this.canonical) this.canonical.project(reference, sessionId, nonce); else super.project(reference, sessionId, nonce); + } + references(): readonly AttachmentReference[] { return this.canonical ? this.canonical.references() : super.references(); } +} +export function attachmentMessageUri(reference: AttachmentReference): Readonly<{ type: "uri"; modality: AttachmentReference["modality"]; mime_type: string; uri: string }> { + return Object.freeze({ type: "uri", modality: reference.modality, mime_type: reference.mime_type, uri: `fi-attachment:${reference.occurrence_id}` }); +} diff --git a/typescript/packages/fi-core/src/attachments/disabled.ts b/typescript/packages/fi-core/src/attachments/disabled.ts new file mode 100644 index 000000000..10d294ea2 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/disabled.ts @@ -0,0 +1,53 @@ +import { AttachmentTicket } from "./admission"; +import { AttachmentCorrelation, AttachmentDeferredCorrelation } from "./correlation"; +import { AttachmentManager } from "./manager"; +import type { + AttachmentAdmissionTicket, AttachmentCompletedRecordingRequest, AttachmentCompletion, AttachmentDeferredCorrelationOptions, AttachmentFlushResult, AttachmentHandle, AttachmentImportHandle, + AttachmentImportOptions, AttachmentManagerOptions, AttachmentOptions, AttachmentOwnedFileRef, AttachmentOwnedRecordingHandle, AttachmentOwnedRecordingOptions, + AttachmentPreflightOptions, AttachmentRecordingOptions, AttachmentRecordingWriter, AttachmentSource, AttachmentSpan, AttachmentStatus, AttachmentStream, + AttachmentStreamOptions, AttachmentTarget, +} from "./types"; + +const DISABLED = Object.freeze({ status: "disabled" as const }) as AttachmentCompletion; +const DONE = Promise.resolve(DISABLED); +const EMPTY: AttachmentFlushResult = Object.freeze({ unfinished: 0, dropped: 0 }); +const STATUS: AttachmentStatus = Object.freeze({ state: "disabled", queued: 0, active: 0, openTickets: 0, openStreams: 0, openRecordings: 0, ownedBytes: 0, dropped: 0 }); +const HANDLE: AttachmentHandle = Object.freeze({ attachmentId: "00000000-0000-0000-0000-000000000000", occurrenceId: "00000000-0000-0000-0000-000000000000", + admission: "disabled", completion: DONE }); +const STREAM: AttachmentStream = Object.freeze({ reason: "disabled" as const, tryWrite: () => false, finish: () => HANDLE, abort: () => undefined }); +const OWNED: AttachmentOwnedRecordingHandle = Object.freeze({ tier: "denied" as const, reason: "disabled" as const, completion: DONE, wait: () => DONE, cancel: () => undefined }); +const IMPORT: AttachmentImportHandle = Object.freeze({ reason: "disabled" as const, completion: DONE, status: async () => ({ state: "failed", reason: "disabled" }) }); +const RESULT = Object.freeze({ recordingId: "00000000-0000-0000-0000-000000000000", state: "denied" as const, reason: "disabled", segmentsPublished: 0, gapMs: 0, + captureComplete: false, manifestRevision: 0 }); +const WRITER: AttachmentRecordingWriter = Object.freeze({ recordingId: RESULT.recordingId, reason: "disabled" as const, tryWrite: () => false, markGap: () => undefined, + markPlayout: () => undefined, finish: async () => RESULT, abort: () => undefined, + status: () => Object.freeze({ state: "terminal" as const, reason: "disabled" as const, sealed: 0, uploaded: 0, gapMs: 0, manifestRevision: 0, ownedBytes: 0 }) }); + +/** + * Attachments OFF (the default): the full public surface, every call returning immediately. It starts no timer, + * socket, process listener, finalizer or bootstrap, writes no span attribute and never touches a source (design 8.1). + */ +export class DisabledAttachmentManager extends AttachmentManager { + constructor(_options: AttachmentManagerOptions = {}) { super({ enabled: false, environment: () => ({}) }); } + async ready(): Promise { return false; } + correlation(spanOrContext?: AttachmentSpan): AttachmentCorrelation { return new AttachmentCorrelation(spanOrContext); } + /** Inert: no host, so binding it schedules nothing; every capture against it is `disabled`. */ + deferredCorrelation(_options?: AttachmentDeferredCorrelationOptions): AttachmentDeferredCorrelation { return new AttachmentDeferredCorrelation(undefined, 0); } + preflight(_target: AttachmentTarget, _options: AttachmentPreflightOptions): AttachmentAdmissionTicket { + return new AttachmentTicket({ commitTicket: () => HANDLE, commitStream: () => STREAM, releaseReservation: () => undefined, deniedHandle: () => HANDLE, deniedStream: () => STREAM }, + "disabled", 0, undefined, undefined, -1, 0); + } + attach(_target: AttachmentTarget, _source: AttachmentSource, _options: AttachmentOptions): AttachmentHandle { return HANDLE; } + startAttachmentStream(_target: AttachmentTarget, _options: AttachmentStreamOptions): AttachmentStream { return STREAM; } + startRecording(_target: AttachmentTarget, _options: AttachmentRecordingOptions): AttachmentRecordingWriter { return WRITER; } + attachOwnedRecording(_target: AttachmentTarget, _source: { kind: "OwnedRecordingFile"; lease: AttachmentOwnedFileRef }, _options: AttachmentOwnedRecordingOptions): AttachmentOwnedRecordingHandle { return OWNED; } + /** The resolver is NOT called. */ + async importCompletedRecording(_target: AttachmentTarget, _request: AttachmentCompletedRecordingRequest, _options: AttachmentImportOptions): Promise { return IMPORT; } + updatePolicy(): void { /* can never turn a disabled manager on */ } + invalidate(): void { /* nothing to revoke */ } + async flush(): Promise { return EMPTY; } + shutdown(): Promise { return Promise.resolve(EMPTY); } + status(): AttachmentStatus { return STATUS; } +} +/** Same public surface whether attachments are on or off. */ +export type AttachmentManagerLike = AttachmentManager; diff --git a/typescript/packages/fi-core/src/attachments/imports.ts b/typescript/packages/fi-core/src/attachments/imports.ts new file mode 100644 index 000000000..c3b534127 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/imports.ts @@ -0,0 +1,42 @@ +import { validAttachmentUuid } from "./correlation"; +import type { + AttachmentAdmission, AttachmentCompletedRecordingRequest, AttachmentCompletedRecordingResolver, AttachmentDirection, AttachmentResolvedCompletedRecording, +} from "./types"; + +const AUDIO = ["audio/wav", "audio/ogg", "audio/mpeg", "audio/mp4", "audio/webm", "audio/flac", "audio/L16"]; +const TOKEN = /^[a-zA-Z0-9_-]{1,128}$/; +const RESOLVER_REASONS: readonly AttachmentAdmission[] = ["source_unavailable", "policy_denied", "credential_scope_unsupported", "too_large"]; + +/** Core default: the core never talks to providers (design 5.2). */ +export class AttachmentUnavailableResolver implements AttachmentCompletedRecordingResolver { + async resolve(): Promise { return "source_unavailable"; } +} +export function validAttachmentImportRequest(request: AttachmentCompletedRecordingRequest): boolean { + return !!request && typeof request.provider === "string" && TOKEN.test(request.provider) && typeof request.providerRef === "string" && + TOKEN.test(request.providerRef) && (request.direction === undefined || ["input", "output", "mixed"].includes(request.direction)); +} +/** Resolved direction must be at least as inclusive as the request; `mixed` wins (trusted ancestry). */ +export function attachmentImportDirection(requested: AttachmentDirection, resolved: AttachmentDirection): AttachmentDirection | undefined { + if (resolved === requested) return resolved; + if (resolved === "mixed") return "mixed"; + return undefined; +} +/** Validates a resolver result (design 5.3 step 3). Returns a reason, or undefined when valid. */ +export function attachmentResolvedRecordingDenial(value: unknown, limits: { recording_bytes?: number; recording_duration_ms?: number } | undefined): AttachmentAdmission | "unsupported_version" | undefined { + if (typeof value === "string") return RESOLVER_REASONS.includes(value as AttachmentAdmission) ? value as AttachmentAdmission : "source_unavailable"; + const r = value as AttachmentResolvedCompletedRecording; + if (!r || typeof r !== "object" || !r.source || typeof r.source !== "object") return "unsupported_version"; + const s = r.source as Record; + if (s.kind === "ProviderFile") { + if (Object.keys(s).length !== 3 || !validAttachmentUuid(s.connectionId) || !validAttachmentUuid(s.registeredOutputId)) return "unsupported_version"; + } else if (s.kind === "RemoteUrlEnvelope") { + if (Object.keys(s).length !== 2 || !validAttachmentUuid(s.envelopeId)) return "unsupported_version"; + } else return "unsupported_version"; + if (!AUDIO.includes(r.mimeType) || !["input", "output", "mixed"].includes(r.direction)) return "unsupported_version"; + if (!Number.isInteger(r.declaredBytes) || !Number.isInteger(r.declaredDurationMs)) return "unsupported_version"; + const maxBytes = Math.min(limits?.recording_bytes ?? 1073741824, 1073741824); + const maxMs = Math.min(limits?.recording_duration_ms ?? 7200000, 7200000); + // Never truncated: an out-of-range completed source is too_large (F01 import rule 2). + if (r.declaredBytes < 1 || r.declaredBytes > maxBytes || r.declaredDurationMs < 1 || r.declaredDurationMs > maxMs) return "too_large"; + return undefined; +} diff --git a/typescript/packages/fi-core/src/attachments/index.ts b/typescript/packages/fi-core/src/attachments/index.ts new file mode 100644 index 000000000..cb4adbcba --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/index.ts @@ -0,0 +1,18 @@ +/** Browser-safe leaf: no Node filesystem/network imports or ambient credential discovery. */ +export { AttachmentManager } from "./manager"; +export { DisabledAttachmentManager } from "./disabled"; +export type { AttachmentManagerLike } from "./disabled"; +export { AttachmentTicket } from "./admission"; +export { AttachmentConsumedStream } from "./stream"; +export { AttachmentRecording, attachmentBoundaryStep, attachmentSegmentFrames, attachmentFramesToMs, attachmentWavHeader, validAttachmentManifest } from "./recording"; +export { AttachmentUnavailableResolver } from "./imports"; +export { validAttachmentWire, ATTACHMENT_WIRE_SHAPES } from "./wire"; +export { ATTACHMENT_WARNING_LEGACY_TRANSPORT, ATTACHMENT_WARNING_WIRE_REJECTED } from "./warn"; +export { AttachmentCorrelation, AttachmentDeferredCorrelation, canonicalAttachmentTraceId, attachmentTraceUuid, attachmentMessageUri } from "./correlation"; +export { evaluateAttachmentPolicy, attachmentModality } from "./policy"; +export { + createAttachmentHttpTransport, createAttachmentHttpTransportV1, attachmentFetchClient, AttachmentTransportError, attachmentHttpError, + attachmentIdempotencyKey, attachmentRetryAfterMs, attachmentStrictJson, attachmentLegacyTransport, ATTACHMENT_TRANSPORT_V1, +} from "./transport-v1"; +export type { AttachmentHttpClient, AttachmentHttpRequest, AttachmentHttpResponse, AttachmentHttpTransportOptions, AttachmentErrorCode } from "./transport-v1"; +export type * from "./types"; diff --git a/typescript/packages/fi-core/src/attachments/internal.ts b/typescript/packages/fi-core/src/attachments/internal.ts new file mode 100644 index 000000000..76d703dd3 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/internal.ts @@ -0,0 +1,67 @@ +/** @internal Contracts between the manager and its ticket, stream, recording and import helpers. Not public API. */ +import type { + AttachmentAdmission, AttachmentCapturePolicy, AttachmentClock, AttachmentCompletion, AttachmentContentSource, AttachmentDirection, + AttachmentOperation, AttachmentPurpose, AttachmentRuntimeState, AttachmentSession, AttachmentSourceKind, AttachmentUpload, +} from "./types"; + +export interface AdmissionFacts { + mimeType: string; + direction: AttachmentDirection; + purpose?: AttachmentPurpose; + capture?: AttachmentCapturePolicy; + operation: AttachmentOperation; + sourceKind: AttachmentSourceKind; + /** Only the completed-import path may name ProviderFile / RemoteUrl (design 1.4 step 8). */ + importSource?: boolean; +} +export interface CoreJob { + readonly lane: string; + /** Owned bytes released by the manager when this job ends. */ + bytes: number; + readonly controller: AbortController; + run(signal: AbortSignal): Promise; + /** Re-evaluates admission against the current policy epoch; a reason drops the job. */ + denied(): AttachmentAdmission | undefined; + fail(reason: AttachmentCompletion["status"]): void; + settled?: boolean; +} +export interface UploadSpec { + /** Serialized once at admission; replays are byte-identical. */ + body: string; + occurrenceId: string; + declaredBytes: number; + source: AttachmentContentSource; + recheck(): AttachmentAdmission | undefined; + /** Recording segments under BR-7 fallback PUT the manifest between content and complete. */ + beforeComplete?(outcome: UploadOutcome, signal: AbortSignal): Promise; +} +export interface UploadOutcome { + status: AttachmentCompletion["status"]; + upload?: AttachmentUpload; + body: string; + renewals: number; +} +export interface AttachmentHost { + readonly clock: AttachmentClock; + readonly sessionId: string; + readonly nonce: string; + readonly chunkBytes: number; + readonly manifestTimeoutMs: number; + readonly finalizationTimeoutMs: number; + currentSession(): AttachmentSession | undefined; + runtimeState(): AttachmentRuntimeState; + policyEpoch(): number; + stopping(): boolean; + recheck(facts: AdmissionFacts, options?: { ignoreSessionExpiry?: boolean }): AttachmentAdmission | undefined; + reserveBytes(bytes: number): boolean; + releaseBytes(bytes: number): void; + enqueue(job: CoreJob): void; + runUpload(spec: UploadSpec, signal: AbortSignal): Promise; + renewUpload(outcome: UploadOutcome, signal: AbortSignal): Promise; + putManifest(capability: string, recordingId: string, body: string, key: string, signal: AbortSignal, replay: boolean): Promise; + recordingClosed(id: string): void; + /** Schedules a deferred session-renewal check (no I/O on the caller's stack); recording writes call it (design 3.1). */ + touch(): void; + countDrop(): void; + notify(): void; +} diff --git a/typescript/packages/fi-core/src/attachments/manager.ts b/typescript/packages/fi-core/src/attachments/manager.ts new file mode 100644 index 000000000..0019c4eb2 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/manager.ts @@ -0,0 +1,1329 @@ +import { context, trace } from "@opentelemetry/api"; +import { isTracingSuppressed } from "@opentelemetry/core"; +import { v4 as uuid } from "uuid"; +import { AttachmentTicket, type TicketHost } from "./admission"; +import { AttachmentCorrelation, AttachmentDeferredCorrelation, attachmentTraceUuid, validAttachmentUuid, type AttachmentDeferredHost } from "./correlation"; +import { attachmentImportDirection, attachmentResolvedRecordingDenial, validAttachmentImportRequest } from "./imports"; +import type { AdmissionFacts, AttachmentHost, CoreJob, UploadOutcome, UploadSpec } from "./internal"; +import { validAttachmentWire } from "./wire"; +import { attachmentOwnedFile, type AttachmentOwnedFileRecord, type AttachmentOwnedLease } from "./owned"; +import { attachmentEnabled, attachmentModality, evaluateAttachmentPolicy } from "./policy"; +import { + ATTACHMENT_WAV_HEADER_BYTES, AttachmentRecording, attachmentChunkedSource, attachmentRecordingShape, attachmentSegmentFrames, attachmentWavHeader, + type RecordingFileSegment, +} from "./recording"; +import { AttachmentConsumedStream, type StreamHost, type StreamState } from "./stream"; +import { AttachmentTransportError, attachmentIdempotencyKey, attachmentIsLegacyWire, attachmentWireTransport, validAttachmentSession, validAttachmentUpload } from "./transport-v1"; +import { attachmentWireRejected } from "./warn"; +import type { + AttachmentAdmission, AttachmentAdmissionTicket, AttachmentClock, AttachmentCompletedRecordingRequest, AttachmentCompletion, AttachmentContentSource, + AttachmentDeferredCorrelationOptions, AttachmentFields, AttachmentFlushResult, AttachmentHandle, AttachmentImportHandle, AttachmentImportOptions, AttachmentManagerOptions, AttachmentOptions, + AttachmentOwnedFileRef, AttachmentOwnedRecordingHandle, AttachmentOwnedRecordingOptions, AttachmentPacer, AttachmentPreflightOptions, + AttachmentRecordingOptions, AttachmentRecordingWriter, AttachmentReference, AttachmentRuntimeState, AttachmentSession, AttachmentSource, + AttachmentSourceKind, AttachmentSpan, AttachmentStatus, AttachmentStream, AttachmentStreamOptions, AttachmentTarget, AttachmentTransportV1, + AttachmentUploadRequest, +} from "./types"; + +const MiB = 1024 * 1024; +const CHUNK = 256 * 1024; +// Reserve bounded transfer/read/response scratch for both workers inside the 32 MiB envelope. +const MAX_MEMORY = 32 * MiB - 2 * (CHUNK + 2 * 65536); +const SKEW = 5000; +const QUALIFIED = new Set(["Bytes", "Blob", "OwnedRecordingFile"]); +const PURPOSES = ["conversation_recording", "turn_recording", "model_input", "model_output", "tool_result", "computer_screenshot", "diagnostic", "document"]; +const ROLES = ["user", "assistant", "tool", "system", "unknown"]; +const TOKEN = /^[a-zA-Z0-9_-]{1,128}$/; + +function bounded(value: number | undefined, fallback: number, max: number): number { + return Number.isInteger(value) && value! > 0 ? Math.min(value!, max) : fallback; +} +function unref(timer: ReturnType): void { (timer as unknown as { unref?: () => void }).unref?.(); } +const defaultClock: AttachmentClock = { + now: () => (typeof performance !== "undefined" && typeof performance.now === "function" ? performance.now() : Date.now()), + wall: () => Date.now(), +}; +type HandleControl = { value: AttachmentHandle; finish: (status: AttachmentCompletion["status"]) => void }; +function handle(admission: AttachmentAdmission): HandleControl { + let resolve!: (result: AttachmentCompletion) => void; + const completion = new Promise(done => { resolve = done; }); + const value = Object.freeze({ attachmentId: uuid(), occurrenceId: uuid(), admission, completion }); + let settled = false; + const finish = (status: AttachmentCompletion["status"]) => { if (!settled) { settled = true; resolve(Object.freeze({ status })); } }; + if (admission !== "queued") finish(admission); + return { value, finish }; +} +function codeOf(error: unknown): Exclude { + if (error instanceof AttachmentTransportError) return error.code === "failed" ? "incomplete" : error.code; + return "incomplete"; +} +function raceAbort(promise: Promise, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(new AttachmentTransportError("incomplete")); + return new Promise((resolve, reject) => { + const abort = () => reject(new AttachmentTransportError("incomplete", false)); + signal.addEventListener("abort", abort, { once: true }); + promise.then(value => { signal.removeEventListener("abort", abort); resolve(value); }, error => { signal.removeEventListener("abort", abort); reject(error); }); + }); +} +function pause(ms: number, signal: AbortSignal): Promise { + return new Promise(resolve => { + const done = () => { clearTimeout(timer); signal.removeEventListener("abort", done); resolve(); }; + const timer = setTimeout(done, Math.max(0, ms)); unref(timer); + signal.addEventListener("abort", done, { once: true }); + if (signal.aborted) done(); + }); +} +/** One token bucket per manager at the negotiated upload_bytes_per_second. */ +class TokenBucket implements AttachmentPacer { + private tokens = 0; private last = 0; private started = false; + constructor(private readonly rate: () => number, private readonly clock: AttachmentClock) {} + async take(bytes: number, signal: AbortSignal): Promise { + const rate = this.rate(); + if (!(rate > 0) || rate >= 1073741824) return; + const now = this.clock.now(); + if (!this.started) { this.started = true; this.tokens = rate; this.last = now; } + this.tokens = Math.min(rate, this.tokens + (now - this.last) * rate / 1000); this.last = now; + this.tokens -= bytes; + if (this.tokens < 0) await pause(-this.tokens / rate * 1000, signal); + } +} +/** ES2021 FinalizationRegistry, typed locally so consumers compiling this source with an older `lib` still build. */ +interface Finalizer { register(target: object, held: () => void, token?: object): void; unregister(token: object): boolean } +type FinalizerConstructor = new (cleanup: (held: () => void) => void) => Finalizer; +class Lane { readonly jobs: CoreJob[] = []; busy = false; constructor(readonly id: string) {} } +/** `deferred`: the ticket was admitted against an unbound deferred target, so its bytes count toward the pre-bind cap. */ +interface Reservation { bytes: number; expiresAt: number; deferred?: AttachmentDeferredCorrelation } +/** A capture admitted and committed against a deferred target that has not bound yet: bounded bytes held, nothing sent. */ +interface DeferredCapture { + readonly target: AttachmentDeferredCorrelation; + readonly facts: AdmissionFacts; + readonly bytes: number; + start(bound: AttachmentCorrelation): void; + fail(reason: AttachmentAdmission): void; +} +interface StreamExtra { facts: AdmissionFacts; fields: AttachmentFields; correlation: AttachmentCorrelation; purpose?: AttachmentOptions["purpose"] } + +/** One per provider/runtime; explicit attachment APIs and adapters use this same instance. */ +export class AttachmentManager implements TicketHost, StreamHost, AttachmentHost, AttachmentDeferredHost { + /** @internal */ readonly clock: AttachmentClock; + /** @internal */ readonly sessionId = uuid(); + /** @internal */ readonly nonce = uuid(); + /** @internal */ readonly chunkBytes = CHUNK; + /** @internal */ readonly manifestTimeoutMs: number; + /** @internal */ readonly finalizationTimeoutMs: number; + private readonly config: AttachmentManagerOptions; + private readonly enabled: boolean; + private session?: AttachmentSession; + private sessionLifetime = 0; + private bootStatus: AttachmentRuntimeState = "bootstrap_pending"; + private readonly bootController?: AbortController; + private readonly boot: Promise; + private readonly wire?: AttachmentTransportV1; + /** A four-method transport: no renewal, manifests or imports (warned once at construction). */ + private readonly legacyWire: boolean = false; + private readonly lanes = new Map(); + private order: Lane[] = []; + private cursor = 0; + private queued = 0; + private readonly active = new Set(); + private readonly correlations = new WeakMap(); + private readonly listeners = new Set<() => void>(); + private readonly reservations = new Map(); + private nextReservation = 0; + private readonly streams = new Set(); + private readonly deferredCaptures = new Set(); + private readonly recordings = new Map(); + private recordingSlots = 0; + private bytes = 0; + private dropped = 0; + private epoch = 0; + private closed = false; + private stoppingFlag = false; + private closedReason: AttachmentRuntimeState = "incomplete"; + private shutdownPromise?: Promise; + private renewal = 0; + private renewing = false; + private nextRenewalAt = 0; + private renewalFailures = 0; + private renewCheckPending = false; + private registry?: Finalizer; + private readonly pacer: TokenBucket; + private readonly maxBytes: number; + private readonly maxDescriptors: number; + private readonly workers: number; + private readonly requestTimeout: number; + private readonly shutdownTimeout: number; + private readonly contentTimeout: number; + private readonly syncCopyBytes: number; + private readonly ticketTtl: number; + private readonly streamIdleTtl: number; + + constructor(options: AttachmentManagerOptions = {}) { + this.config = { ...options, traceConfig: { ...options.traceConfig } }; + this.clock = options.clock ?? defaultClock; + this.maxBytes = bounded(options.maxOwnedBytes, MAX_MEMORY, MAX_MEMORY); + this.maxDescriptors = bounded(options.maxDescriptors, 128, 128); + this.workers = bounded(options.workers, 2, 2); + this.requestTimeout = bounded(options.requestTimeoutMs, 2000, 30000); + this.shutdownTimeout = bounded(options.shutdownTimeoutMs, 5000, 30000); + this.contentTimeout = bounded(options.contentTimeoutMs, 120000, 600000); + this.manifestTimeoutMs = bounded(options.manifestTimeoutMs, 10000, 30000); + this.syncCopyBytes = bounded(options.syncCopyBytes, 8 * MiB, 32 * MiB); + this.ticketTtl = bounded(options.ticketTtlMs, 10000, 30000); + this.streamIdleTtl = bounded(options.streamIdleTtlMs, 60000, 300000); + this.finalizationTimeoutMs = bounded(options.finalizationTimeoutMs, 30000, 30000); + this.pacer = new TokenBucket(() => this.session?.limits.upload_bytes_per_second ?? 0, this.clock); + // Resolved ONCE (design 8.1): enabling later means building a new provider. + this.enabled = attachmentEnabled(this.config, this.env()); + if (!this.enabled) { this.bootStatus = "disabled"; this.boot = Promise.resolve(false); return; } + this.bootController = new AbortController(); + this.wire = attachmentWireTransport(options.transport); + this.legacyWire = attachmentIsLegacyWire(this.wire); + this.boot = this.bootstrap(); + } + private env(): Readonly> { + try { return this.config.environment?.() ?? {}; } + catch { return { FI_ATTACHMENTS_ENABLED: "false" }; } + } + private async bootstrap(): Promise { + try { + if (!this.wire) { this.bootStatus = "endpoint_unconfigured"; return false; } + if (typeof this.config.projectName !== "string" || !this.config.projectName || this.config.projectName.length > 255 || + typeof this.config.projectType !== "string" || !this.config.projectType || this.config.projectType.length > 64 || + (this.config.workspaceId !== undefined && !validAttachmentUuid(this.config.workspaceId))) { this.bootStatus = "scope_unresolved"; return false; } + const request = { schema_versions: [1] as [1], project_name: this.config.projectName, project_type: this.config.projectType, + capture_session_id: this.sessionId, correlation_nonce: this.nonce, + ...(this.config.workspaceId ? { workspace_id: this.config.workspaceId } : {}) }; + const key = attachmentIdempotencyKey("bootstrap", this.sessionId, 0); + const session = await this.attempt(signal => this.wire!.bootstrap(request, key, signal), this.bootController!.signal); + if (!validAttachmentSession(session, this.sessionId, this.clock.wall())) throw attachmentWireRejected("AttachmentSession", new AttachmentTransportError("unsupported_version")); + if (this.config.workspaceId && session.scope.workspace_id !== this.config.workspaceId) throw new AttachmentTransportError("scope_unresolved"); + if (this.closed) return false; + // Detached bounded wire object: a caller/transport cannot later loosen policy through mutation. + this.session = JSON.parse(JSON.stringify(session)); + this.sessionLifetime = Math.max(0, Date.parse(session.expires_at) - this.clock.wall()); + this.bootStatus = "ready"; + return true; + } catch (error) { + const code = error instanceof AttachmentTransportError ? error.code : "source_unavailable"; + this.bootStatus = code === "credential_scope_unsupported" || code === "policy_denied" ? "credential_scope_unsupported" : + code === "scope_unresolved" || code === "not_found" ? "scope_unresolved" : + code === "unsupported_version" || code === "endpoint_unconfigured" ? code : "source_unavailable"; + return false; + } + } + /** Explicit startup wait; setup and inference do not await it. */ + async ready(timeoutMs = 5000): Promise { + if (!this.enabled) return false; + return this.deadline(this.boot, bounded(timeoutMs, 5000, 30000), false); + } + /** Snapshot (cached per span) for later or late capture. Stamping happens only at an ADMITTED capture (design 0.8). */ + correlation(spanOrContext?: AttachmentSpan): AttachmentCorrelation { + if (!spanOrContext) return new AttachmentCorrelation(); + let result = this.correlations.get(spanOrContext); + if (!result) { result = new AttachmentCorrelation(spanOrContext); this.correlations.set(spanOrContext, result); } + return result; + } + /** + * A target for a span that does not exist yet (for example a wire span emitted only after the body completes). + * `preflight` / `attach` / `startAttachmentStream` admit Bytes, Blob and consumed-stream captures against it with the + * full deny-wins lattice; committed bytes stay in the owned budget (at most half of it pre-bind) until + * `target.bind(span)`, where policy and sampling are evaluated again before anything is sent. Timeout (`bindTimeoutMs`), + * `target.cancel()` and manager shutdown end pending captures `correlation_unavailable`, uploading nothing. + */ + deferredCorrelation(options: AttachmentDeferredCorrelationOptions = {}): AttachmentDeferredCorrelation { + if (this.enabled) this.enter(); + return new AttachmentDeferredCorrelation(this.enabled ? this : undefined, this.clock.now() + bounded(options?.bindTimeoutMs, 60000, 300000)); + } + + // --- state and admission ------------------------------------------------------------------------ + /** @internal */ + currentSession(): AttachmentSession | undefined { return this.session; } + /** @internal */ + policyEpoch(): number { return this.epoch; } + /** @internal */ + stopping(): boolean { return this.stoppingFlag || this.closed; } + /** @internal */ + runtimeState(): AttachmentRuntimeState { + if (this.closed) return this.closedReason; + if (this.stoppingFlag) return "incomplete"; + if (this.bootStatus === "ready" && this.session && this.clock.wall() >= Date.parse(this.session.expires_at) - SKEW) return "expired"; + return this.bootStatus; + } + /** @internal Design 1.4 steps 1-8 against the CURRENT policy epoch. Never touches a source. */ + recheck(facts: AdmissionFacts, options: { ignoreSessionExpiry?: boolean; admission?: boolean } = {}): AttachmentAdmission | undefined { + const env = this.env(); + if (!this.enabled || !attachmentEnabled(this.config, env)) return "disabled"; + if (this.closed) return this.closedReason as AttachmentAdmission; + if (options.admission && (this.stoppingFlag || this.shutdownPromise)) return "incomplete"; + const state = this.runtimeState(); + if (state === "expired") { if (!options.ignoreSessionExpiry) return "expired"; } + else if (state !== "ready" && !(state === "incomplete" && !options.admission)) return state as AttachmentAdmission; + if (!this.session) return "bootstrap_pending"; + if (options.admission) { try { if (isTracingSuppressed(context.active())) return "policy_denied"; } catch { return "policy_denied"; } } + if (facts.purpose !== undefined && !PURPOSES.includes(facts.purpose)) return "policy_denied"; + const capture = { ...facts.capture, newRecording: facts.operation === "create_recording" || facts.capture?.newRecording === true }; + for (const key of Object.keys(facts.capture ?? {})) { + const value = (facts.capture as Record)[key]; + if (value !== undefined && typeof value !== "boolean") return "policy_denied"; + } + const denied = evaluateAttachmentPolicy(this.config, this.config.traceConfig ?? {}, this.session.policy, facts.sourceKind, + { mimeType: facts.mimeType, direction: facts.direction, purpose: facts.purpose, capture }, env); + if (denied) return denied; + if (!QUALIFIED.has(facts.sourceKind) && !(facts.importSource && (facts.sourceKind === "ProviderFile" || facts.sourceKind === "RemoteUrl"))) return "source_unavailable"; + return undefined; + } + private resolveCorrelation(target: AttachmentTarget): AttachmentCorrelation | undefined { + try { + if (target instanceof AttachmentDeferredCorrelation) return target.canonical ?? target; + if (target instanceof AttachmentCorrelation) return target; + if (target === undefined) { + // The current span is read ONCE at call time. + const span = trace.getSpan(context.active()); + return span ? this.correlation(span) : undefined; + } + return this.correlation(target); + } catch { return undefined; } + } + private descriptors(): number { return this.queued + this.active.size + this.reservations.size + this.streams.size + this.recordings.size + this.deferredCaptures.size; } + /** Owned by this manager, unbound, not cancelled and within its bind deadline. */ + private deferredOpen(target: AttachmentDeferredCorrelation): boolean { + return target.host === this && target.pending && this.clock.now() < target.expiresAt; + } + /** The bound identity (the canonical correlation when the span already had one) is sampled and was recording at bind. */ + private static sampledTarget(correlation: AttachmentCorrelation): boolean { + const bound = correlation instanceof AttachmentDeferredCorrelation ? correlation.canonical ?? correlation : correlation; + return bound.snapshot()?.sampled === true; + } + private static pendingTarget(correlation: AttachmentCorrelation | undefined): AttachmentDeferredCorrelation | undefined { + return correlation instanceof AttachmentDeferredCorrelation && correlation.pending ? correlation : undefined; + } + /** Bytes held for targets that have not bound yet. Capped at half the owned budget so bound captures keep headroom. */ + private deferredBytes(): number { + let total = 0; + for (const reservation of this.reservations.values()) if (reservation.deferred?.pending) total += reservation.bytes; + for (const state of this.streams) if (AttachmentManager.pendingTarget(state.correlation)) total += state.reserved; + for (const capture of this.deferredCaptures) total += capture.bytes; + return total; + } + private deferredCap(): number { return Math.floor(this.maxBytes / 2); } + private uploadCap(mimeType: string, sourceKind: AttachmentSourceKind): number { + const limit = Math.min(this.session?.limits.max_upload_bytes ?? 0, 50 * MiB); + const kind = attachmentModality(mimeType) === "image" ? Math.min(limit, 20 * MiB) : limit; + return sourceKind === "OwnedRecordingFile" ? kind : Math.min(kind, this.maxBytes); + } + /** @internal */ + reserveBytes(bytes: number): boolean { + if (!(bytes >= 0) || this.bytes + bytes > this.maxBytes) return false; + this.bytes += bytes; return true; + } + /** @internal */ + releaseBytes(bytes: number): void { if (bytes > 0) this.bytes = Math.max(0, this.bytes - bytes); } + /** @internal */ + countDrop(): void { this.dropped++; } + /** @internal */ + notify(): void { for (const notify of this.listeners) notify(); } + /** @internal */ + recordingClosed(id: string): void { this.recordings.delete(id); this.notify(); } + private validFields(fields: AttachmentFields): boolean { + if (fields.role !== undefined && !ROLES.includes(fields.role)) return false; + for (const [value, max] of [[fields.messageIndex, 65535], [fields.partIndex, 65535], [fields.channel, 31]] as const) { + if (value !== undefined && (!Number.isInteger(value) || value < 0 || value > max)) return false; + } + return [fields.conversationId, fields.turnId, fields.toolCallId].every(value => + value === undefined || typeof value === "string" && value.trim() === value && TOKEN.test(value)); + } + private validOptions(options: AttachmentOptions): boolean { + if (options.purpose !== undefined && !PURPOSES.includes(options.purpose)) return false; + return this.validFields(options); + } + private finalizer(): Finalizer | undefined { + const Registry = (globalThis as unknown as { FinalizationRegistry?: FinalizerConstructor }).FinalizationRegistry; + if (!this.registry && typeof Registry === "function") this.registry = new Registry((release: () => void) => { try { release(); } catch { /* bounded */ } }); + return this.registry; + } + /** Lazy sweep at every entry point: no timers are armed for tickets or streams (design 1.5, 6.3). */ + private sweep(): void { + const now = this.clock.now(); + for (const [id, reservation] of this.reservations) if (reservation.expiresAt <= now) this.releaseReservation(id); + for (const state of this.streams) if (state.open && now - state.lastWrite > state.idleTtl) this.streamAbort(state, "incomplete"); + // Deferred targets past their bind deadline: fail closed, discard retained bytes (no timer; design 1.5 style). + for (const [id, reservation] of this.reservations) if (reservation.deferred?.pending && now >= reservation.deferred.expiresAt) this.releaseReservation(id); + for (const state of [...this.streams]) { + const target = AttachmentManager.pendingTarget(state.correlation); + if (state.open && target && now >= target.expiresAt) this.streamAbort(state, "correlation_unavailable"); + } + for (const capture of [...this.deferredCaptures]) if (now >= capture.target.expiresAt) capture.fail("correlation_unavailable"); + } + private maybeRenew(): void { + const session = this.session; + // A legacy wire has no renewal route: retrying it would only spin; the session runs to `expired` (warned at construction). + if (!session || this.renewing || this.closed || this.stoppingFlag || this.bootStatus !== "ready" || !this.wire || this.legacyWire) return; + const now = this.clock.wall(); const expires = Date.parse(session.expires_at); + if (expires - now > Math.max(60000, 0.2 * this.sessionLifetime)) return; + if (this.clock.now() < this.nextRenewalAt) return; + this.renewing = true; + const sequence = this.renewal + 1; + void (async () => { + try { + const next = await this.attempt(signal => this.wire!.renewSession(session, sequence, signal), this.bootController!.signal, () => undefined, 3); + if (!validAttachmentSession(next, this.sessionId, this.clock.wall())) throw attachmentWireRejected("AttachmentSession", new AttachmentTransportError("unsupported_version")); + const scope = (s: AttachmentSession) => `${s.scope.organization_id}/${s.scope.workspace_id}/${s.scope.project_id}`; + if (scope(next) !== scope(session)) { this.invalidate("scope_unresolved" as "deleted"); return; } + if (this.session !== session || this.closed) return; + const changed = JSON.stringify(next.policy) !== JSON.stringify(session.policy); + this.renewal = sequence; + this.session = JSON.parse(JSON.stringify(next)); + this.sessionLifetime = Math.max(0, Date.parse(next.expires_at) - this.clock.wall()); + this.renewalFailures = 0; this.nextRenewalAt = this.clock.now() + 30000; + if (changed) { this.epoch++; this.cancelDenied(); } + } catch { + this.renewalFailures++; + this.nextRenewalAt = this.clock.now() + Math.min(30000, 1000 * 2 ** Math.min(this.renewalFailures, 5)); + } finally { this.renewing = false; } + })(); + } + private enter(): void { this.sweep(); this.maybeRenew(); } + /** @internal Deduped deferred renewal check for hot paths such as recording `tryWrite` (design 3.1). */ + touch(): void { + if (this.renewCheckPending || !this.enabled) return; + this.renewCheckPending = true; + void Promise.resolve().then(() => { this.renewCheckPending = false; this.maybeRenew(); }); + } + private preflightFacts(options: AttachmentPreflightOptions, importSource = false): AdmissionFacts { + return { mimeType: options.mimeType, direction: options.direction, purpose: options.purpose, + capture: options.capture ? { ...options.capture } : undefined, operation: options.operation ?? "observe_consumed", + sourceKind: options.sourceKind ?? "Bytes", importSource }; + } + private admitTarget(target: AttachmentTarget, facts: AdmissionFacts, allowDeferred = false): + { reason?: AttachmentAdmission; correlation?: AttachmentCorrelation; identity?: { traceId: string; spanId: string }; deferred?: AttachmentDeferredCorrelation } { + if (typeof facts.mimeType !== "string" || !["input", "output", "mixed"].includes(facts.direction) || + !["observe_consumed", "import_existing", "create_recording"].includes(facts.operation)) { + const first = this.recheck(facts, { admission: true }); + return { reason: first ?? "policy_denied" }; + } + const reason = this.recheck(facts, { admission: true }); + if (reason) return { reason }; + const correlation = this.resolveCorrelation(target); + // An unbound deferred target is admitted on the lattice alone; sampling and policy are evaluated again at bind. + if (correlation instanceof AttachmentDeferredCorrelation && !correlation.snapshot()) { + return allowDeferred && this.deferredOpen(correlation) ? { correlation, deferred: correlation } : { reason: "correlation_unavailable" }; + } + const identity = correlation?.snapshot(); + if (!correlation || !identity) return { reason: "correlation_unavailable" }; + if (!identity.sampled) return { reason: "unsampled" }; + return { correlation, identity }; + } + /** @internal */ + deniedHandle(reason: AttachmentAdmission): AttachmentHandle { return handle(reason).value; } + /** @internal */ + deniedStream(reason: AttachmentAdmission): AttachmentStream { + const denied = handle(reason).value; + return new AttachmentConsumedStream(undefined, { reason, chunks: [], length: 0, maxBytes: 0, reserved: 0, lastWrite: 0, idleTtl: 0, epoch: 0, open: false, handle: denied }); + } + + /** + * Full deny-wins check BEFORE any decode, read, copy, file open, provider call or stream observer (design section 1). + * Returns a single-use ticket holding a descriptor and a byte budget. + */ + preflight(target: AttachmentTarget, options: AttachmentPreflightOptions): AttachmentAdmissionTicket { + const denied = (reason: AttachmentAdmission) => new AttachmentTicket(this, reason, 0, undefined, undefined, -1, 0); + try { + this.enter(); + const facts = this.preflightFacts(options ?? ({} as AttachmentPreflightOptions)); + const { reason, correlation, deferred } = this.admitTarget(target, facts, facts.sourceKind === "Bytes" || facts.sourceKind === "Blob"); + if (reason) return denied(reason); + const maxBytes = options.maxBytes; + if (!Number.isInteger(maxBytes) || maxBytes <= 0 || maxBytes > this.uploadCap(facts.mimeType, facts.sourceKind)) return denied("too_large"); + const reserve = facts.sourceKind === "OwnedRecordingFile" ? CHUNK : maxBytes; + if (this.descriptors() >= this.maxDescriptors || this.bytes + reserve > this.maxBytes) return denied("queue_full"); + if (deferred && this.deferredBytes() + reserve > this.deferredCap()) return denied("queue_full"); + const ttl = bounded(options.ttlMs, this.ticketTtl, 30000); + const id = ++this.nextReservation; const expiresAt = this.clock.now() + ttl; + this.bytes += reserve; this.reservations.set(id, { bytes: reserve, expiresAt, ...(deferred ? { deferred } : {}) }); + const ticket = new AttachmentTicket(this, "queued", maxBytes, correlation, facts, id, expiresAt); + this.finalizer()?.register(ticket, () => this.releaseReservation(id), ticket); + return ticket; + } catch { return denied("source_unavailable"); } + } + /** @internal */ + releaseReservation(id: number): void { + const reservation = this.reservations.get(id); + if (!reservation) return; + this.reservations.delete(id); this.releaseBytes(reservation.bytes); this.notify(); + } + private takeTicket(ticket: AttachmentTicket): { reason?: AttachmentAdmission; reservation?: Reservation } { + ticket.used = true; + try { this.registry?.unregister(ticket); } catch { /* bounded */ } + this.sweep(); + const target = ticket.correlation; + if (target instanceof AttachmentDeferredCorrelation && !target.snapshot() && !this.deferredOpen(target)) { + this.releaseReservation(ticket.reservation); return { reason: "correlation_unavailable" }; + } + // A ticket preflighted against a deferred target and committed after bind: the bound span must be sampled. + if (target instanceof AttachmentDeferredCorrelation && target.snapshot() && !AttachmentManager.sampledTarget(target)) { + this.releaseReservation(ticket.reservation); return { reason: "unsampled" }; + } + const reservation = this.reservations.get(ticket.reservation); + if (!reservation || this.clock.now() >= ticket.expiresAt) { this.releaseReservation(ticket.reservation); return { reason: "expired" }; } + // Commit re-check against the CURRENT policy epoch; tightening denies without reading the source. + const reason = this.recheck(ticket.facts!, { admission: true }); + if (reason) { this.releaseReservation(ticket.reservation); return { reason }; } + this.reservations.delete(ticket.reservation); + return { reservation }; + } + /** @internal */ + commitTicket(ticket: AttachmentTicket, source: Extract, fields: AttachmentFields): AttachmentHandle { + const taken = this.takeTicket(ticket); + if (taken.reason) return handle(taken.reason).value; + const reserved = taken.reservation!.bytes; + const fail = (reason: AttachmentAdmission) => { this.releaseBytes(reserved); this.notify(); return handle(reason).value; }; + const facts = ticket.facts!; + if (!source || source.kind !== facts.sourceKind) return fail("conflict" as AttachmentAdmission); + if (!this.validFields(fields)) return fail("policy_denied"); + const read = this.readSource(source, ticket.maxBytes); + if ("reason" in read) return fail(read.reason); + const keep = facts.sourceKind === "OwnedRecordingFile" ? reserved : read.size; + this.releaseBytes(reserved - keep); // the reservation shrinks to the actual length + return this.enqueueUpload(facts, fields, ticket.correlation!, read.size, read.content, keep, read.owned); + } + private readSource(source: AttachmentSource, maxBytes: number): { reason: AttachmentAdmission } | { size: number; content?: AttachmentContentSource; owned?: AttachmentOwnedFileRecord } { + if (source.kind === "Bytes") { + const data = source.data; + if (!(data instanceof Uint8Array)) return { reason: "source_unavailable" }; + const size = data.byteLength; + if (!size || size > maxBytes || size > this.syncCopyBytes) return { reason: "too_large" }; + // Mutable inputs are detached synchronously, after all admission and budget checks. + return { size, content: new Uint8Array(data) }; + } + if (source.kind === "Blob") { + const data = source.data; + if (typeof Blob === "undefined" || !(data instanceof Blob)) return { reason: "source_unavailable" }; + if (!data.size || data.size > maxBytes) return { reason: "too_large" }; + return { size: data.size, content: data }; + } + if (source.kind === "OwnedRecordingFile") { + const record = attachmentOwnedFile(source.lease); + if (!record || record.closed()) return { reason: "source_unavailable" }; + const size = record.identity.size; + if (!size || size > maxBytes) return { reason: "too_large" }; + return { size, owned: record }; + } + return { reason: "source_unavailable" }; + } + private uploadRequest(facts: AdmissionFacts, fields: AttachmentFields, identity: { traceId: string; spanId: string }, ids: { attachmentId: string; occurrenceId: string }, size: number): AttachmentUploadRequest { + return { schema_version: 1, attachment_id: ids.attachmentId, occurrence_id: ids.occurrenceId, + association: { capture_session_id: this.sessionId, correlation_nonce: this.nonce, parent: { trace_id: attachmentTraceUuid(identity.traceId), span_id: identity.spanId } }, + policy_version: this.session!.policy.version, deletion_generation: 0, mime_type: facts.mimeType, + modality: attachmentModality(facts.mimeType)!, direction: facts.direction, source_kind: facts.sourceKind, declared_bytes: size, + ...(facts.purpose ? { purpose: facts.purpose } : {}), + ...(fields.messageIndex === undefined ? {} : { message_index: fields.messageIndex }), + ...(fields.partIndex === undefined ? {} : { part_index: fields.partIndex }), + ...(fields.conversationId ? { conversation_id: fields.conversationId } : {}), + ...(fields.turnId ? { turn_id: fields.turnId } : {}), + ...(fields.toolCallId ? { tool_call_id: fields.toolCallId } : {}) }; + } + private reference(request: AttachmentUploadRequest, fields: AttachmentFields): AttachmentReference { + // role and channel are display hints on the live span only; UploadRequest never carries them (design 7). + return { attachment_id: request.attachment_id, occurrence_id: request.occurrence_id, direction: request.direction, modality: request.modality, + mime_type: request.mime_type, ...(request.purpose ? { purpose: request.purpose } : {}), + ...(fields.messageIndex === undefined ? {} : { message_index: fields.messageIndex }), + ...(fields.partIndex === undefined ? {} : { part_index: fields.partIndex }), + ...(fields.role ? { role: fields.role } : {}), ...(fields.channel === undefined ? {} : { channel: fields.channel }), + ...(fields.conversationId ? { conversation_id: fields.conversationId } : {}), + ...(fields.turnId ? { turn_id: fields.turnId } : {}), + ...(fields.toolCallId ? { tool_call_id: fields.toolCallId } : {}) }; + } + /** The correlation to upload against now, or undefined while a deferred target is unbound. */ + private boundTarget(correlation: AttachmentCorrelation): AttachmentCorrelation | undefined { + if (!(correlation instanceof AttachmentDeferredCorrelation)) return correlation; + return correlation.snapshot() ? correlation.canonical ?? correlation : undefined; + } + /** Holds a committed capture (its bytes stay reserved) until its deferred target binds, fails or times out. */ + private park(target: AttachmentDeferredCorrelation, facts: AdmissionFacts, bytes: number, result: HandleControl, + start: (bound: AttachmentCorrelation) => void, discard: () => void): void { + const capture: DeferredCapture = { target, facts, bytes, + start: bound => { if (this.deferredCaptures.delete(capture)) start(bound); }, + fail: reason => { + if (!this.deferredCaptures.delete(capture)) return; + discard(); this.releaseBytes(bytes); this.dropped++; result.finish(reason); this.notify(); + } }; + this.deferredCaptures.add(capture); + if (!this.deferredOpen(target)) capture.fail("correlation_unavailable"); + } + private enqueueUpload(facts: AdmissionFacts, fields: AttachmentFields, correlation: AttachmentCorrelation, size: number, + content: AttachmentContentSource | undefined, bytes: number, owned?: AttachmentOwnedFileRecord, project = true, result: HandleControl = handle("queued")): AttachmentHandle { + const bound = this.boundTarget(correlation); + if (!bound) { + // Retained, bounded and not sent: the request (parent, policy version) is built only once the span is known. + this.park(correlation as AttachmentDeferredCorrelation, facts, bytes, result, + target => { this.enqueueUpload(facts, fields, target, size, content, bytes, owned, project, result); }, () => { content = undefined; }); + return result.value; + } + if (!AttachmentManager.sampledTarget(bound)) { + // Backstop for captures admitted against a deferred target: an unsampled bound span never receives media. + content = undefined; this.releaseBytes(bytes); this.dropped++; result.finish("unsampled"); this.notify(); + return result.value; + } + correlation = bound; + const identity = correlation.snapshot()!; + const request = this.uploadRequest(facts, fields, identity, result.value, size); + const body = JSON.stringify(request); + const controller = new AbortController(); + let data = content; + const job: CoreJob = { + lane: request.occurrence_id, bytes, controller, + denied: () => this.recheck(facts), + fail: reason => { data = undefined; result.finish(reason); }, + run: async signal => { + let lease: AttachmentOwnedLease | undefined; + try { + let source = data; + if (owned) { + lease = await raceAbort(owned.open(), signal); + const opened = lease; + source = { size: opened.size, read: (offset, length) => opened.read(offset, length), check: () => opened.check() }; + } + if (!source) { result.finish("incomplete"); return; } + const outcome = await this.runUpload({ body, occurrenceId: request.occurrence_id, declaredBytes: size, source, recheck: () => this.recheck(facts, { ignoreSessionExpiry: true }) }, signal); + result.finish(outcome.status); + if (outcome.status !== "validating") this.dropped++; + } catch (error) { result.finish(codeOf(error)); this.dropped++; } + finally { data = undefined; if (lease) await lease.close().catch(() => undefined); } + }, + }; + this.enqueue(job); + if (project) correlation.project(this.reference(request, fields), this.sessionId, this.nonce); + return result.value; + } + attach(spanOrContext: AttachmentTarget, source: AttachmentSource, options: AttachmentOptions): AttachmentHandle { + try { + if (this.shutdownPromise) return handle("incomplete").value; + this.enter(); + // No access to source.data/path/url until the complete privacy gate passes. + const kind = source.kind; + const facts: AdmissionFacts = { mimeType: options.mimeType, direction: options.direction, purpose: options.purpose, + capture: options.capture ? { ...options.capture } : undefined, operation: options.capture?.newRecording ? "create_recording" : "observe_consumed", sourceKind: kind }; + const { reason, correlation, deferred } = this.admitTarget(spanOrContext, facts, kind === "Bytes" || kind === "Blob"); + if (reason) return handle(reason).value; + if (!this.validOptions(options)) return handle("policy_denied").value; + if (this.descriptors() >= this.maxDescriptors) return handle("queue_full").value; + const read = this.readSource(source, this.uploadCap(options.mimeType, kind)); + if ("reason" in read) return handle(read.reason).value; + const reserve = kind === "OwnedRecordingFile" ? CHUNK : read.size; + if (this.descriptors() >= this.maxDescriptors || this.bytes + reserve > this.maxBytes) return handle("queue_full").value; + if (deferred && this.deferredBytes() + reserve > this.deferredCap()) return handle("queue_full").value; + this.bytes += reserve; + const fields: AttachmentFields = { messageIndex: options.messageIndex, partIndex: options.partIndex, role: options.role, channel: options.channel, + conversationId: options.conversationId, turnId: options.turnId, toolCallId: options.toolCallId }; + return this.enqueueUpload(facts, fields, correlation!, read.size, read.content, reserve, read.owned); + } catch { return handle("source_unavailable").value; } + } + + // --- consumed streams ------------------------------------------------------------------------------ + startAttachmentStream(target: AttachmentTarget, options: AttachmentStreamOptions): AttachmentStream { + try { + // The reservation is min(requested, negotiated single-upload cap) (design 0.7): a server whose max_upload_bytes is + // below the adapter default still admits the stream, and a body over the effective cap is discarded whole (too_large). + let maxBytes = options.maxBytes; + if (this.session && Number.isInteger(maxBytes) && maxBytes > 0) maxBytes = Math.min(maxBytes, this.uploadCap(options.mimeType, "Bytes")); + const ticket = this.preflight(target, { mimeType: options.mimeType, direction: options.direction, maxBytes, + purpose: options.purpose, capture: options.capture, operation: "observe_consumed", sourceKind: "Bytes" }); + return ticket.startStream(maxBytes === options.maxBytes ? options : { ...options, maxBytes }); + } catch { return this.deniedStream("source_unavailable"); } + } + /** @internal */ + commitStream(ticket: AttachmentTicket, options: AttachmentStreamOptions): AttachmentStream { + const taken = this.takeTicket(ticket); + if (taken.reason) return this.deniedStream(taken.reason); + const reserved = taken.reservation!.bytes; + const facts = ticket.facts!; + const fail = (reason: AttachmentAdmission) => { this.releaseBytes(reserved); this.notify(); return this.deniedStream(reason); }; + // Narrowing only: a stream cannot change MIME, direction, purpose or source kind, nor exceed the ticket budget. + if (!options || options.mimeType !== facts.mimeType || options.direction !== facts.direction || options.purpose !== facts.purpose || facts.sourceKind !== "Bytes") return fail("conflict" as AttachmentAdmission); + if (!Number.isInteger(options.maxBytes) || options.maxBytes <= 0 || options.maxBytes > ticket.maxBytes) return fail("too_large"); + const fields: AttachmentFields = { messageIndex: options.messageIndex, partIndex: options.partIndex, role: options.role, channel: options.channel, + conversationId: options.conversationId, turnId: options.turnId, toolCallId: options.toolCallId }; + if (!this.validFields(fields)) return fail("policy_denied"); + const keep = options.maxBytes; this.releaseBytes(reserved - keep); + const state: StreamState & StreamExtra = { reason: "queued", attachmentId: uuid(), occurrenceId: uuid(), chunks: [], length: 0, maxBytes: options.maxBytes, + reserved: keep, lastWrite: this.clock.now(), idleTtl: bounded(options.idleTtlMs, this.streamIdleTtl, 300000), epoch: this.epoch, open: true, + facts, fields, correlation: ticket.correlation!, purpose: options.purpose }; + this.streams.add(state); + const stream = new AttachmentConsumedStream(this, state); + this.finalizer()?.register(stream, () => { if (state.open) this.streamAbort(state, "incomplete"); }, state); + return stream; + } + /** @internal */ + streamWrite(state: StreamState, chunk: Uint8Array): boolean { + const s = state as StreamState & StreamExtra; + if (!s.open || !(chunk instanceof Uint8Array)) return false; + if (s.epoch !== this.epoch || this.closed) { + const reason = this.recheck(s.facts); + if (reason) { this.streamAbort(s, "policy_denied"); return false; } + } + if (s.length + chunk.byteLength > s.maxBytes) { this.streamAbort(s, "too_large"); return false; } + // Copy in bounded slices; the caller may reuse a BYOB buffer immediately. + for (let offset = 0; offset < chunk.byteLength; offset += CHUNK) s.chunks.push(chunk.slice(offset, Math.min(chunk.byteLength, offset + CHUNK))); + s.length += chunk.byteLength; s.lastWrite = this.clock.now(); + return true; + } + /** @internal */ + streamFinish(state: StreamState): AttachmentHandle { + const s = state as StreamState & StreamExtra; + if (!s.open) return s.handle ?? handle(s.reason === "queued" ? "incomplete" : s.reason).value; + s.open = false; this.streams.delete(s); + try { this.registry?.unregister(s); } catch { /* bounded */ } + const chunks = s.chunks; s.chunks = []; + const release = (reason: AttachmentAdmission) => { this.releaseBytes(s.reserved); s.reserved = 0; s.reason = reason; s.handle = handle(reason).value; this.notify(); return s.handle; }; + if (!s.length) return release("incomplete"); + const reason = this.recheck(s.facts); + if (reason) return release(reason); + this.releaseBytes(s.reserved - s.length); const bytes = s.length; s.reserved = 0; + const content = attachmentChunkedSource(new Uint8Array(0), chunks); + s.handle = this.enqueueUploadWithIds(s, content, bytes); + return s.handle; + } + private enqueueUploadWithIds(s: StreamState & StreamExtra, content: AttachmentContentSource, bytes: number, + result: HandleControl = handle("queued"), parked?: AttachmentHandle): AttachmentHandle { + const ids = { attachmentId: s.attachmentId!, occurrenceId: s.occurrenceId! }; + const bound = this.boundTarget(s.correlation); + if (!bound) { + const value = Object.freeze({ attachmentId: ids.attachmentId, occurrenceId: ids.occurrenceId, admission: "queued" as AttachmentAdmission, completion: result.value.completion }); + let data: AttachmentContentSource | undefined = content; + this.park(s.correlation as AttachmentDeferredCorrelation, s.facts, bytes, result, + () => { if (data) this.enqueueUploadWithIds(s, data, bytes, result, value); }, () => { data = undefined; }); + return value; + } + const value = parked ?? Object.freeze({ attachmentId: ids.attachmentId, occurrenceId: ids.occurrenceId, admission: "queued" as AttachmentAdmission, completion: result.value.completion }); + if (!AttachmentManager.sampledTarget(bound)) { + // Backstop: a stream opened against a deferred target that bound to an unsampled span uploads nothing. + this.releaseBytes(bytes); this.dropped++; result.finish("unsampled"); this.notify(); + return value; + } + s.correlation = bound; + const identity = s.correlation.snapshot()!; + const request = this.uploadRequest(s.facts, s.fields, identity, ids, bytes); + const body = JSON.stringify(request); + let data: AttachmentContentSource | undefined = content; + this.enqueue({ lane: ids.occurrenceId, bytes, controller: new AbortController(), denied: () => this.recheck(s.facts), + fail: reason => { data = undefined; result.finish(reason); }, + run: async signal => { + try { + if (!data) { result.finish("incomplete"); return; } + const outcome = await this.runUpload({ body, occurrenceId: ids.occurrenceId, declaredBytes: bytes, source: data, recheck: () => this.recheck(s.facts, { ignoreSessionExpiry: true }) }, signal); + result.finish(outcome.status); if (outcome.status !== "validating") this.dropped++; + } finally { data = undefined; } + } }); + // Projected only at finish, only within the cap, only while the span still records. + s.correlation.project(this.reference(request, s.fields), this.sessionId, this.nonce); + return value; + } + /** @internal */ + streamAbort(state: StreamState, reason: "incomplete" | "policy_denied" | "too_large" | "correlation_unavailable" | "unsampled"): void { + if (!state.open) return; + state.open = false; this.streams.delete(state as StreamState & StreamExtra); + try { this.registry?.unregister(state); } catch { /* bounded */ } + state.chunks = []; state.length = 0; + this.releaseBytes(state.reserved); state.reserved = 0; + state.reason = reason; state.handle = handle(reason).value; + this.notify(); + } + + // --- recordings ------------------------------------------------------------------------------------ + private recordingFacts(options: AttachmentRecordingOptions, direction: "input" | "output" | "mixed"): AdmissionFacts { + return { mimeType: "audio/wav", direction, purpose: options.purpose ?? "conversation_recording", + capture: options.capture ? { ...options.capture } : undefined, operation: options.operation, sourceKind: "Bytes" }; + } + private trackDenials(options: AttachmentRecordingOptions, admission = true): { reason?: AttachmentAdmission; tracks?: (AttachmentAdmission | undefined)[] } { + if (options.layout !== "separate_tracks") { + const reason = this.recheck(this.recordingFacts(options, "mixed"), { admission }); + return reason ? { reason } : { tracks: [] }; + } + const tracks = options.tracks.map(track => this.recheck(this.recordingFacts(options, track.direction), { admission })); + if (tracks.every(Boolean)) return { reason: tracks[0] }; + return { tracks }; + } + private recordingLimit(): number { return Math.min(this.session?.limits.active_recordings ?? 4, 4); } + startRecording(target: AttachmentTarget, options: AttachmentRecordingOptions): AttachmentRecordingWriter { + const inert = (reason: AttachmentAdmission) => new AttachmentRecording({ host: this, options, admission: reason, readmit: () => ({ reason }), stamp: () => undefined, close: () => undefined }); + try { + this.enter(); + if (!attachmentRecordingShape(options)) { + const first = this.recheck(this.recordingFacts(options ?? ({} as AttachmentRecordingOptions), "mixed"), { admission: true }); + return inert(first ?? "policy_denied"); + } + const pending = this.enabled && attachmentEnabled(this.config, this.env()) && !this.closed && !this.stoppingFlag && this.runtimeState() === "bootstrap_pending"; + let trackDenials: (AttachmentAdmission | undefined)[] | undefined; + if (!pending) { + const evaluated = this.trackDenials(options); + if (evaluated.reason) return inert(evaluated.reason); + trackDenials = evaluated.tracks; + } + // Fail at start, not at the manifest: a legacy wire cannot publish, so no bytes are retained or uploaded. + if (this.legacyWire) return inert("unsupported_version"); + const correlation = this.resolveCorrelation(target); + const identity = correlation?.snapshot(); + if (!correlation || !identity) return inert("correlation_unavailable"); + if (!identity.sampled) return inert("unsampled"); + if (this.descriptors() >= this.maxDescriptors) return inert("queue_full"); + if (!pending && this.recordingSlots >= this.recordingLimit()) return inert("queue_full"); + let slot = false; + const take = () => { if (!slot) { slot = true; this.recordingSlots++; } }; + if (!pending) take(); + const writer: AttachmentRecording = new AttachmentRecording({ host: this, options, identity, admission: pending ? "bootstrap_pending" : "queued", trackDenials, + readmit: () => { + const again = this.trackDenials(options, false); + if (again.reason) return { reason: again.reason }; + if (!slot && this.recordingSlots >= this.recordingLimit()) return { reason: "queue_full" }; + take(); + return { reason: "queued", trackDenials: again.tracks }; + }, + stamp: () => correlation.prepare(this.sessionId, this.nonce), + close: () => { if (slot) { slot = false; this.recordingSlots--; } } }); + if (writer.reason === "queued" || writer.reason === "bootstrap_pending") this.recordings.set(writer.recordingId, writer); + return writer; + } catch { return inert("source_unavailable"); } + } + + // --- owned recording files --------------------------------------------------------------------------- + attachOwnedRecording(target: AttachmentTarget, source: { kind: "OwnedRecordingFile"; lease: AttachmentOwnedFileRef }, options: AttachmentOwnedRecordingOptions): AttachmentOwnedRecordingHandle { + const denied = (reason: AttachmentAdmission): AttachmentOwnedRecordingHandle => { + const completion = Promise.resolve(Object.freeze({ status: reason }) as AttachmentCompletion); + return Object.freeze({ tier: "denied" as const, reason, completion, wait: () => completion, cancel: () => undefined }); + }; + try { + this.enter(); + const record = source && source.kind === "OwnedRecordingFile" ? attachmentOwnedFile(source.lease) : undefined; + const purpose = options?.purpose ?? "conversation_recording"; + const facts: AdmissionFacts = { mimeType: options?.mimeType, direction: options?.direction, purpose, capture: options?.capture ? { ...options.capture } : undefined, + operation: "import_existing", sourceKind: "OwnedRecordingFile" }; + const { reason, correlation, identity } = this.admitTarget(target, facts); + if (reason) return denied(reason); + if (!record || record.closed()) return denied("source_unavailable"); + if (purpose !== "conversation_recording" && purpose !== "turn_recording") return denied("policy_denied"); + if (options.conversationId !== undefined && !TOKEN.test(options.conversationId)) return denied("policy_denied"); + const size = record.identity.size; + const singleCap = Math.min(this.session!.limits.max_upload_bytes, 50 * MiB); + const wav = options.mimeType === "audio/wav" && (size > singleCap || !!options.wavTracks); + if (!Number.isInteger(size) || size <= 0 || (!wav && size > singleCap)) return denied("too_large"); + if (this.descriptors() >= this.maxDescriptors || this.bytes + CHUNK > this.maxBytes) return denied("queue_full"); + if (wav && options.wavTracks && !this.validWavTracks(options.wavTracks)) return denied("policy_denied"); + if (wav && this.legacyWire) return denied("unsupported_version"); + this.bytes += CHUNK; + return this.ownedTransfer(record, facts, correlation!, identity!, options, wav, singleCap); + } catch { return denied("source_unavailable"); } + } + private validWavTracks(tracks: readonly { channels: readonly number[]; direction: string; role: string }[]): boolean { + return Array.isArray(tracks) && tracks.length >= 1 && tracks.length <= 32 && tracks.every(t => Array.isArray(t.channels) && t.channels.length >= 1 && + ["input", "output", "mixed"].includes(t.direction) && ["user", "assistant", "mixed", "unknown"].includes(t.role)); + } + private ownedTransfer(record: AttachmentOwnedFileRecord, facts: AdmissionFacts, correlation: AttachmentCorrelation, identity: { traceId: string; spanId: string }, + options: AttachmentOwnedRecordingOptions, wav: boolean, singleCap: number): AttachmentOwnedRecordingHandle { + let tier: "single" | "wav_recording" | "denied" = wav ? "wav_recording" : "single"; + let recordingId: string | undefined; + let resolve!: (value: AttachmentCompletion) => void; + const completion = new Promise(done => { resolve = done; }); + let settled = false; + const finish = (status: AttachmentCompletion["status"]) => { if (!settled) { settled = true; resolve(Object.freeze({ status })); } }; + const controller = new AbortController(); + let lease: AttachmentOwnedLease | undefined; + let reservation = CHUNK; + const releaseSlice = () => { this.releaseBytes(reservation); reservation = 0; }; + const closeLease = async () => { const open = lease; lease = undefined; if (open) await open.close().catch(() => undefined); }; + correlation.prepare(this.sessionId, this.nonce); + const fields: AttachmentFields = { conversationId: options.conversationId }; + const job: CoreJob = { + lane: uuid(), bytes: 0, controller, + denied: () => this.recheck(facts), + fail: reason => { finish(reason); releaseSlice(); void closeLease(); }, + run: async signal => { + try { + lease = await raceAbort(record.open(), signal); + if (tier === "wav_recording") { + const layout = await this.wavLayout(lease, options, singleCap); + if (typeof layout === "string") { + if (layout === "single") tier = "single"; + else { tier = "denied"; finish(layout); releaseSlice(); await closeLease(); return; } + } else { + // Segmented recording from lease ranges; the outer job returns so it never holds a worker. + const writer = this.ownedWavRecording(identity, correlation, options, layout, lease); + if (typeof writer === "string") { tier = "denied"; finish(writer); releaseSlice(); await closeLease(); return; } + recordingId = writer.recordingId; + const openLease = lease; lease = undefined; + void writer.completeFile(controller.signal).then(status => finish(status), () => finish("incomplete")) + .finally(() => { releaseSlice(); void openLease.close().catch(() => undefined); }); + return; + } + } + const opened = lease; + const request = this.uploadRequest({ ...facts }, fields, identity, { attachmentId: uuid(), occurrenceId: uuid() }, opened.size); + const outcome = await this.runUpload({ body: JSON.stringify(request), occurrenceId: request.occurrence_id, declaredBytes: opened.size, + source: { size: opened.size, read: (o, n) => opened.read(o, n), check: () => opened.check() }, recheck: () => this.recheck(facts, { ignoreSessionExpiry: true }) }, signal); + finish(outcome.status); + releaseSlice(); await closeLease(); + } catch (error) { finish(codeOf(error)); releaseSlice(); await closeLease(); } + }, + }; + this.enqueue(job); + const cancel = () => { controller.abort(); finish("incomplete"); }; + const value: AttachmentOwnedRecordingHandle = { + get tier() { return tier; }, + reason: "queued", + get recordingId() { return recordingId; }, + completion, + wait: (timeoutMs?: number) => new Promise(done => { + const timer = setTimeout(() => { cancel(); done(Object.freeze({ status: "incomplete" as const })); }, bounded(timeoutMs, this.finalizationTimeoutMs, 30000)); + void completion.then(result => { clearTimeout(timer); done(result); }); + }), + cancel, + }; + return value; + } + /** Parses at most 4 KiB of a WAV header through the lease, after admission (design 4.6). */ + private async wavLayout(lease: AttachmentOwnedLease, options: AttachmentOwnedRecordingOptions, singleCap: number): + Promise<{ rate: number; channels: number; dataOffset: number; dataBytes: number; layout: AttachmentRecordingOptions["layout"]; tracks: AttachmentRecordingOptions["tracks"] } | "single" | AttachmentAdmission> { + const fallback = lease.size <= singleCap ? "single" as const : "too_large" as const; + const head = await lease.read(0, Math.min(4096, lease.size)); + const view = new DataView(head.buffer, head.byteOffset, head.byteLength); + const tag = (offset: number) => String.fromCharCode(...head.subarray(offset, offset + 4)); + if (head.byteLength < 44 || tag(0) !== "RIFF" || tag(8) !== "WAVE") return fallback; + let offset = 12; let fmt: { format: number; channels: number; rate: number; bits: number; sub?: string } | undefined; + while (offset + 8 <= head.byteLength) { + const id = tag(offset); const size = view.getUint32(offset + 4, true); const body = offset + 8; + if (id === "fmt ") { + if (size < 16 || body + size > head.byteLength) return fallback; + const format = view.getUint16(body, true); + fmt = { format, channels: view.getUint16(body + 2, true), rate: view.getUint32(body + 4, true), bits: view.getUint16(body + 14, true) }; + if (format === 0xfffe) { + if (size < 40) return fallback; + fmt.sub = Array.from(head.subarray(body + 24, body + 40)).map(b => b.toString(16).padStart(2, "0")).join(""); + } + } else if (id === "data") { + if (!fmt) return fallback; + const pcm = fmt.format === 1 || (fmt.format === 0xfffe && fmt.sub === "0100000000001000800000aa00389b71"); + if (!pcm || fmt.bits !== 16 || fmt.channels < 1 || fmt.channels > 32 || fmt.rate < 8000 || fmt.rate > 192000) return fallback; + if (body + size !== lease.size || size % (fmt.channels * 2) !== 0 || size === 0) return fallback; + const frames = size / (fmt.channels * 2); + if (lease.size > Math.min(this.session!.limits.recording_bytes, 1073741824) || + Math.ceil(frames * 1000 / fmt.rate) > Math.min(this.session!.limits.recording_duration_ms, 7200000)) return "too_large"; + const all = Array.from({ length: fmt.channels }, (_, i) => i); + const given = options.wavTracks; + let layout: AttachmentRecordingOptions["layout"] = "mixed"; + let tracks: AttachmentRecordingOptions["tracks"] = [{ channels: all, direction: "mixed", role: fmt.channels === 1 ? (given?.[0]?.role ?? "unknown") : "unknown" }]; + if (fmt.channels === 2 && given && given.length === 2 && given[0].channels.length === 1 && given[0].channels[0] === 0 && + given[1].channels.length === 1 && given[1].channels[0] === 1) { layout = "stereo"; tracks = given.map(t => ({ channels: [...t.channels], direction: t.direction, role: t.role })); } + return { rate: fmt.rate, channels: fmt.channels, dataOffset: body, dataBytes: size, layout, tracks }; + } + offset = body + size + (size % 2); + } + return fallback; + } + private ownedWavRecording(identity: { traceId: string; spanId: string }, correlation: AttachmentCorrelation, options: AttachmentOwnedRecordingOptions, + layout: { rate: number; channels: number; dataOffset: number; dataBytes: number; layout: AttachmentRecordingOptions["layout"]; tracks: AttachmentRecordingOptions["tracks"] }, + lease: AttachmentOwnedLease): AttachmentRecording | AttachmentAdmission { + if (this.recordingSlots >= this.recordingLimit()) return "queue_full"; + this.recordingSlots++; + let slot = true; + const recordingOptions: AttachmentRecordingOptions = { format: { sampleRateHz: layout.rate }, layout: layout.layout, tracks: layout.tracks, + operation: "observe_consumed", purpose: options.purpose ?? "conversation_recording", capture: options.capture, conversationId: options.conversationId, windowSegments: 1 }; + const writer = new AttachmentRecording({ host: this, options: recordingOptions, identity, admission: "queued", sourceKind: "OwnedRecordingFile", + readmit: () => ({ reason: "queued" }), stamp: () => correlation.prepare(this.sessionId, this.nonce), + close: () => { if (slot) { slot = false; this.recordingSlots--; } } }); + if (writer.reason !== "queued") return writer.reason; + const frameBytes = layout.channels * 2; + const limits = this.session!.limits; + const F = attachmentSegmentFrames(layout.rate, frameBytes, limits.segment_duration_ms, Math.min(limits.segment_bytes, limits.max_upload_bytes)); + const total = layout.dataBytes / frameBytes; + const segments: RecordingFileSegment[] = []; + for (let start = 0; start < total; start += F) { + const end = Math.min(total, start + F); const bytes = (end - start) * frameBytes; + const header = attachmentWavHeader(layout.channels, layout.rate, bytes); const base = layout.dataOffset + start * frameBytes; + segments.push({ track: 0, startFrame: start, endFrame: end, source: { + size: ATTACHMENT_WAV_HEADER_BYTES + bytes, + read: async (offset, length) => { + // WavSegmentSource: a synthesized header then a pread of the frame range; never materialized. + const out = new Uint8Array(Math.min(length, ATTACHMENT_WAV_HEADER_BYTES + bytes - offset)); let written = 0; + if (offset < ATTACHMENT_WAV_HEADER_BYTES) { const part = header.subarray(offset, Math.min(ATTACHMENT_WAV_HEADER_BYTES, offset + out.byteLength)); out.set(part); written = part.byteLength; } + if (written < out.byteLength) { + const from = Math.max(0, offset - ATTACHMENT_WAV_HEADER_BYTES); + const part = await lease.read(base + from, out.byteLength - written); + if (part.byteLength !== out.byteLength - written) throw new AttachmentTransportError("source_unavailable"); + out.set(part, written); + } + return out; + }, + check: () => lease.check(), + } }); + } + this.recordings.set(writer.recordingId, writer); + writer.loadFileSegments(segments, total); + return writer; + } + + // --- completed-recording import --------------------------------------------------------------------- + async importCompletedRecording(target: AttachmentTarget, request: AttachmentCompletedRecordingRequest, options: AttachmentImportOptions): Promise { + const finished = (reason: AttachmentAdmission | AttachmentCompletion["status"]): AttachmentImportHandle => { + const completion = Promise.resolve(Object.freeze({ status: reason }) as AttachmentCompletion); + return Object.freeze({ reason, completion, status: async () => ({ state: "failed", reason: String(reason) }) }) as AttachmentImportHandle; + }; + try { + this.enter(); + const direction = request?.direction ?? "mixed"; + const facts: AdmissionFacts = { mimeType: "audio/ogg", direction, purpose: "conversation_recording", capture: options?.capture ? { ...options.capture } : undefined, + operation: "import_existing", sourceKind: "ProviderFile", importSource: true }; + const { reason, correlation, identity } = this.admitTarget(target, facts); + // Denied preflight never calls the resolver. + if (reason) return finished(reason); + if (this.legacyWire) return finished("unsupported_version"); + if (!validAttachmentImportRequest(request) || !options?.resolver || typeof options.resolver.resolve !== "function") return finished("policy_denied"); + if (this.descriptors() >= this.maxDescriptors || this.bytes + 1 > this.maxBytes) return finished("queue_full"); + const id = ++this.nextReservation; this.bytes += 1; this.reservations.set(id, { bytes: 1, expiresAt: Infinity }); + const release = () => this.releaseReservation(id); + correlation!.prepare(this.sessionId, this.nonce); + const timeout = bounded(options.timeoutMs, 30000, 30000); + const controller = new AbortController(); + let resolved: unknown; + try { + resolved = await new Promise(resolve => { + const timer = setTimeout(() => { controller.abort(); resolve("source_unavailable"); }, timeout); + Promise.resolve().then(() => options.resolver.resolve({ provider: request.provider, providerRef: request.providerRef, direction }, controller.signal)) + .then(value => { clearTimeout(timer); resolve(value); }, () => { clearTimeout(timer); resolve("source_unavailable"); }); + }); + } catch { resolved = "source_unavailable"; } + const invalid = attachmentResolvedRecordingDenial(resolved, this.session?.limits as { recording_bytes?: number; recording_duration_ms?: number } | undefined); + if (invalid) { release(); return finished(invalid); } + const result = resolved as import("./types").AttachmentResolvedCompletedRecording; + const finalDirection = attachmentImportDirection(direction, result.direction); + if (!finalDirection) { release(); return finished("policy_denied"); } + const sourceKind: AttachmentSourceKind = result.source.kind === "ProviderFile" ? "ProviderFile" : "RemoteUrl"; + const finalFacts: AdmissionFacts = { ...facts, mimeType: result.mimeType, direction: finalDirection, sourceKind }; + const again = this.recheck(finalFacts, { admission: true }); + if (again) { release(); return finished(again); } + const recordingId = uuid(); + const body = JSON.stringify({ mode: "completed_recording", recording: { schema_version: 1, + association: { capture_session_id: this.sessionId, correlation_nonce: this.nonce, parent: { trace_id: attachmentTraceUuid(identity!.traceId), span_id: identity!.spanId } }, + policy_version: this.session!.policy.version, deletion_generation: 0, mime_type: result.mimeType, direction: finalDirection, source_kind: sourceKind, + recording_id: recordingId, modality: "audio", declared_bytes: result.declaredBytes, declared_duration_ms: result.declaredDurationMs }, + source: result.source.kind === "ProviderFile" ? { kind: "ProviderFile", connection_id: result.source.connectionId, registered_output_id: result.source.registeredOutputId } : + { kind: "RemoteUrlEnvelope", envelope_id: result.source.envelopeId } }); + if (!validAttachmentWire("RecordingImportRequest", JSON.parse(body))) { release(); return finished("unsupported_version"); } + let state: AttachmentAdmission | Exclude = "queued"; + let jobId: string | undefined; + const done = handle("queued"); + this.reservations.delete(id); // the descriptor becomes the job + this.enqueue({ lane: recordingId, bytes: 1, controller: new AbortController(), denied: () => this.recheck(finalFacts), + fail: reason => { state = reason; done.finish(state); }, + run: async signal => { + try { + const job = await this.attempt(s => this.wire!.submitImport(this.session!, body, attachmentIdempotencyKey("import", recordingId), s), signal, + () => { const r = this.recheck(finalFacts); if (r) throw new AttachmentTransportError(r); }); + if (!job || job.recordingId !== recordingId || !validAttachmentUuid(job.jobId) || job.state !== "importing") throw new AttachmentTransportError("unsupported_version"); + jobId = job.jobId; state = "queued"; done.finish("importing"); + } catch (error) { state = codeOf(error); done.finish(state); this.dropped++; } + } }); + const manager = this; + return { + get reason() { return state; }, + recordingId, + get jobId() { return jobId; }, + completion: done.value.completion, + async status(timeoutMs?: number) { + if (!jobId || !manager.session || !manager.wire) return { state: "failed", reason: String(state === "queued" ? "incomplete" : state) }; + const c = new AbortController(); + const timer = setTimeout(() => c.abort(), bounded(timeoutMs, 2000, 30000)); unref(timer); + try { return await raceAbort(manager.wire.importStatus(manager.session, jobId, c.signal), c.signal); } + catch (error) { return { state: "failed", reason: codeOf(error) }; } + finally { clearTimeout(timer); } + }, + }; + } catch { return finished("source_unavailable"); } + } + + // --- queue, workers and transfers ---------------------------------------------------------------------- + /** @internal */ + enqueue(job: CoreJob): void { + let lane = this.lanes.get(job.lane); + if (!lane) { lane = new Lane(job.lane); this.lanes.set(job.lane, lane); this.order.push(lane); } + lane.jobs.push(job); this.queued++; + // Avoid even synchronous user transport execution in the caller's callback. + void Promise.resolve().then(() => this.pump()); + } + private nextLane(): Lane | undefined { + const count = this.order.length; + for (let i = 0; i < count; i++) { + const index = (this.cursor + i) % count; const lane = this.order[index]; + if (!lane.busy && lane.jobs.length) { this.cursor = index + 1; return lane; } + } + return undefined; + } + private dropLane(lane: Lane): void { + if (lane.busy || lane.jobs.length) return; + this.lanes.delete(lane.id); this.order = this.order.filter(item => item !== lane); + if (this.cursor > this.order.length) this.cursor = 0; + } + private endJob(job: CoreJob): void { + if (job.settled) return; + job.settled = true; this.releaseBytes(job.bytes); job.bytes = 0; + } + private pump(): void { + // Worker wake is a renewal trigger (design 3.1): recording-only workloads never pass through preflight. + this.maybeRenew(); + while (this.active.size < this.workers) { + const lane = this.nextLane(); + if (!lane) break; + const job = lane.jobs.shift()!; this.queued--; + if (job.controller.signal.aborted) { job.fail("incomplete"); this.endJob(job); this.dropLane(lane); this.notify(); continue; } + lane.busy = true; this.active.add(job); + void (async () => { try { await job.run(job.controller.signal); } catch { job.fail("incomplete"); } })().finally(() => { + this.endJob(job); this.active.delete(job); lane.busy = false; this.dropLane(lane); + this.notify(); this.pump(); + }); + } + } + private async attempt(operation: (signal: AbortSignal) => Promise, parent: AbortSignal, guard: () => void = () => undefined, attempts = 3, + timeoutMs = this.requestTimeout): Promise { + for (let attempt = 0; ; attempt++) { + if (parent.aborted) throw new AttachmentTransportError("incomplete"); + guard(); + const controller = new AbortController(); + const abort = () => controller.abort(); + parent.addEventListener("abort", abort, { once: true }); + const timer = setTimeout(abort, timeoutMs); unref(timer); + try { + // Raced against the abort signal: a transport that ignores its signal can never hold shutdown. + const result = await raceAbort(Promise.resolve().then(() => operation(controller.signal)), controller.signal); + if (controller.signal.aborted) throw new AttachmentTransportError("incomplete", !parent.aborted); + return result; + } catch (raw) { + const timedOut = controller.signal.aborted && !parent.aborted; + const error = raw instanceof AttachmentTransportError ? (raw.code === "failed" ? new AttachmentTransportError("incomplete", raw.retryable, raw.retryAfterMs, raw.httpStatus) : raw) + : new AttachmentTransportError("incomplete", timedOut); + const retryable = error.retryable || (timedOut && error.code === "incomplete"); + if (!retryable || attempt + 1 >= attempts || parent.aborted) throw error; + const backoff = (0.5 + Math.random() * 0.5) * 100 * 2 ** (attempt + 1); + await pause(Math.max(Math.min(error.retryAfterMs, 5000), backoff), parent); + } finally { clearTimeout(timer); parent.removeEventListener("abort", abort); } + } + } + /** @internal Initialize -> content -> complete, with capability renewal and a best-effort abort (design 2.7, 3.2). */ + async runUpload(spec: UploadSpec, signal: AbortSignal): Promise { + const wire = this.wire!; + const outcome: UploadOutcome = { status: "incomplete", body: spec.body, renewals: 0 }; + let request: AttachmentUploadRequest; + try { request = JSON.parse(spec.body); } catch { outcome.status = "unsupported_version"; return outcome; } + // Closed-schema check before any request: never a role, channel, URL, path or credential (T-CH-01). + if (!validAttachmentWire("UploadRequest", request)) { outcome.status = "unsupported_version"; return outcome; } + const guard = () => { + if (signal.aborted) throw new AttachmentTransportError("incomplete"); + const reason = spec.recheck(); if (reason) throw new AttachmentTransportError(reason); + if (!this.session || request.policy_version !== this.session.policy.version) throw new AttachmentTransportError("policy_denied"); + }; + try { + const live = () => { guard(); if (this.runtimeState() === "expired") throw new AttachmentTransportError("expired"); }; + const upload = await this.attempt(s => wire.initialize(this.session!, spec.body, attachmentIdempotencyKey("upload", spec.occurrenceId), s), signal, live); + if (!validAttachmentUpload(upload, request, this.clock.wall())) throw new AttachmentTransportError("unsupported_version"); + outcome.upload = upload; + } catch (error) { outcome.status = codeOf(error); return outcome; } + const fresh = async () => { + if (this.clock.wall() >= Date.parse(outcome.upload!.expires_at) - SKEW && !(await this.renewUpload(outcome, signal))) throw new AttachmentTransportError("expired"); + }; + const withCapability = async (op: (s: AbortSignal) => Promise, timeout: number): Promise => { + for (let renewed = false; ;) { + try { return await this.attempt(async s => { await fresh(); return op(s); }, signal, guard, 3, timeout); } + catch (error) { + // One renewal after a 401 on a capability route, then one more try. + if (error instanceof AttachmentTransportError && error.httpStatus === 401 && !renewed) { + renewed = true; + if (await this.renewUpload(outcome, signal)) continue; + throw new AttachmentTransportError("expired"); + } + throw error; + } + } + }; + try { + const transferTimeout = Math.min(this.contentTimeout, (this.session?.limits.transfer_timeout_seconds ?? 600) * 1000); + await withCapability(s => wire.putContent(this.session!, outcome.upload!, spec.source, s, this.pacer), transferTimeout); + if (spec.beforeComplete) await spec.beforeComplete(outcome, signal); + await withCapability(s => wire.complete(this.session!, outcome.upload!, attachmentIdempotencyKey("complete", outcome.upload!.upload_id), s), this.requestTimeout); + guard(); + outcome.status = "validating"; + } catch (error) { + outcome.status = codeOf(error); + // Exactly one best-effort abort after a terminal failure that followed a successful initialize. + const upload = outcome.upload; + if (upload && this.session) { + const c = new AbortController(); const timer = setTimeout(() => c.abort(), this.requestTimeout); unref(timer); + try { await raceAbort(wire.abort(this.session, upload, attachmentIdempotencyKey("abort", upload.upload_id), c.signal), c.signal); } + catch { /* best effort */ } finally { clearTimeout(timer); } + } + outcome.upload = undefined; + } + return outcome; + } + /** @internal Replay initialize with the original identity for a fresh capability (BR-1). */ + async renewUpload(outcome: UploadOutcome, signal: AbortSignal): Promise { + const upload = outcome.upload; + if (!upload || !this.session || !this.wire || outcome.renewals >= 3 || signal.aborted) return false; + outcome.renewals++; + try { + const next = await this.attempt(s => this.wire!.renewUpload(this.session!, upload, outcome.body, s), signal, () => undefined, 1); + if (!next || next.upload_id !== upload.upload_id || next.attachment_id !== upload.attachment_id || next.occurrence_id !== upload.occurrence_id || + next.reserved_bytes !== upload.reserved_bytes || next.write_capability === upload.write_capability || + !(Date.parse(next.expires_at) > Date.parse(upload.expires_at))) return false; + outcome.upload = next; + return true; + } catch { return false; } + } + /** @internal Manifest PUT authorized by a segment capability only; a 409 on a replayed revision is success (BR-5). */ + async putManifest(capability: string, recordingId: string, body: string, key: string, signal: AbortSignal, replay: boolean): Promise { + if (!this.wire || !this.session) return "incomplete"; + let tries = 0; + try { + await this.attempt(async s => { + const n = tries++; + try { await this.wire!.putManifest(this.session!, capability, recordingId, body, key, s); } + catch (error) { if ((n > 0 || replay) && error instanceof AttachmentTransportError && error.code === "conflict") return; throw error; } + }, signal, () => undefined, 3, this.manifestTimeoutMs); + return "ok"; + } catch (error) { return codeOf(error); } + } + private drop(job: CoreJob, reason: AttachmentCompletion["status"]): void { this.dropped++; job.fail(reason); } + + // --- deferred targets ------------------------------------------------------------------------------------ + /** @internal Called once by `AttachmentDeferredCorrelation.bind`. Synchronous; the transport runs only from the queue. */ + deferredBound(target: AttachmentDeferredCorrelation, spanOrContext: AttachmentSpan): void { + if (target.host !== this) return; + // One reference list per span: a span that already has a correlation keeps it, and the target forwards to it. + try { + const existing = this.correlations.get(spanOrContext); + if (existing && existing !== target) target.canonical = existing; + else this.correlations.set(spanOrContext, target); + } catch { /* identity is still bound; projection falls back to the target itself */ } + this.sweep(); // a capture past its deadline fails closed before anything else + const bound = target.canonical ?? target; + const identity = bound.snapshot(); + const late = this.closed || this.stoppingFlag || !!this.shutdownPromise || this.clock.now() >= target.expiresAt; + // Open streams and outstanding tickets on this target are refused now, not only captures already parked: once the + // identity is bound they are no longer `pending`, so no later sweep would see them. + const refused = late ? "correlation_unavailable" : identity?.sampled ? undefined : "unsampled"; + if (refused) { + for (const state of [...this.streams]) if (state.correlation === target) this.streamAbort(state, refused); + for (const [id, reservation] of this.reservations) if (reservation.deferred === target) this.releaseReservation(id); + } + const captures = [...this.deferredCaptures].filter(capture => capture.target === target); + if (!captures.length) { if (refused) this.notify(); return; } + for (const capture of captures) { + if (late) { capture.fail("correlation_unavailable"); continue; } + // Policy is evaluated again at bind against the CURRENT epoch, environment and session. + const reason = this.recheck(capture.facts); + if (reason) { capture.fail(reason); continue; } + if (!identity?.sampled) { capture.fail("unsampled"); continue; } + capture.start(bound); + } + this.notify(); + } + /** @internal Asked by `AttachmentDeferredCorrelation.bind` before it binds: the deadline uses this manager's clock. */ + deferredExpired(target: AttachmentDeferredCorrelation): boolean { + return target.host === this && this.clock.now() >= target.expiresAt; + } + /** @internal Called once by `AttachmentDeferredCorrelation.cancel`. */ + deferredCancelled(target: AttachmentDeferredCorrelation): void { + if (target.host !== this) return; + for (const capture of [...this.deferredCaptures]) if (capture.target === target) capture.fail("correlation_unavailable"); + for (const state of [...this.streams]) if (state.correlation === target) this.streamAbort(state, "correlation_unavailable"); + for (const [id, reservation] of this.reservations) if (reservation.deferred === target) this.releaseReservation(id); + this.notify(); + } + + // --- policy and lifecycle ------------------------------------------------------------------------------ + /** Tightening discards all queued data and cancels transfers; re-enable never resurrects it or turns a disabled manager on. */ + updatePolicy(update: Pick): void { + if (!this.enabled) return; + if (update.enabled === false) this.config.enabled = false; + if (update.traceConfig) this.config.traceConfig = { ...this.config.traceConfig, ...update.traceConfig }; + this.epoch++; + this.cancelDenied(); + } + /** A revoked capability/session/generation never resumes or retains capture data. */ + invalidate(reason: "deleted" | "expired" | "policy_denied" = "deleted"): void { + this.closedReason = reason; this.closed = true; this.bootController?.abort(); this.epoch++; + this.cancelDenied(); + for (const state of [...this.streams]) this.streamAbort(state, "policy_denied"); + for (const id of [...this.reservations.keys()]) this.releaseReservation(id); + for (const writer of [...this.recordings.values()]) writer.abort(); + } + private cancelDenied(): void { + for (const lane of [...this.order]) { + for (let index = lane.jobs.length - 1; index >= 0; index--) { + const job = lane.jobs[index]; const reason = job.denied(); + if (reason) { lane.jobs.splice(index, 1); this.queued--; this.endJob(job); this.drop(job, reason); } + } + this.dropLane(lane); + } + for (const job of this.active) { + const reason = job.denied(); + if (reason) { job.controller.abort(); this.drop(job, reason); } + } + for (const state of [...this.streams]) if (this.recheck(state.facts)) this.streamAbort(state, "policy_denied"); + for (const capture of [...this.deferredCaptures]) { const reason = this.recheck(capture.facts); if (reason) capture.fail(reason); } + this.notify(); + } + private deadline(promise: Promise, timeout: number, fallback: T): Promise { + return new Promise(resolve => { + // An explicit lifecycle wait owns a bounded grace period; background work stays unref'd. + const timer = setTimeout(() => resolve(fallback), timeout); + void promise.then(value => { clearTimeout(timer); resolve(value); }, () => { clearTimeout(timer); resolve(fallback); }); + }); + } + async flush(timeoutMs = this.shutdownTimeout): Promise { + if (!this.enabled) return Object.freeze({ unfinished: 0, dropped: 0 }); + this.enter(); + this.cancelDenied(); + if (this.queued || this.active.size) { + await new Promise(resolve => { + const done = () => { clearTimeout(timer); this.listeners.delete(changed); resolve(); }; + const changed = () => { if (!this.queued && !this.active.size) done(); }; + const timer = setTimeout(done, bounded(timeoutMs, this.shutdownTimeout, 30000)); + this.listeners.add(changed); changed(); + }); + } + return Object.freeze({ unfinished: this.queued + this.active.size + this.streams.size + this.recordings.size + this.deferredCaptures.size, dropped: this.dropped }); + } + shutdown(timeoutMs = this.shutdownTimeout): Promise { + if (!this.shutdownPromise) { + // Stop new admissions, but give accepted jobs and open recordings a bounded finalization window. + this.shutdownPromise = (async () => { + if (!this.enabled) return Object.freeze({ unfinished: 0, dropped: 0 }); + const budget = bounded(timeoutMs, this.shutdownTimeout, 30000); + const deadline = this.clock.now() + budget; + this.stoppingFlag = true; + const writers = [...this.recordings.values()]; + if (writers.length) await this.deadline(Promise.all(writers.map(writer => writer.shutdown(Math.max(1, deadline - this.clock.now())))), Math.max(1, deadline - this.clock.now()), []); + for (const state of [...this.streams]) this.streamAbort(state, "incomplete"); + // Captures still waiting for a span never upload after close. + for (const capture of [...this.deferredCaptures]) capture.fail("correlation_unavailable"); + for (const id of [...this.reservations.keys()]) this.releaseReservation(id); + const result = await this.flush(Math.max(1, Math.floor(deadline - this.clock.now()))); + // An earlier invalidate() keeps its reason (policy_denied / deleted / expired / scope_unresolved). + if (!this.closed) this.closedReason = "incomplete"; + this.closed = true; this.bootController?.abort(); + for (const lane of [...this.order]) { + for (const job of lane.jobs.splice(0)) { this.queued--; this.endJob(job); this.drop(job, "incomplete"); } + this.dropLane(lane); + } + for (const job of this.active) { job.controller.abort(); this.drop(job, "incomplete"); } + for (const writer of [...this.recordings.values()]) writer.abort(); + this.notify(); + return Object.freeze({ unfinished: result.unfinished, dropped: this.dropped }); + })(); + } + return this.shutdownPromise; + } + /** Aggregate diagnostics only; no per-source metadata, names, hashes, capabilities or exception text. */ + status(): AttachmentStatus { + if (this.enabled) this.enter(); + return Object.freeze({ state: this.runtimeState(), queued: this.queued, active: this.active.size, openTickets: this.reservations.size, + openStreams: this.streams.size, openRecordings: this.recordings.size, ownedBytes: this.bytes, dropped: this.dropped }); + } +} diff --git a/typescript/packages/fi-core/src/attachments/node-sources.ts b/typescript/packages/fi-core/src/attachments/node-sources.ts new file mode 100644 index 000000000..2743ee571 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/node-sources.ts @@ -0,0 +1,112 @@ +/** + * Node-only owned recording files (design section 4). Reachable only from `./attachments/node`. + * + * Guarantees, stated honestly (design 4.4): the root must be canonical (realpath equals the path) with no symlink + * component; every intermediate component is lstat'ed (a directory, not a symlink) immediately before the leaf is + * opened with O_NOFOLLOW; the opened handle must match the identified (dev, ino, size) and its realpath must equal the + * expected path. RESIDUAL RACE: an intermediate directory swapped to a symlink between the lstat and the open can + * redirect the leaf; the identity bind and the post-open realpath check are the mitigation. Windows is unsupported. + */ +import { constants, promises as fs } from "node:fs"; +import * as path from "node:path"; +import { registerAttachmentOwnedFile, type AttachmentOwnedLease } from "./owned"; +import { AttachmentTransportError } from "./transport-v1"; +import type { AttachmentFileIdentity, AttachmentOwnedFileRef, AttachmentRecordingFileScope } from "./types"; + +const unavailable = () => new AttachmentTransportError("source_unavailable"); +function validRelative(relativePath: string): string[] | undefined { + if (typeof relativePath !== "string" || !relativePath || relativePath.length > 4096 || relativePath.includes("\0") || path.isAbsolute(relativePath)) return undefined; + const parts = relativePath.split("/"); + if (parts.some(part => !part || part === "." || part === ".." || part.includes("\\"))) return undefined; + return parts; +} +async function noSymlinkDirectory(full: string): Promise { + const info = await fs.lstat(full); + if (info.isSymbolicLink() || !info.isDirectory()) throw unavailable(); +} +async function openLeaf(root: string, parts: readonly string[]) { + let current = root; + for (const part of parts.slice(0, -1)) { current = path.join(current, part); await noSymlinkDirectory(current); } + const full = path.join(root, ...parts); + const handle = await fs.open(full, constants.O_RDONLY | constants.O_NOFOLLOW | (constants.O_NONBLOCK ?? 0)); + try { + const stat = await handle.stat({ bigint: true }); + if (!stat.isFile()) throw unavailable(); + if (await fs.realpath(full) !== full) throw unavailable(); + return { handle, stat, full }; + } catch (error) { await handle.close().catch(() => undefined); throw error; } +} +class Scope implements AttachmentRecordingFileScope { + private closedFlag = false; + private readonly leases = new Set(); + constructor(private readonly root: string) {} + closed(): boolean { return this.closedFlag; } + async identify(relativePath: string): Promise { + const parts = validRelative(relativePath); + if (this.closedFlag || !parts) throw unavailable(); + const { handle, stat } = await openLeaf(this.root, parts).catch(() => { throw unavailable(); }); + await handle.close().catch(() => undefined); + if (stat.size > BigInt(Number.MAX_SAFE_INTEGER)) throw unavailable(); + return Object.freeze({ dev: stat.dev, ino: stat.ino, size: Number(stat.size) }); + } + source(relativePath: string, identity: AttachmentFileIdentity): { kind: "OwnedRecordingFile"; lease: AttachmentOwnedFileRef } { + const parts = validRelative(relativePath); + if (!parts || !identity || typeof identity.dev !== "bigint" || typeof identity.ino !== "bigint" || !Number.isSafeInteger(identity.size)) { + // A forged or invalid identity yields a ref that always fails closed. + return { kind: "OwnedRecordingFile", lease: registerAttachmentOwnedFile({ identity: { dev: BigInt(0), ino: BigInt(0), size: 0 }, open: async () => { throw unavailable(); }, closed: () => true }) }; + } + const fixed = Object.freeze({ dev: identity.dev, ino: identity.ino, size: identity.size }); + const ref = registerAttachmentOwnedFile({ + identity: fixed, + closed: () => this.closedFlag, + open: async () => { + if (this.closedFlag) throw unavailable(); + const { handle, stat } = await openLeaf(this.root, parts).catch(() => { throw unavailable(); }); + // Identity bind: a substituted file must also reproduce (dev, ino, size). + if (stat.dev !== fixed.dev || stat.ino !== fixed.ino || stat.size !== BigInt(fixed.size)) { await handle.close().catch(() => undefined); throw unavailable(); } + const snapshot = { dev: stat.dev, ino: stat.ino, size: stat.size, mtime: stat.mtimeNs, ctime: stat.ctimeNs }; + let open = true; + const lease: AttachmentOwnedLease = { + size: fixed.size, + // Positional reads on our own handle never move an application's file position. + read: async (offset, length) => { + if (!open || this.closedFlag) throw unavailable(); + const buffer = new Uint8Array(length); let done = 0; + while (done < length) { + const { bytesRead } = await handle.read(buffer, done, length - done, offset + done); + if (!bytesRead) break; done += bytesRead; + } + return done === length ? buffer : buffer.subarray(0, done); + }, + check: async () => { + if (!open || this.closedFlag) throw unavailable(); + const now = await handle.stat({ bigint: true }); + if (now.dev !== snapshot.dev || now.ino !== snapshot.ino || now.size !== snapshot.size || now.mtimeNs !== snapshot.mtime || now.ctimeNs !== snapshot.ctime) throw unavailable(); + }, + close: async () => { if (!open) return; open = false; this.leases.delete(lease); await handle.close().catch(() => undefined); }, + }; + this.leases.add(lease); + return lease; + }, + }); + return { kind: "OwnedRecordingFile", lease: ref }; + } + async close(): Promise { + this.closedFlag = true; + await Promise.all([...this.leases].map(lease => lease.close())); + } +} +/** An explicitly bound session's recording root. A report path alone is NOT authority (F01). */ +export async function createRecordingFileScope(root: string, options: { enabled: true }): Promise { + if (!options || options.enabled !== true || process.platform === "win32" || typeof root !== "string" || !path.isAbsolute(root)) throw unavailable(); + const normalized = path.resolve(root); + if (normalized !== root.replace(/\/+$/, "") && normalized !== root) throw unavailable(); + let real: string; + try { real = await fs.realpath(normalized); } catch { throw unavailable(); } + if (real !== normalized) throw unavailable(); + // lstat every root component: none may be a symlink. + const parts = normalized.split(path.sep).filter(Boolean); + let current: string = path.sep; + for (const part of parts) { current = path.join(current, part); await noSymlinkDirectory(current).catch(() => { throw unavailable(); }); } + return new Scope(normalized); +} diff --git a/typescript/packages/fi-core/src/attachments/node.ts b/typescript/packages/fi-core/src/attachments/node.ts new file mode 100644 index 000000000..a911f7aa2 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/node.ts @@ -0,0 +1,99 @@ +import { request } from "node:https"; +import { DisabledAttachmentManager } from "./disabled"; +import { AttachmentManager } from "./manager"; +import { attachmentEnabled } from "./policy"; +import { ATTACHMENT_CHUNK_BYTES, AttachmentTransportError, createAttachmentHttpTransport } from "./transport-v1"; +import type { AttachmentHttpClient } from "./transport-v1"; +import type { AttachmentContentSource, AttachmentManagerOptions } from "./types"; + +type Ranged = Exclude; +function sizeOf(body: string | AttachmentContentSource | undefined): number { + if (body === undefined) return 0; + if (typeof body === "string") return Buffer.byteLength(body); + if (body instanceof Uint8Array) return body.byteLength; + return (body as { size: number }).size; +} +/** Node-only bounded transport. No keep-alive pool, redirect following, cookies or environment proxy. */ +export const attachmentNodeHttpClient: AttachmentHttpClient = input => new Promise((resolve, reject) => { + let settled = false; + const fail = (code: "incomplete" | "unsupported_version" | "source_unavailable" = "incomplete", retryable = !input.signal.aborted) => { + if (!settled) { settled = true; reject(new AttachmentTransportError(code, retryable)); } + }; + try { + const size = sizeOf(input.body); + const req = request(input.url, { method: input.method, headers: { ...input.headers, ...(input.body === undefined ? {} : { "content-length": String(size) }) }, + signal: input.signal, agent: false }, response => { + const status = response.statusCode ?? 0; + const ok = status >= 200 && status < 300; + // Success bodies up to 64 KiB; error bodies up to 4 KiB (parsed only as F01 Error). + const limit = ok ? 65536 : 4096; + const chunks: Buffer[] = []; let bytes = 0; let over = false; + response.on("data", (chunk: Buffer) => { + if (over) return; + bytes += chunk.byteLength; + if (bytes > limit) { + over = true; + if (ok) { fail("unsupported_version", false); response.destroy(); req.destroy(); return; } + settled = true; resolve({ status, body: new Uint8Array(), retryAfter: response.headers["retry-after"] as string | undefined }); + response.destroy(); req.destroy(); return; + } + chunks.push(chunk); + }); + response.on("error", () => fail()); + response.on("aborted", () => fail()); + response.on("end", () => { + if (!settled) { settled = true; resolve({ status, body: Buffer.concat(chunks, bytes), retryAfter: response.headers["retry-after"] as string | undefined }); } + }); + }); + req.on("socket", socket => socket.unref()); + req.on("error", () => fail()); + // Per-slice stall timeout on content PUTs. + if (input.stallTimeoutMs) req.setTimeout(input.stallTimeoutMs, () => { req.destroy(); fail("incomplete", true); }); + void (async () => { + const body = input.body; + if (body === undefined) { req.end(); return; } + if (typeof body === "string") { req.end(body); return; } + const ranged = !(body instanceof Uint8Array) && !(typeof Blob !== "undefined" && body instanceof Blob) ? body as Ranged : undefined; + for (let offset = 0; offset < size; offset += ATTACHMENT_CHUNK_BYTES) { + if (input.signal.aborted || req.destroyed) return; + const length = Math.min(ATTACHMENT_CHUNK_BYTES, size - offset); + let chunk: Uint8Array; + if (body instanceof Uint8Array) chunk = body.subarray(offset, offset + length); + else if (ranged) { + // check() before every slice: a changed owned file aborts the transfer. + await ranged.check(); + chunk = await ranged.read(offset, length); + if (chunk.byteLength !== length) throw new AttachmentTransportError("source_unavailable"); + } else chunk = new Uint8Array(await (body as Blob).slice(offset, offset + length).arrayBuffer()); + if (input.pacer) await input.pacer.take(chunk.byteLength, input.signal); + if (input.signal.aborted || req.destroyed) return; + if (!req.write(chunk)) await new Promise(done => { + const finish = () => { req.removeListener("drain", finish); req.removeListener("close", finish); done(); }; + req.once("drain", finish); req.once("close", finish); + }); + } + if (ranged) await ranged.check(); + req.end(); + })().catch(error => { req.destroy(); fail(error instanceof AttachmentTransportError ? error.code as "source_unavailable" : "incomplete", false); }); + } catch { fail(); } +}); +export interface NodeAttachmentManagerOptions extends AttachmentManagerOptions { headers?: Readonly> } +/** + * Provider setup passes only its resolved FI credentials, never a provider/model client's key. Returns the inert + * DisabledAttachmentManager when attachments are OFF; `enabled` is resolved once here, from config then env. + */ +export function createNodeAttachmentManager(options: NodeAttachmentManagerOptions = {}): AttachmentManager { + const environment = options.environment ?? (() => process.env); + let env: Readonly>; + try { env = environment(); } catch { env = { FI_ATTACHMENTS_ENABLED: "false" }; } + if (!attachmentEnabled(options, env)) return new DisabledAttachmentManager(); + const endpoint = options.endpoint ?? env.FI_ATTACHMENTS_ENDPOINT; + let transport = options.transport; + if (!transport && endpoint) { + try { transport = createAttachmentHttpTransport({ endpoint, headers: options.headers ?? {}, client: attachmentNodeHttpClient, stallTimeoutMs: options.requestTimeoutMs }); } + catch { /* Unconfigured/untrusted origin remains fail closed without affecting tracer setup. */ } + } + return new AttachmentManager({ ...options, endpoint, environment, transport }); +} +export { createRecordingFileScope } from "./node-sources"; +export * from "./index"; diff --git a/typescript/packages/fi-core/src/attachments/owned.ts b/typescript/packages/fi-core/src/attachments/owned.ts new file mode 100644 index 000000000..a3e4fef26 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/owned.ts @@ -0,0 +1,28 @@ +/** Browser-safe registry of owned-file references minted by this core copy's Node scope (design 4.3). */ +import type { AttachmentFileIdentity, AttachmentOwnedFileRef } from "./types"; + +/** @internal An open lease over one owned file. Reads never move any application file position. */ +export interface AttachmentOwnedLease { + readonly size: number; + read(offset: number, size: number): Promise; + /** Throws source_unavailable when (dev, ino, size, mtime, ctime) changed since open. */ + check(): Promise; + close(): Promise; +} +/** @internal */ +export interface AttachmentOwnedFileRecord { + readonly identity: AttachmentFileIdentity; + open(): Promise; + closed(): boolean; +} +const records = new WeakMap(); +/** @internal Only the Node scope calls this. */ +export function registerAttachmentOwnedFile(record: AttachmentOwnedFileRecord): AttachmentOwnedFileRef { + const ref = Object.freeze(Object.create(null)) as AttachmentOwnedFileRef; + records.set(ref as unknown as object, record); + return ref; +} +/** @internal A foreign or forged object resolves to undefined (source_unavailable). */ +export function attachmentOwnedFile(ref: unknown): AttachmentOwnedFileRecord | undefined { + return ref && typeof ref === "object" ? records.get(ref) : undefined; +} diff --git a/typescript/packages/fi-core/src/attachments/policy.ts b/typescript/packages/fi-core/src/attachments/policy.ts new file mode 100644 index 000000000..2bd2a0fc0 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/policy.ts @@ -0,0 +1,59 @@ +import type { TraceConfigOptions } from "../trace/trace-config/types"; +import type { AttachmentAdmission, AttachmentConfig, AttachmentModality, AttachmentOptions, AttachmentServerPolicy, AttachmentSourceKind } from "./types"; + +const MIME_MODALITIES: Readonly> = Object.freeze({ + "audio/wav": "audio", "audio/ogg": "audio", "audio/mpeg": "audio", "audio/mp4": "audio", "audio/webm": "audio", "audio/flac": "audio", "audio/L16": "audio", + "image/png": "image", "image/jpeg": "image", "image/webp": "image", "video/mp4": "video", "video/webm": "video", + "application/pdf": "document", "application/octet-stream": "file", +}); +export function attachmentModality(mime: string): AttachmentModality | undefined { + return Object.prototype.hasOwnProperty.call(MIME_MODALITIES, mime) ? MIME_MODALITIES[mime] : undefined; +} +const enabledValue = (value: string | undefined) => /^(true|1|yes)$/i.test(value?.trim() ?? ""); +const restrictiveValue = (value: string | undefined) => value !== undefined && !/^(false|0|no)$/i.test(value.trim()); +export function attachmentEnabled(config: AttachmentConfig, env: Readonly>): boolean { + return config.enabled !== false && (config.enabled === true || enabledValue(env.FI_ATTACHMENTS_ENABLED)) && + (env.FI_ATTACHMENTS_ENABLED === undefined || enabledValue(env.FI_ATTACHMENTS_ENABLED)); +} +/** Pure admission. This function never accesses, copies, decodes or hashes a source. */ +export function evaluateAttachmentPolicy(config: AttachmentConfig, traceConfig: TraceConfigOptions, + server: AttachmentServerPolicy | undefined, sourceKind: AttachmentSourceKind, options: AttachmentOptions, + env: Readonly> = {}): AttachmentAdmission | undefined { + if (!attachmentEnabled(config, env)) return "disabled"; + if (!server) return "bootstrap_pending"; + if (!server.enabled) return "policy_denied"; + const capture = options.capture ?? {}; + for (const key of ["captureContent", "recordInputs", "recordOutputs", "muted", "temporaryChat", "requested", "modelInput", "containsInput", "containsOutput", "errorDetail", "newRecording"] as const) { + if (capture[key] !== undefined && typeof capture[key] !== "boolean") return "policy_denied"; + } + if (capture.captureContent === false || capture.requested === false || capture.muted || capture.temporaryChat) return "policy_denied"; + if (!["input", "output", "mixed"].includes(options.direction)) return "policy_denied"; + const input = options.direction !== "output" || capture.containsInput || capture.modelInput || options.purpose === "model_input" || options.purpose === "computer_screenshot"; + const output = options.direction !== "input" || capture.containsOutput; + if ((input && capture.recordInputs === false) || (output && capture.recordOutputs === false)) return "policy_denied"; + const hidden = (key: keyof TraceConfigOptions, name: string) => (traceConfig[key] !== undefined && traceConfig[key] !== false) || restrictiveValue(env[name]); + const inputHidden = hidden("hideInputs", "FI_HIDE_INPUTS") || hidden("hideInputMessages", "FI_HIDE_INPUT_MESSAGES") || hidden("hideInputText", "FI_HIDE_INPUT_TEXT"); + const outputHidden = hidden("hideOutputs", "FI_HIDE_OUTPUTS") || hidden("hideOutputMessages", "FI_HIDE_OUTPUT_MESSAGES") || hidden("hideOutputText", "FI_HIDE_OUTPUT_TEXT"); + if ((input && inputHidden) || (output && outputHidden)) return "policy_denied"; + const modality = attachmentModality(options.mimeType); + if (!modality) return "source_unavailable"; + if (input && modality === "image" && hidden("hideInputImages", "FI_HIDE_INPUT_IMAGES")) return "policy_denied"; + const errorHidden = hidden("hideErrorDetails", "FI_HIDE_ERROR_DETAILS") || + (traceConfig.hideErrorDetails === undefined && env.FI_HIDE_ERROR_DETAILS === undefined && (inputHidden || outputHidden)); + if ((capture.errorDetail || options.purpose === "diagnostic") && errorHidden) return "policy_denied"; + // No local qualified media sanitizer ships with the core. Never trust a caller's "sanitized" flag. + if (server.require_media_redaction || server.require_metadata_removal) return "redaction_unavailable"; + const unredacted = config.allowUnredactedMediaWithTextPii === true || + (config.allowUnredactedMediaWithTextPii === undefined && enabledValue(env.FI_ATTACHMENTS_ALLOW_UNREDACTED_MEDIA_WITH_TEXT_PII)); + if (hidden("piiRedaction", "FI_PII_REDACTION") && !(unredacted && server.allow_unredacted_media_with_text_pii)) return "redaction_unavailable"; + if (!server.allowed_modalities.includes(modality) || !server.allowed_sources.includes(sourceKind)) return "policy_denied"; + if (capture.newRecording) { + const key = modality === "audio" ? "recordAudio" : modality === "video" ? "recordVideo" : undefined; + if (!key) return "recording_disabled"; + const name = key === "recordAudio" ? "FI_ATTACHMENTS_RECORD_AUDIO" : "FI_ATTACHMENTS_RECORD_VIDEO"; + const permitted = key === "recordAudio" ? server.record_audio : server.record_video; + if (!permitted || config[key] === false || !(config[key] === true || enabledValue(env[name])) || + (env[name] !== undefined && !enabledValue(env[name]))) return "recording_disabled"; + } + return undefined; +} diff --git a/typescript/packages/fi-core/src/attachments/recording.ts b/typescript/packages/fi-core/src/attachments/recording.ts new file mode 100644 index 000000000..432def5ca --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/recording.ts @@ -0,0 +1,845 @@ +import { v4 as uuid } from "uuid"; +import { attachmentTraceUuid, validAttachmentUuid } from "./correlation"; +import type { AttachmentHost, CoreJob, UploadOutcome } from "./internal"; +import { attachmentIdempotencyKey } from "./transport-v1"; +import type { + AttachmentAdmission, AttachmentCapturePolicy, AttachmentCompletion, AttachmentContentSource, AttachmentDirection, AttachmentGapReason, + AttachmentRecordingManifest, AttachmentRecordingOptions, AttachmentRecordingResult, AttachmentRecordingStatus, AttachmentRecordingWriter, + AttachmentTrackSpec, AttachmentUploadRequest, +} from "./types"; + +export const ATTACHMENT_WAV_HEADER_BYTES = 44; +const MAX_SEGMENT_BYTES = 8 * 1024 * 1024; +const MAX_SEGMENT_MS = 30000; +const MAX_RECORDING_MS = 7200000; +const MAX_RECORDING_BYTES = 1024 * 1024 * 1024; +const MAX_MANIFEST_BYTES = 1048576; +const MAX_PLAYOUT = 4096; +const GAP_REASONS: readonly AttachmentGapReason[] = ["queue_full", "bootstrap_pending", "source_unavailable", "policy_denied", "truncated", "incomplete"]; + +function gcd(a: number, b: number): number { while (b) { [a, b] = [b, a % b]; } return a; } +/** Smallest frame count that is a whole number of milliseconds (design 2.5). */ +export function attachmentBoundaryStep(sampleRateHz: number): number { return sampleRateHz / gcd(sampleRateHz, 1000); } +/** Largest multiple of the boundary step within both the duration and the byte bound (header included). */ +export function attachmentSegmentFrames(sampleRateHz: number, frameBytes: number, segmentMs = MAX_SEGMENT_MS, segmentBytes = MAX_SEGMENT_BYTES): number { + const g = attachmentBoundaryStep(sampleRateHz); + const byMs = Math.floor(Math.min(segmentMs, MAX_SEGMENT_MS) * sampleRateHz / 1000); + const byBytes = Math.floor((Math.min(segmentBytes, MAX_SEGMENT_BYTES) - ATTACHMENT_WAV_HEADER_BYTES) / frameBytes); + return Math.floor(Math.min(byMs, byBytes) / g) * g; +} +export function attachmentFramesToMs(frames: number, sampleRateHz: number, roundUp = false): number { + return roundUp ? Math.ceil(frames * 1000 / sampleRateHz) : Math.floor(frames * 1000 / sampleRateHz); +} +/** Canonical 44-byte RIFF/WAVE PCM header (format tag 1, 16-bit). */ +export function attachmentWavHeader(channels: number, sampleRateHz: number, dataBytes: number): Uint8Array { + const header = new Uint8Array(ATTACHMENT_WAV_HEADER_BYTES); const view = new DataView(header.buffer); + const ascii = (offset: number, text: string) => { for (let i = 0; i < text.length; i++) header[offset + i] = text.charCodeAt(i); }; + ascii(0, "RIFF"); view.setUint32(4, 36 + dataBytes, true); ascii(8, "WAVE"); ascii(12, "fmt "); + view.setUint32(16, 16, true); view.setUint16(20, 1, true); view.setUint16(22, channels, true); view.setUint32(24, sampleRateHz, true); + view.setUint32(28, sampleRateHz * channels * 2, true); view.setUint16(32, channels * 2, true); view.setUint16(34, 16, true); + ascii(36, "data"); view.setUint32(40, dataBytes, true); + return header; +} +/** Header plus retained chunks, served as one bounded content source without re-materializing. */ +export function attachmentChunkedSource(header: Uint8Array, chunks: readonly Uint8Array[]): AttachmentContentSource { + let size = header.byteLength; for (const chunk of chunks) size += chunk.byteLength; + if (size <= MAX_SEGMENT_BYTES) { + const out = new Uint8Array(size); out.set(header, 0); let offset = header.byteLength; + for (const chunk of chunks) { out.set(chunk, offset); offset += chunk.byteLength; } + return out; + } + const parts = [header, ...chunks]; + return { + size, + async read(offset: number, length: number) { + const out = new Uint8Array(Math.min(length, size - offset)); let base = 0; let written = 0; + for (const part of parts) { + const end = base + part.byteLength; + if (end > offset && base < offset + out.byteLength) { + const from = Math.max(0, offset - base); const to = Math.min(part.byteLength, offset + out.byteLength - base); + out.set(part.subarray(from, to), written); written += to - from; + } + base = end; if (written === out.byteLength) break; + } + return out; + }, + async check() { /* in-memory */ }, + }; +} + +/** Cross-field manifest rules mirrored from F01 validate.py, applied locally before every PUT. */ +export function validAttachmentManifest(value: AttachmentRecordingManifest): boolean { + try { + if (!value || value.schema_version !== 1 || !validAttachmentUuid(value.recording_id) || value.timebase !== "session_relative_ms") return false; + if (new TextEncoder().encode(JSON.stringify(value)).byteLength > MAX_MANIFEST_BYTES) return false; + if (!Number.isInteger(value.sample_rate_hz) || value.sample_rate_hz < 8000 || value.sample_rate_hz > 192000) return false; + if (!Number.isInteger(value.channels) || value.channels < 1 || value.channels > 32) return false; + if (!Number.isInteger(value.duration_ms) || value.duration_ms < 0 || value.duration_ms > MAX_RECORDING_MS) return false; + if (value.capture_complete && value.gaps.length) return false; + if (!Array.isArray(value.tracks) || value.tracks.length < 1 || value.tracks.length > 32) return false; + const ids = value.tracks.map(track => track.track_id); + if (new Set(ids).size !== ids.length || !ids.every(validAttachmentUuid)) return false; + const channels = value.tracks.flatMap(track => track.channels).sort((a, b) => a - b); + if (channels.length !== value.channels || channels.some((channel, index) => channel !== index)) return false; + if (value.tracks.some(track => new Set(track.channels).size !== track.channels.length || !["user", "assistant", "mixed", "unknown"].includes(track.role) || + !["input", "output", "mixed"].includes(track.direction))) return false; + if (value.layout === "mixed" && value.tracks.some(track => track.direction !== "mixed")) return false; + if (value.layout === "stereo" && value.channels !== 2) return false; + const separate = value.layout === "separate_tracks"; + const seen = new Set(); let previous = -1; + for (let index = 0; index < value.segments.length; index++) { + const s = value.segments[index]; + if (s.index !== index || s.start_ms < previous || seen.has(s.occurrence_id) || !validAttachmentUuid(s.occurrence_id)) return false; + if (!Number.isInteger(s.duration_ms) || s.duration_ms < 1 || s.duration_ms > MAX_SEGMENT_MS) return false; + seen.add(s.occurrence_id); previous = s.start_ms; + } + const timelines = new Map(); + if (separate) for (const id of ids) timelines.set(id, []); else timelines.set(undefined, []); + for (const interval of [...value.segments, ...value.gaps]) { + if (separate ? interval.track_id === undefined : interval.track_id !== undefined) return false; + const list = timelines.get(interval.track_id); if (!list) return false; + if (!Number.isInteger(interval.start_ms) || !Number.isInteger(interval.duration_ms) || interval.duration_ms < 1) return false; + list.push(interval); + } + if (value.gaps.some(gap => !GAP_REASONS.includes(gap.reason))) return false; + for (const list of timelines.values()) { + let end = 0; + for (const interval of [...list].sort((a, b) => a.start_ms - b.start_ms)) { if (interval.start_ms !== end) return false; end += interval.duration_ms; } + if (end !== value.duration_ms) return false; + } + for (const interval of [...value.segments, ...value.gaps, ...value.playout]) { + if (interval.start_ms + interval.duration_ms > value.duration_ms || interval.duration_ms < 1) return false; + } + return true; + } catch { return false; } +} + +interface Piece { + kind: "segment" | "gap"; + track: number; + start: number; + end: number; + reason?: AttachmentGapReason; + // Segments only. + source?: AttachmentContentSource; + bytes?: number; + state?: "sealed" | "queued" | "uploading" | "done" | "failed"; + occurrenceId?: string; + attachmentId?: string; + index?: number; + outcome?: UploadOutcome; + failure?: AttachmentCompletion["status"]; + job?: CoreJob; +} +interface Track { + ordinal: number; + spec: AttachmentTrackSpec; + manifestId?: string; + frameBytes: number; + channels: number; + denied?: AttachmentAdmission; + pos: number; + fill: number; + blockGap?: AttachmentGapReason; + blockData: number; + runStart: number; + runFrames: number; + chunks: Uint8Array[]; + chunkBytes: number; + carry?: Uint8Array; + pieces: Piece[]; +} +export interface RecordingIdentity { traceId: string; spanId: string } +export interface RecordingInit { + host: AttachmentHost; + options: AttachmentRecordingOptions; + identity?: RecordingIdentity; + /** queued (active), bootstrap_pending (pending mode) or a denial (inert). */ + admission: AttachmentAdmission; + /** Per-track denial for separate_tracks. */ + trackDenials?: readonly (AttachmentAdmission | undefined)[]; + /** Re-run full admission (pending writers and policy epoch changes). */ + readmit(): { reason: AttachmentAdmission; trackDenials?: readonly (AttachmentAdmission | undefined)[] }; + /** Applies the carrier stamp once admitted. */ + stamp(): void; + /** Releases the recording slot and descriptor exactly once. */ + close(): void; + sourceKind?: "Bytes" | "OwnedRecordingFile"; +} +export interface RecordingFileSegment { track: number; startFrame: number; endFrame: number; source: AttachmentContentSource } + +/** Validates options, returning the layout's stream shape or undefined (fail closed). */ +export function attachmentRecordingShape(options: AttachmentRecordingOptions): { streams: number; frameBytes: number[]; channels: number } | undefined { + try { + const rate = options?.format?.sampleRateHz; + if (!Number.isInteger(rate) || rate < 8000 || rate > 192000 || (options.format.sampleFormat ?? "s16le") !== "s16le") return undefined; + if (attachmentBoundaryStep(rate) > 4800) return undefined; + if (!["mixed", "stereo", "separate_tracks"].includes(options.layout) || !["observe_consumed", "create_recording"].includes(options.operation)) return undefined; + if (options.purpose !== undefined && options.purpose !== "conversation_recording" && options.purpose !== "turn_recording") return undefined; + const tracks = options.tracks; + if (!Array.isArray(tracks) || tracks.length < 1 || tracks.length > 32) return undefined; + for (const track of tracks) { + if (!track || !Array.isArray(track.channels) || track.channels.length < 1 || new Set(track.channels).size !== track.channels.length || + track.channels.some((c: number) => !Number.isInteger(c) || c < 0 || c > 31) || !["input", "output", "mixed"].includes(track.direction) || + !["user", "assistant", "mixed", "unknown"].includes(track.role)) return undefined; + } + const all = tracks.flatMap(track => [...track.channels]).sort((a, b) => a - b); + if (all.some((channel, index) => channel !== index)) return undefined; + const window = options.windowSegments ?? 2; + if (!Number.isInteger(window) || window < 1 || window > 4) return undefined; + for (const id of [options.conversationId, options.turnId]) if (id !== undefined && !(typeof id === "string" && /^[a-zA-Z0-9_-]{1,128}$/.test(id))) return undefined; + if (options.layout === "mixed") return tracks.length === 1 && tracks[0].direction === "mixed" ? { streams: 1, frameBytes: [all.length * 2], channels: all.length } : undefined; + if (options.layout === "stereo") return tracks.length === 2 && tracks.every(track => track.channels.length === 1) ? { streams: 1, frameBytes: [4], channels: 2 } : undefined; + return { streams: tracks.length, frameBytes: tracks.map(track => track.channels.length * 2), channels: all.length }; + } catch { return undefined; } +} + +/** Live PCM recording writer and the shared segment/manifest pipeline (design section 2). */ +export class AttachmentRecording implements AttachmentRecordingWriter { + readonly recordingId = uuid(); + private mode: "pending" | "active" | "finishing" | "terminal"; + private admissionReason: AttachmentAdmission; + private terminalReason?: AttachmentAdmission | AttachmentCompletion["status"]; + private readonly host: AttachmentHost; + private readonly options: AttachmentRecordingOptions; + private readonly identity?: RecordingIdentity; + private readonly init: RecordingInit; + private readonly rate: number; + private readonly g: number; + private F = 0; + private readonly tracks: Track[] = []; + private readonly manifestTracks: { track_id: string; channels: number[]; direction: AttachmentDirection; role: AttachmentTrackSpec["role"] }[] = []; + private readonly windowSegments: number; + private epoch: number; + private totalBytes = 0; + private truncated = false; + private maxFrames = 0; + private recordingBytesCap = MAX_RECORDING_BYTES; + private completed = 0; + private sinceManifest = 0; + private lastPutEnd = 0; + private revision = 0; + private publishedSegments = 0; + private publishedGapMs = 0; + private publishedComplete = false; + private readonly cuts: number[] = []; + private manifestQueued = false; + private manifestFallback = false; + private manifestDisabled = false; + private indexReused = false; + private readonly playout: { startMs: number; durationMs: number; evidence: "generated" | "played" | "unknown" | "truncated" }[] = []; + private outstanding = 0; + private idleWaiters: (() => void)[] = []; + private resultPromise?: Promise; + private closed = false; + private finalEnd?: number; + private stamped = false; + private finalPublished = false; + private sealWaiters: (() => void)[] = []; + + constructor(init: RecordingInit) { + this.init = init; this.host = init.host; this.options = init.options; this.identity = init.identity; + this.rate = init.options.format?.sampleRateHz ?? 8000; + this.g = Number.isInteger(this.rate) && this.rate > 0 ? attachmentBoundaryStep(this.rate) : 1; + this.windowSegments = init.options.windowSegments ?? 2; + this.epoch = this.host.policyEpoch(); + this.admissionReason = init.admission; + const shape = attachmentRecordingShape(init.options); + if (!shape) { this.admissionReason = this.admissionReason === "queued" || this.admissionReason === "bootstrap_pending" ? "policy_denied" : this.admissionReason; } + if (shape) { + const specs = init.options.tracks; + for (const spec of specs) this.manifestTracks.push({ track_id: uuid(), channels: [...spec.channels], direction: spec.direction, role: spec.role }); + for (let i = 0; i < shape.streams; i++) { + this.tracks.push({ ordinal: i, spec: init.options.layout === "separate_tracks" ? specs[i] : { channels: specs.flatMap(s => [...s.channels]), direction: "mixed", role: "mixed" }, + manifestId: init.options.layout === "separate_tracks" ? this.manifestTracks[i].track_id : undefined, + frameBytes: shape.frameBytes[i], channels: shape.frameBytes[i] / 2, denied: init.trackDenials?.[i], pos: 0, fill: 0, blockData: 0, + runStart: 0, runFrames: 0, chunks: [], chunkBytes: 0, pieces: [] }); + } + } + if (this.admissionReason === "queued") { this.mode = "active"; this.activate(); } + else if (this.admissionReason === "bootstrap_pending" && shape) this.mode = "pending"; + else { this.mode = "terminal"; this.terminalReason = this.admissionReason; this.closeOnce(); } + } + get reason(): AttachmentAdmission { return this.admissionReason; } + + private activate(): boolean { + const session = this.host.currentSession(); + if (!session) return false; + const limits = session.limits; + const segMs = Math.min(limits.segment_duration_ms ?? MAX_SEGMENT_MS, MAX_SEGMENT_MS); + const segBytes = Math.min(limits.segment_bytes ?? MAX_SEGMENT_BYTES, limits.max_upload_bytes ?? MAX_SEGMENT_BYTES, MAX_SEGMENT_BYTES); + this.F = attachmentSegmentFrames(this.rate, Math.max(...this.tracks.map(t => t.frameBytes)), segMs, segBytes); + if (this.F < this.g) { this.terminal("too_large"); return false; } + const recMs = Math.min(limits.recording_duration_ms ?? MAX_RECORDING_MS, MAX_RECORDING_MS); + this.maxFrames = Math.floor(recMs * this.rate / 1000); + this.recordingBytesCap = Math.min(limits.recording_bytes ?? MAX_RECORDING_BYTES, MAX_RECORDING_BYTES); + if (!this.stamped) { this.stamped = true; try { this.init.stamp(); } catch { /* never throw into audio callbacks */ } } + return true; + } + private terminal(reason: AttachmentAdmission | AttachmentCompletion["status"]): void { + if (this.mode === "terminal") return; + this.mode = "terminal"; this.terminalReason = reason; this.wakeSeal(); + } + private wakeSeal(): void { const waiters = this.sealWaiters; this.sealWaiters = []; for (const wake of waiters) wake(); } + private closeOnce(): void { if (!this.closed) { this.closed = true; try { this.init.close(); } catch { /* bounded */ } this.host.recordingClosed(this.recordingId); } } + + // --- block timeline ----------------------------------------------------------------------------- + private releaseTrackBlock(track: Track): void { + if (!track.blockData) return; + let drop = track.blockData * track.frameBytes; track.blockData = 0; + this.host.releaseBytes(drop); track.chunkBytes -= drop; + while (drop > 0 && track.chunks.length) { + const last = track.chunks[track.chunks.length - 1]; + if (last.byteLength <= drop) { drop -= last.byteLength; track.chunks.pop(); } + else { track.chunks[track.chunks.length - 1] = last.subarray(0, last.byteLength - drop); drop = 0; } + } + } + private commitBlock(track: Track, frames = this.g): void { + if (track.blockGap === undefined) { track.runFrames += frames; track.blockData = 0; } + else { this.closeRun(track); this.appendGap(track, track.blockGap, frames); } + track.pos += frames; track.fill = 0; track.blockGap = undefined; + if (track.runFrames === 0) track.runStart = track.pos; + if (this.F && track.pos % this.F === 0) { this.closeRun(track); this.boundary(track); } + } + private appendGap(track: Track, reason: AttachmentGapReason, frames: number): void { + const last = track.pieces[track.pieces.length - 1]; + const start = track.pos; + if (last && last.kind === "gap" && last.reason === reason && last.end === start) last.end = start + frames; + else track.pieces.push({ kind: "gap", track: track.ordinal, start, end: start + frames, reason }); + track.runStart = start + frames; + } + private closeRun(track: Track): void { + if (track.runFrames > 0) { + const dataBytes = track.runFrames * track.frameBytes; + const chunks = track.chunks; let kept = 0; const used: Uint8Array[] = []; + for (const chunk of chunks) { if (kept >= dataBytes) break; const take = Math.min(chunk.byteLength, dataBytes - kept); used.push(take === chunk.byteLength ? chunk : chunk.subarray(0, take)); kept += take; } + // Bytes of the still-open block stay with the track. + const rest: Uint8Array[] = []; let skip = dataBytes; + for (const chunk of chunks) { if (skip >= chunk.byteLength) { skip -= chunk.byteLength; continue; } rest.push(skip ? chunk.subarray(skip) : chunk); skip = 0; } + track.chunks = rest; track.chunkBytes -= dataBytes; + const header = attachmentWavHeader(track.channels, this.rate, dataBytes); + this.totalBytes += dataBytes + ATTACHMENT_WAV_HEADER_BYTES; + track.pieces.push({ kind: "segment", track: track.ordinal, start: track.runStart, end: track.runStart + track.runFrames, + source: attachmentChunkedSource(header, used), bytes: dataBytes, state: "sealed", occurrenceId: uuid(), attachmentId: uuid() }); + this.wakeSeal(); + } + track.runFrames = 0; track.runStart = track.pos; + this.release(); + } + /** Common boundaries: a track more than one window behind the leader is filled with a gap (design 2.5). */ + private boundary(leader: Track): void { + if (this.tracks.length < 2) return; + const target = leader.pos - this.F; + if (target <= 0) return; + for (const track of this.tracks) { + if (track === leader || track.pos + track.fill >= target) continue; + this.gapFrames(track, target - track.pos - track.fill, track.denied ? "policy_denied" : "source_unavailable"); + } + } + private gapFrames(track: Track, frames: number, reason: AttachmentGapReason): void { + while (frames > 0) { + if (track.blockData) this.releaseTrackBlock(track); + if (track.blockGap === undefined) track.blockGap = reason; + const take = Math.min(frames, this.g - track.fill); + track.fill += take; frames -= take; + if (track.fill === this.g) this.commitBlock(track); + } + } + private dataFrames(track: Track, bytes: Uint8Array): void { + const frames = bytes.byteLength / track.frameBytes; let offset = 0; let remaining = frames; + while (remaining > 0) { + const take = Math.min(remaining, this.g - track.fill); + if (track.blockGap === undefined) { + const part = bytes.subarray(offset * track.frameBytes, (offset + take) * track.frameBytes); + track.chunks.push(part); track.chunkBytes += part.byteLength; track.blockData += take; + } else this.host.releaseBytes(take * track.frameBytes); // tainted block: data becomes part of the gap + track.fill += take; remaining -= take; offset += take; + if (track.fill === this.g) this.commitBlock(track); + } + } + private totalFrames(track: Track): number { return track.pos + track.fill; } + private waitingWindows(): number { + const starts = new Set(); + for (const track of this.tracks) for (const piece of track.pieces) if (piece.kind === "segment" && (piece.state === "sealed" || piece.state === "queued" || piece.state === "uploading")) starts.add(piece.start); + return starts.size; + } + + // --- public writer API ---------------------------------------------------------------------------- + tryWrite(pcm: Uint8Array, track = 0): boolean { + try { + if (this.mode === "terminal" || this.mode === "finishing" || !(pcm instanceof Uint8Array) || !Number.isInteger(track)) return false; + const t = this.tracks[track]; + if (!t) return false; + // Partial frames carry to the next call (at most frameBytes - 1 bytes). + let data = pcm; let ok = true; + if (t.carry) { + const need = t.frameBytes - t.carry.byteLength; + if (pcm.byteLength < need) { const joined = new Uint8Array(t.carry.byteLength + pcm.byteLength); joined.set(t.carry); joined.set(pcm, t.carry.byteLength); t.carry = joined; return true; } + const frame = new Uint8Array(t.frameBytes); frame.set(t.carry); frame.set(pcm.subarray(0, need), t.carry.byteLength); t.carry = undefined; + ok = this.writeFrames(t, frame); + data = pcm.subarray(need); + } + const whole = data.byteLength - (data.byteLength % t.frameBytes); + if (data.byteLength !== whole) t.carry = data.slice(whole); + if (!whole) return ok; + return this.writeFrames(t, data.subarray(0, whole)) && ok; + } catch { return false; } + } + private writeFrames(t: Track, data: Uint8Array): boolean { + if (this.mode === "terminal" || this.mode === "finishing") return false; + this.host.touch(); // deferred renewal check; tryWrite itself does no I/O (design 3.1) + const whole = data.byteLength; const frames = whole / t.frameBytes; + if (this.mode === "pending") { + const state = this.host.runtimeState(); + if (state === "ready") { + const again = this.init.readmit(); + if (again.reason !== "queued") { this.admissionReason = again.reason; this.discardAll(); this.terminal(again.reason); this.closeOnce(); return false; } + for (let i = 0; i < this.tracks.length; i++) this.tracks[i].denied = again.trackDenials?.[i]; + this.admissionReason = "queued"; this.mode = "active"; this.epoch = this.host.policyEpoch(); + if (!this.activate()) return false; + } else if (state === "bootstrap_pending") { this.gapFrames(t, frames, "bootstrap_pending"); return false; } + else { this.admissionReason = state as AttachmentAdmission; this.discardAll(); this.terminal(this.admissionReason); this.closeOnce(); return false; } + } + if (this.host.policyEpoch() !== this.epoch) { + this.epoch = this.host.policyEpoch(); + const again = this.init.readmit(); + if (again.reason !== "queued") { this.gapFrames(t, frames, "policy_denied"); this.policyStop(); return false; } + for (let i = 0; i < this.tracks.length; i++) if (again.trackDenials?.[i] && !this.tracks[i].denied) { this.tracks[i].denied = again.trackDenials[i]; this.dropTrack(this.tracks[i]); } + } + if (t.denied) { this.gapFrames(t, frames, "policy_denied"); return false; } + if (this.host.runtimeState() !== "ready" || this.host.stopping()) { this.gapFrames(t, frames, "incomplete"); return false; } + if (this.truncated || this.totalFrames(t) + frames > this.maxFrames || this.totalBytes + t.chunkBytes + whole > this.recordingBytesCap) { + this.truncate(t, frames); return false; + } + if (this.waitingWindows() >= this.windowSegments) { this.gapFrames(t, frames, "queue_full"); return false; } + let accepted = true; + for (let offset = 0; offset < whole; offset += this.host.chunkBytes) { + const slice = data.subarray(offset, Math.min(whole, offset + this.host.chunkBytes)); + if (!this.host.reserveBytes(slice.byteLength)) { this.gapFrames(t, slice.byteLength / t.frameBytes, "queue_full"); accepted = false; continue; } + // Mutable input: copied before the caller can reuse it. + this.dataFrames(t, slice.slice()); + } + return accepted; + } + private truncate(track: Track, frames: number): void { + this.truncated = true; + const ceiling = Math.floor(MAX_RECORDING_MS * this.rate / 1000); + const allowed = Math.max(0, Math.min(frames, ceiling - this.totalFrames(track))); + if (allowed) this.gapFrames(track, allowed, "truncated"); + } + markGap(durationMs: number, reason: AttachmentGapReason = "source_unavailable", track?: number): void { + try { + if (this.mode === "terminal" || this.mode === "finishing" || !Number.isFinite(durationMs) || durationMs <= 0 || !GAP_REASONS.includes(reason)) return; + let frames = Math.ceil(durationMs * this.rate / 1000); frames = Math.ceil(frames / this.g) * this.g; + const targets = track === undefined ? this.tracks : [this.tracks[track]].filter(Boolean); + for (const t of targets) { + const ceiling = Math.floor(MAX_RECORDING_MS * this.rate / 1000); + const allowed = Math.max(0, Math.min(frames, ceiling - this.totalFrames(t))); + if (allowed) this.gapFrames(t, allowed, reason); + } + } catch { /* bounded */ } + } + markPlayout(startMs: number, durationMs: number, evidence: "generated" | "played" | "unknown" | "truncated"): void { + if (this.mode === "terminal" || !Number.isInteger(startMs) || !Number.isInteger(durationMs) || startMs < 0 || durationMs < 1 || + startMs + durationMs > MAX_RECORDING_MS || !["generated", "played", "unknown", "truncated"].includes(evidence)) return; + if (this.playout.length >= MAX_PLAYOUT) return; + this.playout.push({ startMs, durationMs, evidence }); + } + status(): AttachmentRecordingStatus { + let sealed = 0; + for (const track of this.tracks) for (const piece of track.pieces) if (piece.kind === "segment") sealed++; + let owned = 0; for (const track of this.tracks) owned += track.chunkBytes; + for (const track of this.tracks) for (const piece of track.pieces) if (piece.kind === "segment" && piece.state !== "done" && piece.state !== "failed") owned += piece.bytes ?? 0; + return Object.freeze({ state: this.mode === "active" ? "recording" as const : this.mode, reason: (this.terminalReason ?? this.admissionReason) as AttachmentRecordingStatus["reason"], sealed, + uploaded: this.completed, gapMs: this.publishedGapMs, manifestRevision: this.revision, ownedBytes: owned }); + } + + // --- upload pipeline ------------------------------------------------------------------------------ + /** Where the next not-yet-sealed interval of this track starts. */ + private openStart(track: Track): number { return track.runFrames > 0 ? track.runStart : track.pos; } + private frontier(track: Track): number { + for (const piece of track.pieces) if (piece.kind === "segment" && piece.state !== "done" && piece.state !== "failed") return Math.min(piece.start, this.openStart(track)); + return this.openStart(track); + } + /** Releases sealed segments in (start, track ordinal) order once no earlier interval can still appear. */ + private release(): void { + if (this.mode === "terminal" && this.terminalReason !== "policy_denied") return; + for (;;) { + let head: Piece | undefined; + for (const track of this.tracks) for (const piece of track.pieces) { + if (piece.kind === "segment" && piece.state === "sealed" && (!head || piece.start < head.start || (piece.start === head.start && piece.track < head.track))) head = piece; + } + if (!head) return; + const finishing = this.mode === "finishing" || this.finalEnd !== undefined; + if (!finishing) { + for (const track of this.tracks) { + if (track.ordinal === head.track) continue; + const open = this.openStart(track); + if (open < head.start || (open === head.start && track.ordinal < head.track)) return; + } + } + this.enqueueSegment(head); + } + } + private enqueueSegment(piece: Piece): void { + piece.state = "queued"; this.outstanding++; + const controller = new AbortController(); + const job: CoreJob = { + lane: this.recordingId, bytes: piece.bytes ?? 0, controller, + run: signal => this.uploadSegment(piece, signal), + denied: () => this.segmentDenied(piece), + fail: reason => this.segmentSettled(piece, reason), + }; + piece.job = job; piece.bytes = 0; // the manager now owns the reservation and releases it when the job ends + this.host.enqueue(job); + } + private segmentDenied(piece: Piece): AttachmentAdmission | undefined { + if (this.mode === "terminal" && this.terminalReason !== "policy_denied" && this.terminalReason !== "conflict") return "incomplete"; + const track = this.tracks[piece.track]; + if (track?.denied) return "policy_denied"; + const reason = this.host.recheck({ mimeType: "audio/wav", direction: this.trackDirection(piece.track), purpose: this.options.purpose ?? "conversation_recording", + capture: this.options.capture, operation: this.init.sourceKind === "OwnedRecordingFile" ? "import_existing" : this.options.operation, + sourceKind: this.init.sourceKind ?? "Bytes" }, { ignoreSessionExpiry: false }); + return reason; + } + private trackDirection(ordinal: number): AttachmentDirection { + return this.options.layout === "separate_tracks" ? this.tracks[ordinal].spec.direction : "mixed"; + } + private segmentRequest(piece: Piece, index: number): AttachmentUploadRequest | undefined { + const session = this.host.currentSession(); + if (!session || !this.identity) return undefined; + const declared = piece.source instanceof Uint8Array ? piece.source.byteLength : (piece.source as { size: number }).size; + return { schema_version: 1, attachment_id: piece.attachmentId!, occurrence_id: piece.occurrenceId!, + association: { capture_session_id: this.host.sessionId, correlation_nonce: this.host.nonce, + parent: { trace_id: attachmentTraceUuid(this.identity.traceId), span_id: this.identity.spanId } }, + policy_version: session.policy.version, deletion_generation: 0, mime_type: "audio/wav", modality: "audio", + direction: this.trackDirection(piece.track), source_kind: this.init.sourceKind ?? "Bytes", + purpose: this.options.purpose ?? "conversation_recording", declared_bytes: declared, + recording_id: this.recordingId, segment_index: index, + ...(this.options.conversationId ? { conversation_id: this.options.conversationId } : {}), + ...(this.options.turnId ? { turn_id: this.options.turnId } : {}) }; + } + private async uploadSegment(piece: Piece, signal: AbortSignal): Promise { + if (piece.state !== "queued") return; + // Both manifest orders failed: the recording cannot publish, so stop spending uploads (design 2.8). + if (this.manifestFallback && this.manifestDisabled) { this.segmentSettled(piece, "incomplete"); return; } + const reason = this.segmentDenied(piece); + if (reason) { this.segmentSettled(piece, reason); return; } + piece.state = "uploading"; + const index = this.completed; const reused = this.indexReused; + const request = this.segmentRequest(piece, index); + if (!request) { this.segmentSettled(piece, "incomplete"); return; } + const body = JSON.stringify(request); + const outcome = await this.host.runUpload({ body, occurrenceId: piece.occurrenceId!, declaredBytes: request.declared_bytes, source: piece.source!, + recheck: () => this.segmentDenied(piece), + beforeComplete: this.manifestFallback ? async (o, s) => { await this.fallbackManifest(piece, o, s); } : undefined }, signal); + piece.outcome = outcome; + if (outcome.status === "validating") { piece.index = index; this.segmentSettled(piece, "validating"); return; } + if (outcome.status === "conflict" && reused && outcome.upload === undefined) this.closeOnConflict(); + this.segmentSettled(piece, outcome.status); + } + private segmentSettled(piece: Piece, status: AttachmentCompletion["status"]): void { + if (piece.state === "done" || piece.state === "failed") return; + piece.source = undefined; + if (status === "validating") { piece.state = "done"; this.completed++; this.sinceManifest++; this.indexReused = false; } + else { piece.state = "failed"; piece.failure = status; this.indexReused = true; this.host.countDrop(); } + this.outstanding--; + this.maybeManifest(false); + if (this.outstanding === 0) { const waiters = this.idleWaiters; this.idleWaiters = []; for (const w of waiters) w(); } + } + private closeOnConflict(): void { + // BR-4 fallback: a rejected reused index closes the recording with the published prefix plus an incomplete gap. + this.discardAll(); this.terminal("conflict"); + } + + // --- manifest ------------------------------------------------------------------------------------- + private settledEnd(): number { + let end = Infinity; + for (const track of this.tracks) end = Math.min(end, this.frontier(track)); + if (!Number.isFinite(end)) return 0; + for (let changed = true; changed;) { + changed = false; + for (const track of this.tracks) for (const piece of track.pieces) { + if (piece.kind === "segment" && piece.start < end && piece.end > end) { end = piece.start; changed = true; } + } + } + return end; + } + private latestCapability(): UploadOutcome | undefined { + let best: Piece | undefined; + for (const track of this.tracks) for (const piece of track.pieces) if (piece.state === "done" && piece.outcome?.upload && (!best || (piece.index ?? -1) > (best.index ?? -1))) best = piece; + return best?.outcome; + } + private capabilities(): UploadOutcome[] { + const all: Piece[] = []; + for (const track of this.tracks) for (const piece of track.pieces) if (piece.state === "done" && piece.outcome?.upload) all.push(piece); + return all.sort((a, b) => (b.index ?? 0) - (a.index ?? 0)).map(piece => piece.outcome!); + } + private ms(frames: number, final: boolean): number { + return final && this.finalEnd !== undefined && frames === this.finalEnd ? attachmentFramesToMs(frames, this.rate, true) : attachmentFramesToMs(frames, this.rate); + } + /** Cumulative, prefix-monotonic manifest over [0, end): earlier cut points always split later gaps. */ + buildManifest(end: number, final: boolean, extra?: Piece): AttachmentRecordingManifest { + const separate = this.options.layout === "separate_tracks"; + const segments: AttachmentRecordingManifest["segments"] = []; + const gaps: AttachmentRecordingManifest["gaps"] = []; + const cuts = [...this.cuts].sort((a, b) => a - b); + for (const track of this.tracks) { + const trackId = separate ? track.manifestId : undefined; + for (const piece of track.pieces) { + if (piece.start >= end) continue; + const pieceEnd = Math.min(piece.end, end); + const done = piece.kind === "segment" && (piece.state === "done" || piece === extra); + if (done && piece.end <= end) { + segments.push({ occurrence_id: piece.occurrenceId!, index: piece === extra ? this.completed : piece.index!, start_ms: this.ms(piece.start, final), + duration_ms: this.ms(piece.end, final) - this.ms(piece.start, final), ...(trackId ? { track_id: trackId } : {}) }); + continue; + } + const reason: AttachmentGapReason = piece.kind === "gap" ? piece.reason! : + piece.failure === "policy_denied" ? "policy_denied" : "incomplete"; + let start = piece.start; + for (const cut of [...cuts.filter(c => c > piece.start && c < pieceEnd), pieceEnd]) { + const a = this.ms(start, final); const b = this.ms(cut, final); + if (b > a) gaps.push({ start_ms: a, duration_ms: b - a, reason, ...(trackId ? { track_id: trackId } : {}) }); + start = cut; + } + } + } + segments.sort((a, b) => a.index - b.index); + const duration = this.ms(end, final); + const playout = this.playout.filter(p => p.startMs + p.durationMs <= duration).map(p => ({ start_ms: p.startMs, duration_ms: p.durationMs, evidence: p.evidence })); + const all = this.options.tracks.flatMap(t => [...t.channels]); + return { schema_version: 1, recording_id: this.recordingId, timebase: "session_relative_ms", sample_format: "s16le", sample_rate_hz: this.rate, + channels: all.length, codec: "pcm", container: "wav", layout: this.options.layout, + tracks: this.manifestTracks.map(t => ({ ...t, channels: [...t.channels] })), segments, gaps, playout, + capture_complete: final && gaps.length === 0 && segments.length > 0, duration_ms: duration }; + } + private maybeManifest(force: boolean): void { + if (this.manifestQueued || this.manifestDisabled || this.manifestFallback || this.finalEnd !== undefined) return; + if (this.mode === "terminal") return; + const end = this.settledEnd(); + if (end <= this.lastPutEnd || !this.latestCapability()) return; + const due = force || this.sinceManifest >= 4 || this.ms(end, false) - this.ms(this.lastPutEnd, false) >= 30000; + if (!due) return; + this.manifestQueued = true; this.outstanding++; + const controller = new AbortController(); + this.host.enqueue({ lane: this.recordingId, bytes: 0, controller, + run: async signal => { try { await this.putManifest(this.settledEnd(), false, signal); } finally { this.manifestQueued = false; this.manifestJobDone(); } }, + denied: () => this.mode === "terminal" && this.terminalReason !== "policy_denied" ? "incomplete" : undefined, + fail: () => { this.manifestQueued = false; this.manifestJobDone(); } }); + } + private manifestJobDone(): void { + this.outstanding--; + if (this.outstanding === 0) { const waiters = this.idleWaiters; this.idleWaiters = []; for (const w of waiters) w(); } + } + private async putManifest(end: number, final: boolean, signal: AbortSignal, extra?: Piece, holder?: UploadOutcome): Promise { + if (end <= 0 && !final) return false; + const manifest = this.buildManifest(end, final, extra); + if (!validAttachmentManifest(manifest)) { this.manifestDisabled = true; return false; } + const body = JSON.stringify(manifest); + if (new TextEncoder().encode(body).byteLength > MAX_MANIFEST_BYTES) { + // Closes as partial at the last successful revision; the writer stops retaining. + this.manifestDisabled = true; this.truncated = true; return false; + } + const revision = this.revision + 1; + const key = attachmentIdempotencyKey("manifest", this.recordingId, revision); + this.cuts.push(end); + const holders = holder ? [holder] : this.capabilities(); + for (const candidate of holders) { + if (signal.aborted) return false; + let result = await this.host.putManifest(candidate.upload!.write_capability, this.recordingId, body, key, signal, false); + if (result === "credential_scope_unsupported" || result === "expired") { + if (await this.host.renewUpload(candidate, signal)) result = await this.host.putManifest(candidate.upload!.write_capability, this.recordingId, body, key, signal, true); + } + if (result === "ok") { + this.revision = revision; this.lastPutEnd = end; this.sinceManifest = 0; + this.publishedSegments = manifest.segments.length; this.publishedComplete = manifest.capture_complete; + this.publishedGapMs = manifest.gaps.reduce((sum, gap) => sum + gap.duration_ms, 0); + return true; + } + if (result === "credential_scope_unsupported" || result === "policy_denied" || result === "conflict") { + // BR-7 fallback: put the manifest before complete for the remaining segments. + if (!final && !this.manifestFallback && !holder) { this.manifestFallback = true; return false; } + if (holder) throw new Error("manifest"); + continue; + } + if (result === "expired") continue; // an older completed segment's capability may still be valid (design 3.3) + return false; + } + return false; + } + private async fallbackManifest(piece: Piece, outcome: UploadOutcome, signal: AbortSignal): Promise { + piece.outcome = outcome; + // Hold this segment's complete until a later segment is sealed or the recording finishes, so the LAST + // segment's capability can still PUT the final manifest (no completed capability can do it afterwards). + const later = () => this.finalEnd !== undefined || this.mode === "terminal" || signal.aborted || + this.tracks.some(track => track.pieces.some(p => p !== piece && p.kind === "segment" && p.state !== "done" && p.state !== "failed")); + if (!later()) { + await new Promise(resolve => { + const done = () => { clearTimeout(timer); signal.removeEventListener("abort", done); resolve(); }; + const timer = setTimeout(done, MAX_SEGMENT_MS + 5000); (timer as unknown as { unref?: () => void }).unref?.(); + this.sealWaiters.push(done); signal.addEventListener("abort", done, { once: true }); + }); + } + // Prospectively include this segment; its own capability authorizes the PUT. + const savedState = piece.state; piece.state = "done"; piece.index = this.completed; + const end = this.settledEnd(); + let pending = 0; + for (const track of this.tracks) for (const p of track.pieces) if (p !== piece && p.kind === "segment" && p.state !== "done" && p.state !== "failed") pending++; + piece.state = savedState; + // The last segment of a finishing recording carries the FINAL manifest (no completed capability can PUT it later). + const final = this.finalEnd !== undefined && pending === 0 && end === this.finalEnd; + let ok = false; + try { ok = await this.putManifest(end, final, signal, piece, outcome); } catch { ok = false; } + if (ok && final) this.finalPublished = true; + piece.index = undefined; + if (!ok) { this.manifestDisabled = true; throw new Error("manifest"); } + } + + // --- finish / abort ------------------------------------------------------------------------------ + private discardAll(): void { + for (const track of this.tracks) { + if (track.chunkBytes) this.host.releaseBytes(track.chunkBytes); + track.chunks = []; track.chunkBytes = 0; track.blockData = 0; track.runFrames = 0; track.carry = undefined; + for (const piece of track.pieces) { + if (piece.kind !== "segment") continue; + if (piece.state === "sealed") { if (piece.bytes) this.host.releaseBytes(piece.bytes); piece.bytes = 0; piece.state = "failed"; piece.failure = "incomplete"; piece.source = undefined; } + else if (piece.state === "queued" || piece.state === "uploading") piece.job?.controller.abort(); + } + } + } + private dropTrack(track: Track): void { + if (track.chunkBytes) this.host.releaseBytes(track.chunkBytes); + track.chunks = []; track.chunkBytes = 0; track.blockData = 0; + if (track.runFrames) { this.appendGap(track, "policy_denied", track.runFrames); track.runFrames = 0; } + for (const piece of track.pieces) if (piece.kind === "segment" && (piece.state === "sealed")) { + if (piece.bytes) this.host.releaseBytes(piece.bytes); piece.bytes = 0; piece.state = "failed"; piece.failure = "policy_denied"; piece.source = undefined; + } + } + private policyStop(): void { + // Tightening: retained bytes are freed and the remaining timeline becomes a policy_denied gap. + for (const track of this.tracks) { + if (track.blockData) this.releaseTrackBlock(track); + if (track.runFrames) { const frames = track.runFrames; this.host.releaseBytes(track.chunkBytes); track.chunks = []; track.chunkBytes = 0; track.runFrames = 0; + track.pieces.push({ kind: "gap", track: track.ordinal, start: track.runStart, end: track.runStart + frames, reason: "policy_denied" }); track.runStart = track.pos; } + for (const piece of track.pieces) if (piece.kind === "segment" && piece.state === "sealed") { + if (piece.bytes) this.host.releaseBytes(piece.bytes); piece.bytes = 0; piece.state = "failed"; piece.failure = "policy_denied"; piece.source = undefined; + } + } + this.terminal("policy_denied"); + } + private sealAll(): void { + for (const track of this.tracks) { + if (track.carry) track.carry = undefined; + if (track.fill) { + const frames = track.fill; + if (track.blockGap === undefined) { track.runFrames += frames; track.blockData = 0; track.pos += frames; track.fill = 0; } + else { this.closeRun(track); this.appendGap(track, track.blockGap, frames); track.pos += frames; track.fill = 0; track.blockGap = undefined; } + } + this.closeRun(track); + } + const end = Math.max(0, ...this.tracks.map(track => track.pos)); + for (const track of this.tracks) if (track.pos < end) { this.appendGap(track, track.denied ? "policy_denied" : "source_unavailable", end - track.pos); track.pos = end; } + this.finalEnd = end; + this.wakeSeal(); + } + private idle(deadline: number): Promise { + if (this.outstanding === 0) return Promise.resolve(); + return new Promise(resolve => { + const done = () => { clearTimeout(timer); resolve(); }; + // The caller awaits finish(): a ref'd timer bounded by its own deadline gives it the grace period. + const timer = setTimeout(() => { this.idleWaiters = this.idleWaiters.filter(w => w !== done); resolve(); }, Math.max(0, deadline - this.host.clock.now())); + this.idleWaiters.push(done); + }); + } + finish(timeoutMs?: number): Promise { + if (!this.resultPromise) this.resultPromise = this.finishInternal(timeoutMs).catch(() => this.result("incomplete")); + return this.resultPromise; + } + private async finishInternal(timeoutMs?: number): Promise { + const budget = Math.min(Number.isFinite(timeoutMs) && timeoutMs! >= 0 ? timeoutMs! : this.host.finalizationTimeoutMs, this.host.finalizationTimeoutMs); + const start = this.host.clock.now(); const deadline = start + budget; + if (this.mode === "pending" || (this.mode === "terminal" && this.revision === 0 && !this.latestCapability() && this.admissionReason !== "queued")) { + const denied = this.admissionReason !== "queued"; + this.discardAll(); this.terminal(this.terminalReason ?? this.admissionReason); this.closeOnce(); + return this.result(denied ? "denied" : undefined); + } + const aborted = this.terminalReason === "incomplete" && this.mode === "terminal"; + if (aborted) { this.closeOnce(); return this.result(); } + const wasTerminal = this.mode === "terminal"; + if (!wasTerminal) this.mode = "finishing"; + this.sealAll(); + this.release(); + const reserve = Math.min(2000, budget / 4); + await this.idle(deadline - reserve); + // At the deadline: queued and in-flight segments are dropped and become incomplete gaps. + for (const track of this.tracks) for (const piece of track.pieces) { + if (piece.kind === "segment" && piece.state !== "done" && piece.state !== "failed") { + piece.job?.controller.abort(); + if (piece.state === "sealed" && piece.bytes) { this.host.releaseBytes(piece.bytes); piece.bytes = 0; } + piece.state = "failed"; piece.failure = "incomplete"; piece.source = undefined; + } + } + if (this.latestCapability() && !this.manifestDisabled && !this.finalPublished) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), Math.max(1, deadline - this.host.clock.now())); + try { await this.raceDeadline(this.putManifest(this.finalEnd ?? 0, true, controller.signal), deadline); } + catch { /* bounded */ } finally { clearTimeout(timer); controller.abort(); } + } + this.terminal(this.terminalReason ?? "incomplete"); + this.closeOnce(); + return this.result(); + } + private raceDeadline(promise: Promise, deadline: number): Promise { + return new Promise(resolve => { + const timer = setTimeout(() => resolve(undefined), Math.max(0, deadline - this.host.clock.now())); + promise.then(value => { clearTimeout(timer); resolve(value); }, () => { clearTimeout(timer); resolve(undefined); }); + }); + } + private result(forced?: "denied" | "incomplete"): AttachmentRecordingResult { + const state: AttachmentRecordingResult["state"] = forced === "denied" || (this.admissionReason !== "queued" && this.revision === 0 && !this.completed) ? "denied" : + this.revision === 0 ? "unpublished" : this.publishedComplete ? "published" : "partial"; + const reason = state === "published" ? undefined : String(this.terminalReason ?? this.admissionReason); + return Object.freeze({ recordingId: this.recordingId, state, ...(reason ? { reason } : {}), segmentsPublished: this.publishedSegments, + gapMs: this.publishedGapMs, captureComplete: this.publishedComplete, manifestRevision: this.revision }); + } + abort(): void { + try { + if (this.mode === "terminal" && this.closed) return; + this.discardAll(); this.terminal("incomplete"); + if (!this.resultPromise) this.resultPromise = Promise.resolve(this.result()); + this.closeOnce(); + } catch { /* abort never throws */ } + } + /** Manager shutdown/fork-style invalidation: finish with the remaining budget. */ + shutdown(timeoutMs: number): Promise { return this.finish(timeoutMs); } + + /** Owned WAV (tier wav_recording): segments are pre-built lease ranges. */ + loadFileSegments(segments: readonly RecordingFileSegment[], totalFrames: number): void { + if (this.mode !== "active") return; + for (const segment of segments) { + const track = this.tracks[segment.track]; + track.pieces.push({ kind: "segment", track: segment.track, start: segment.startFrame, end: segment.endFrame, source: segment.source, bytes: 0, + state: "sealed", occurrenceId: uuid(), attachmentId: uuid() }); + } + for (const track of this.tracks) track.pos = totalFrames; + this.finalEnd = totalFrames; this.mode = "finishing"; + this.release(); + } + /** Owned WAV: wait for every segment and PUT the final manifest; cancellation drops the rest. */ + async completeFile(signal: AbortSignal): Promise { + await new Promise(resolve => { + if (this.outstanding === 0 || signal.aborted) { resolve(); return; } + const done = () => { signal.removeEventListener("abort", done); resolve(); }; + this.idleWaiters.push(done); signal.addEventListener("abort", done, { once: true }); + }); + if (signal.aborted) { this.discardAll(); this.terminal("incomplete"); this.closeOnce(); return "incomplete"; } + let failed: AttachmentCompletion["status"] | undefined; + for (const track of this.tracks) for (const piece of track.pieces) if (piece.state === "failed") failed = failed ?? piece.failure; + if (this.latestCapability() && !this.finalPublished && !this.manifestDisabled) await this.putManifest(this.finalEnd ?? 0, true, signal); + this.terminal(failed ?? "incomplete"); this.closeOnce(); + return this.revision === 0 ? (failed ?? "incomplete") : failed ?? (this.publishedComplete ? "validating" : "incomplete"); + } + /** @internal capture policy snapshot for owned file admission. */ + get capture(): AttachmentCapturePolicy | undefined { return this.options.capture; } +} diff --git a/typescript/packages/fi-core/src/attachments/stream.ts b/typescript/packages/fi-core/src/attachments/stream.ts new file mode 100644 index 000000000..47bf6cf63 --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/stream.ts @@ -0,0 +1,44 @@ +import type { AttachmentAdmission, AttachmentHandle, AttachmentStream } from "./types"; + +/** @internal Mutable state the manager keeps (and sweeps) for one consumed stream. */ +export interface StreamState { + reason: AttachmentAdmission; + readonly attachmentId?: string; + readonly occurrenceId?: string; + chunks: Uint8Array[]; + length: number; + readonly maxBytes: number; + /** Bytes still reserved for this stream (the whole maxBytes until finish). */ + reserved: number; + lastWrite: number; + readonly idleTtl: number; + readonly epoch: number; + open: boolean; + handle?: AttachmentHandle; +} +/** @internal */ +export interface StreamHost { + streamWrite(state: StreamState, chunk: Uint8Array): boolean; + streamFinish(state: StreamState): AttachmentHandle; + streamAbort(state: StreamState, reason: "incomplete" | "policy_denied" | "too_large" | "correlation_unavailable" | "unsampled"): void; +} +/** + * Single-object upload of chunks the application ALREADY consumed (design section 6). The core never pulls, clones + * or tees a stream; adapters forward fulfilled reads to tryWrite. + */ +export class AttachmentConsumedStream implements AttachmentStream { + constructor(private readonly host: StreamHost | undefined, /** @internal */ readonly state: StreamState) {} + get reason(): AttachmentAdmission { return this.state.reason; } + get attachmentId(): string | undefined { return this.state.reason === "queued" || this.state.handle ? this.state.attachmentId : undefined; } + get occurrenceId(): string | undefined { return this.state.reason === "queued" || this.state.handle ? this.state.occurrenceId : undefined; } + tryWrite(chunk: Uint8Array): boolean { + try { return !!this.host && this.state.open && this.host.streamWrite(this.state, chunk); } catch { return false; } + } + finish(): AttachmentHandle { + if (!this.host) return this.state.handle!; + return this.host.streamFinish(this.state); + } + abort(reason: "incomplete" | "policy_denied" = "incomplete"): void { + try { if (this.host && this.state.open) this.host.streamAbort(this.state, reason === "policy_denied" ? "policy_denied" : "incomplete"); } catch { /* never throws */ } + } +} diff --git a/typescript/packages/fi-core/src/attachments/transport-v1.ts b/typescript/packages/fi-core/src/attachments/transport-v1.ts new file mode 100644 index 000000000..8dad0faef --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/transport-v1.ts @@ -0,0 +1,401 @@ +import { context } from "@opentelemetry/api"; +import { suppressTracing } from "@opentelemetry/core"; +import { validAttachmentUuid } from "./correlation"; +import { attachmentWarnLegacyTransport, attachmentWireRejected } from "./warn"; +import type { + AttachmentAdmission, AttachmentBootstrap, AttachmentCompletionStatus, AttachmentContentSource, AttachmentImportJob, AttachmentPacer, + AttachmentSession, AttachmentTransport, AttachmentTransportV1, AttachmentUpload, AttachmentUploadRequest, +} from "./types"; + +export type AttachmentErrorCode = AttachmentAdmission | AttachmentCompletionStatus | "failed"; +/** Safe errors have only fixed codes, never response bodies, credentials or URLs. */ +export class AttachmentTransportError extends Error { + constructor(readonly code: AttachmentErrorCode, readonly retryable = false, readonly retryAfterMs = 0, readonly httpStatus = 0) { + super(code); this.name = "AttachmentTransportError"; + } +} +export interface AttachmentHttpRequest { + url: string; method: "GET" | "POST" | "PUT"; headers: Record; + body?: string | Uint8Array | Blob | AttachmentContentSource; signal: AbortSignal; + /** Content PUT only: per-slice stall timeout and pacing. */ + stallTimeoutMs?: number; pacer?: AttachmentPacer; +} +export interface AttachmentHttpResponse { status: number; body: Uint8Array; retryAfter?: string } +export type AttachmentHttpClient = (request: AttachmentHttpRequest) => Promise; + +export const ATTACHMENT_CHUNK_BYTES = 256 * 1024; +const SUCCESS_BODY = 65536; +const ERROR_BODY = 4096; +const TOKEN = /^[a-zA-Z0-9_-]{1,128}$/; +const REASONS = new Set(["queued", "disabled", "bootstrap_pending", "policy_denied", "credential_scope_unsupported", "source_unavailable", "unsampled", + "too_large", "queue_full", "unsupported_version", "endpoint_unconfigured", "scope_unresolved", "correlation_unavailable", "redaction_unavailable", + "recording_disabled", "binding_pending", "not_found", "conflict", "throttled", "validation_failed", "expired", "deleted", "incomplete", "ready"]); +const STATES = new Set(["pending_upload", "importing", "validating", "ready", "failed", "rejected", "expired", "deleted"]); + +/** Strict JSON: duplicate members and non-JSON numbers are rejected (F01 validate.py strict_json). */ +export function attachmentStrictJson(text: string): unknown { + let i = 0; + const ws = () => { while (i < text.length && " \t\n\r".includes(text[i])) i++; }; + const fail = (): never => { throw new SyntaxError("invalid JSON"); }; + const str = (): string => { + const start = i; i++; + for (; i < text.length; i++) { + const c = text[i]; + if (c === "\\") { i++; continue; } + if (c === '"') { i++; return JSON.parse(text.slice(start, i)); } + if (c < " ") fail(); + } + return fail(); + }; + const value = (depth: number): unknown => { + if (depth > 64) fail(); + ws(); const c = text[i]; + if (c === "{") { + i++; const out: Record = {}; const keys = new Set(); ws(); + if (text[i] === "}") { i++; return out; } + for (;;) { + ws(); if (text[i] !== '"') fail(); const key = str(); if (keys.has(key)) fail(); keys.add(key); + ws(); if (text[i] !== ":") fail(); i++; + Object.defineProperty(out, key, { value: value(depth + 1), enumerable: true, writable: true, configurable: true }); + ws(); if (text[i] === ",") { i++; continue; } if (text[i] === "}") { i++; return out; } fail(); + } + } + if (c === "[") { + i++; const out: unknown[] = []; ws(); + if (text[i] === "]") { i++; return out; } + for (;;) { out.push(value(depth + 1)); ws(); if (text[i] === ",") { i++; continue; } if (text[i] === "]") { i++; return out; } fail(); } + } + if (c === '"') return str(); + const m = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?|^true|^false|^null/.exec(text.slice(i, i + 400)); + if (!m) return fail(); + i += m[0].length; return JSON.parse(m[0]); + }; + const result = value(0); ws(); if (i !== text.length) fail(); + return result; +} + +const REFINE: Readonly> = { + 400: ["unsupported_version"], 401: ["credential_scope_unsupported", "expired"], + 403: ["policy_denied", "redaction_unavailable", "recording_disabled", "correlation_unavailable", "credential_scope_unsupported"], + 404: ["not_found", "scope_unresolved"], 409: ["conflict", "deleted", "expired", "policy_denied"], 410: ["expired", "deleted"], 413: ["too_large"], +}; +const DEFAULT_REASON: Readonly> = { + 400: "unsupported_version", 401: "credential_scope_unsupported", 403: "policy_denied", 404: "not_found", 409: "conflict", 410: "expired", 413: "too_large", 429: "throttled", +}; +/** Integer delta-seconds only, capped at 5 s; an HTTP-date is treated as 1 s (design 0.4). */ +export function attachmentRetryAfterMs(value: string | undefined, status: number): number { + if (value === undefined || value === "") return status === 429 ? 1000 : 0; + const trimmed = String(value).trim(); + if (/^\d{1,9}$/.test(trimmed)) return Math.min(Number(trimmed) * 1000, 5000); + return 1000; +} +/** One HTTP status -> reason table for every route (design 0.5). */ +export function attachmentHttpError(status: number, body?: Uint8Array, retryAfter?: string): AttachmentTransportError { + let code: AttachmentErrorCode = DEFAULT_REASON[status] ?? "incomplete"; + const refine = REFINE[status]; + if (refine && body && body.byteLength && body.byteLength <= ERROR_BODY) { + try { + const parsed = attachmentStrictJson(new TextDecoder("utf-8", { fatal: true }).decode(body)) as Record; + if (object(parsed) && Object.keys(parsed).every(key => key === "code" || key === "request_id") && validAttachmentUuid(parsed.request_id) && + typeof parsed.code === "string" && REASONS.has(parsed.code) && refine.includes(parsed.code)) code = parsed.code as AttachmentErrorCode; + } catch { /* Unparseable error bodies are ignored. */ } + } + const retryable = status === 429 || status === 408 || status === 500 || status === 502 || status === 503 || status === 504; + return new AttachmentTransportError(code, retryable, retryable ? attachmentRetryAfterMs(retryAfter, status) : 0, status); +} + +/** Idempotency keys (design 0.6). F01 Token: ^[a-zA-Z0-9_-]{1,128}$. */ +export function attachmentIdempotencyKey(route: "bootstrap" | "upload" | "complete" | "abort" | "manifest" | "import", id: string, sequence?: number): string { + const hex = String(id).replace(/-/g, ""); + const prefix = { bootstrap: "bs", upload: "up", complete: "cp", abort: "ab", manifest: "mf", import: "im" }[route]; + return sequence === undefined ? `${prefix}-${hex}` : `${prefix}-${hex}-${sequence}`; +} + +async function readBounded(response: Response, limit: number): Promise { + const reader = response.body?.getReader(); + const chunks: Uint8Array[] = []; let size = 0; + if (reader) { + try { + for (;;) { + const { value, done } = await reader.read(); if (done) break; + size += value.byteLength; + if (size > limit) return undefined; + chunks.push(value); + } + } finally { await reader.cancel().catch(() => undefined); } + } + const body = new Uint8Array(size); let offset = 0; + for (const chunk of chunks) { body.set(chunk, offset); offset += chunk.length; } + return body; +} +/** Materialize an in-memory content source (browser fetch has no streaming request bodies everywhere). */ +export async function attachmentContentBytes(source: AttachmentContentSource, signal: AbortSignal): Promise { + if (source instanceof Uint8Array || (typeof Blob !== "undefined" && source instanceof Blob)) return source; + const ranged = source as Exclude; + const out = new Uint8Array(ranged.size); + for (let offset = 0; offset < ranged.size; offset += ATTACHMENT_CHUNK_BYTES) { + if (signal.aborted) throw new AttachmentTransportError("incomplete"); + await ranged.check(); + const part = await ranged.read(offset, Math.min(ATTACHMENT_CHUNK_BYTES, ranged.size - offset)); + if (part.byteLength !== Math.min(ATTACHMENT_CHUNK_BYTES, ranged.size - offset)) throw new AttachmentTransportError("source_unavailable"); + out.set(part, offset); + } + await ranged.check(); + return out; +} +/** No cookies, redirects, arbitrary destination paths or unbounded JSON reads. */ +export const attachmentFetchClient: AttachmentHttpClient = async request => { + try { + const body = request.body === undefined || typeof request.body === "string" ? request.body : await attachmentContentBytes(request.body, request.signal); + if (body !== undefined && typeof body !== "string" && request.pacer) await request.pacer.take(body instanceof Uint8Array ? body.byteLength : body.size, request.signal); + const response = await fetch(request.url, { + method: request.method, headers: request.headers, body: body as unknown as RequestInit["body"], + signal: request.signal, redirect: "error", credentials: "omit", + }); + const ok = response.status >= 200 && response.status < 300; + const data = await readBounded(response, ok ? SUCCESS_BODY : ERROR_BODY); + if (!data) { + if (ok) throw attachmentWireRejected("response size", new AttachmentTransportError("unsupported_version")); + return { status: response.status, body: new Uint8Array(), retryAfter: response.headers.get("retry-after") ?? undefined }; + } + return { status: response.status, body: data, retryAfter: response.headers.get("retry-after") ?? undefined }; + } catch (error) { + if (error instanceof AttachmentTransportError) throw error; + // A refused redirect surfaces as a network TypeError; the abort signal tells cancellation apart. + throw new AttachmentTransportError("incomplete", !request.signal.aborted); + } +}; +function object(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } +const LIMIT_MAX: Readonly> = { max_upload_bytes: 52428800, segment_bytes: 8388608, segment_duration_ms: 30000, + recording_bytes: 1073741824, recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, + reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, links_per_minute: 1000, + upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, + grant_ttl_seconds: 300, revocation_check_seconds: 5 }; +export function validAttachmentSession(value: unknown, sessionId: string, now = Date.now()): value is AttachmentSession { + if (!object(value) || value.schema_version !== 1 || value.capture_session_id !== sessionId || !object(value.scope) || + ![value.scope.organization_id, value.scope.workspace_id, value.scope.project_id].every(validAttachmentUuid) || + !Number.isFinite(Date.parse(value.expires_at)) || Date.parse(value.expires_at) <= now || + !Array.isArray(value.transfer_modes) || value.transfer_modes.length !== 1 || value.transfer_modes[0] !== "authenticated_stream") return false; + const p = value.policy; + if (!object(p) || typeof p.version !== "string" || p.version.trim() !== p.version || !TOKEN.test(p.version) || + !["enabled", "require_media_redaction", "require_metadata_removal", "allow_unredacted_media_with_text_pii", "record_audio", "record_video"].every(key => typeof p[key] === "boolean") || + !Array.isArray(p.allowed_sources) || p.allowed_sources.length > 6 || !p.allowed_sources.every((s: unknown) => ["Bytes", "Blob", "RemoteUrl", "LocalFile", "OwnedRecordingFile", "ProviderFile"].includes(s as string)) || + !Array.isArray(p.allowed_modalities) || p.allowed_modalities.length > 5 || !p.allowed_modalities.every((m: unknown) => ["audio", "image", "video", "document", "file"].includes(m as string))) return false; + return object(value.limits) && Object.entries(LIMIT_MAX).every(([key, max]) => Number.isInteger(value.limits[key]) && value.limits[key] > 0 && value.limits[key] <= max); +} +export function validAttachmentUpload(value: unknown, request: AttachmentUploadRequest, now = Date.now()): value is AttachmentUpload { + return object(value) && validAttachmentUuid(value.upload_id) && value.attachment_id === request.attachment_id && + value.occurrence_id === request.occurrence_id && value.state === "pending_upload" && value.reserved_bytes === request.declared_bytes && + Number.isFinite(Date.parse(value.expires_at)) && Date.parse(value.expires_at) > now && value.transfer_mode === "authenticated_stream" && + typeof value.write_capability === "string" && value.write_capability.trim() === value.write_capability && /^[a-zA-Z0-9_.-]{16,4096}$/.test(value.write_capability); +} +function validStatus(value: unknown, id: string, key: "upload_id" | "job_id", state?: string): value is { state: string; reason: string } { + return object(value) && Object.keys(value).length === 3 && value[key] === id && typeof value.state === "string" && STATES.has(value.state) && + typeof value.reason === "string" && REASONS.has(value.reason) && (state === undefined || value.state === state); +} +export interface AttachmentHttpTransportOptions { + endpoint: string; + /** Explicitly supplied credential/capability. Never read environment in browser code. */ + headers: Readonly>; + client?: AttachmentHttpClient; + /** Per-slice stall timeout for content PUTs (default 2 s). */ + stallTimeoutMs?: number; +} +/** Symbol under which a legacy-shaped transport carries its full versioned surface. */ +export const ATTACHMENT_TRANSPORT_V1: unique symbol = Symbol.for("@traceai/fi-core/attachments/transport-v1"); + +/** The full design section 9 surface over HTTPS. */ +export function createAttachmentHttpTransportV1(options: AttachmentHttpTransportOptions): AttachmentTransportV1 { + const url = new URL(options.endpoint); + if (url.protocol !== "https:" || url.username || url.password || url.search || url.hash || url.pathname !== "/") { + throw new AttachmentTransportError("endpoint_unconfigured"); + } + const origin = url.origin; + const headers: Record = {}; + const allowed = new Set(["authorization", "x-api-key", "x-secret-key", "x-workspace-id"]); + for (const [name, value] of Object.entries(options.headers)) { + if (allowed.has(name.toLowerCase()) && typeof value === "string" && value.length <= 8192 && !/[\r\n]/.test(value)) headers[name.toLowerCase()] = value; + } + const client = options.client ?? attachmentFetchClient; + const stall = Number.isInteger(options.stallTimeoutMs) && options.stallTimeoutMs! > 0 ? options.stallTimeoutMs! : 2000; + const bootstraps = new Map(); + async function send(method: "GET" | "POST" | "PUT", path: string, body: AttachmentHttpRequest["body"], signal: AbortSignal, expected: number, + extra: { key?: string; capability?: string; binary?: boolean; pacer?: AttachmentPacer } = {}): Promise { + if (signal.aborted) throw new AttachmentTransportError("incomplete"); + let response: AttachmentHttpResponse; + const auth = extra.capability !== undefined ? { authorization: `Bearer ${extra.capability}` } : headers; + try { + response = await context.with(suppressTracing(context.active()), () => client({ + url: origin + path, method, signal, body, + headers: { ...auth, ...(body === undefined ? {} : { "content-type": extra.binary ? "application/octet-stream" : "application/json" }), + "cache-control": "no-store", ...(extra.key ? { "idempotency-key": extra.key } : {}) }, + ...(extra.binary ? { stallTimeoutMs: stall, pacer: extra.pacer } : {}), + })); + } catch (error) { + if (error instanceof AttachmentTransportError) throw error.code === "failed" ? new AttachmentTransportError("incomplete", error.retryable) : error; + throw new AttachmentTransportError("incomplete", !signal.aborted); + } + if (response.status !== expected) { + if (response.status >= 200 && response.status < 300) throw attachmentWireRejected("HTTP success status", new AttachmentTransportError("unsupported_version")); + throw attachmentHttpError(response.status, response.body, response.retryAfter); + } + if (response.body.byteLength > SUCCESS_BODY) throw attachmentWireRejected("response size", new AttachmentTransportError("unsupported_version")); + if (expected === 204) return undefined; + try { return attachmentStrictJson(new TextDecoder("utf-8", { fatal: true }).decode(response.body)); } + catch { throw attachmentWireRejected("strict JSON body", new AttachmentTransportError("unsupported_version")); } + } + const base = (session: AttachmentSession) => { + if (!validAttachmentUuid(session.scope.project_id)) throw new AttachmentTransportError("scope_unresolved"); + return `/tracer/v1/projects/${session.scope.project_id}/attachments`; + }; + const uploadPath = (session: AttachmentSession, upload: AttachmentUpload) => { + if (!validAttachmentUuid(upload.upload_id)) throw new AttachmentTransportError("unsupported_version"); + return `${base(session)}/uploads/${upload.upload_id}`; + }; + const parseUploadBody = (body: string): AttachmentUploadRequest => { + try { return attachmentStrictJson(body) as AttachmentUploadRequest; } catch { throw new AttachmentTransportError("unsupported_version"); } + }; + const completeBody = (session: AttachmentSession) => JSON.stringify({ policy_version: session.policy.version, deletion_generation: 0 }); + const v1: AttachmentTransportV1 = { + async bootstrap(body, key, signal) { + const result = await send("POST", "/tracer/v1/attachment-sessions", JSON.stringify(body), signal, 200, { key }); + if (!validAttachmentSession(result, body.capture_session_id)) throw attachmentWireRejected("AttachmentSession", new AttachmentTransportError("unsupported_version")); + bootstraps.set(body.capture_session_id, { ...body }); + return result; + }, + async renewSession(current, sequence, signal) { + // F01 v1 has no renewal route: replay bootstrap with the SAME session id and nonce under a new key (BR-2, CQ-1). + const body = bootstraps.get(current.capture_session_id); + if (!body) throw new AttachmentTransportError("unsupported_version"); + const result = await send("POST", "/tracer/v1/attachment-sessions", JSON.stringify(body), signal, 200, + { key: attachmentIdempotencyKey("bootstrap", current.capture_session_id, sequence) }); + if (!validAttachmentSession(result, body.capture_session_id)) throw attachmentWireRejected("AttachmentSession", new AttachmentTransportError("unsupported_version")); + return result; + }, + async initialize(session, body, key, signal) { + const request = parseUploadBody(body); + const result = await send("POST", `${base(session)}/uploads`, body, signal, 201, { key }); + if (!validAttachmentUpload(result, request)) throw attachmentWireRejected("UploadResponse", new AttachmentTransportError("unsupported_version")); + return result; + }, + async renewUpload(session, upload, body, signal) { + // Replay the byte-identical initialize under the same key with the ORIGINAL identity (BR-1). + const request = parseUploadBody(body); + const result = await send("POST", `${base(session)}/uploads`, body, signal, 201, { key: attachmentIdempotencyKey("upload", request.occurrence_id) }); + if (!validAttachmentUpload(result, request)) throw attachmentWireRejected("UploadResponse", new AttachmentTransportError("unsupported_version")); + if (result.upload_id !== upload.upload_id || result.write_capability === upload.write_capability || + Date.parse(result.expires_at) <= Date.parse(upload.expires_at)) throw new AttachmentTransportError("expired"); + return result; + }, + async putContent(session, upload, source, signal, pacer) { + const size = source instanceof Uint8Array ? source.byteLength : (source as { size: number }).size; + if (size !== upload.reserved_bytes) throw new AttachmentTransportError("too_large"); + const result = await send("PUT", `${uploadPath(session, upload)}/content`, source, signal, 200, { capability: upload.write_capability, binary: true, pacer }); + if (!validStatus(result, upload.upload_id, "upload_id", "pending_upload")) throw attachmentWireRejected("UploadStatus", new AttachmentTransportError("unsupported_version")); + }, + async complete(session, upload, key, signal) { + const result = await send("POST", `${uploadPath(session, upload)}/complete`, completeBody(session), signal, 202, { key, capability: upload.write_capability }); + // validating, never ready: late-root and root-first orders both still need validation (F01 lifecycle fixtures). + if (!validStatus(result, upload.upload_id, "upload_id", "validating")) throw attachmentWireRejected("UploadStatus", new AttachmentTransportError("unsupported_version")); + return "validating"; + }, + async abort(session, upload, key, signal) { + const result = await send("POST", `${uploadPath(session, upload)}/abort`, completeBody(session), signal, 200, { key, capability: upload.write_capability }); + if (!validStatus(result, upload.upload_id, "upload_id")) throw attachmentWireRejected("UploadStatus", new AttachmentTransportError("unsupported_version")); + }, + async uploadStatus(session, upload, signal) { + const result = await send("GET", `${uploadPath(session, upload)}/status`, undefined, signal, 200, { capability: upload.write_capability }); + if (!validStatus(result, upload.upload_id, "upload_id")) throw attachmentWireRejected("UploadStatus", new AttachmentTransportError("unsupported_version")); + return { state: result.state, reason: result.reason }; + }, + async putManifest(session, capability, recordingId, body, key, signal) { + if (!validAttachmentUuid(recordingId) || !/^[a-zA-Z0-9_.-]{16,4096}$/.test(capability)) throw new AttachmentTransportError("unsupported_version"); + if (new TextEncoder().encode(body).byteLength > 1048576) throw new AttachmentTransportError("too_large"); + await send("PUT", `${base(session)}/recordings/${recordingId}/manifest`, body, signal, 204, { key, capability }); + }, + async submitImport(session, body, key, signal) { + let recordingId: string; + try { recordingId = (attachmentStrictJson(body) as { recording: { recording_id: string } }).recording.recording_id; } + catch { throw new AttachmentTransportError("unsupported_version"); } + const result = await send("POST", `${base(session)}/imports`, body, signal, 202, { key }); + if (!object(result) || Object.keys(result).length !== 4 || result.mode !== "completed_recording" || result.recording_id !== recordingId || + result.state !== "importing" || !validAttachmentUuid(result.job_id)) throw attachmentWireRejected("RecordingImportResponse", new AttachmentTransportError("unsupported_version")); + return { jobId: result.job_id, recordingId, state: "importing" } as AttachmentImportJob; + }, + async importStatus(session, jobId, signal) { + if (!validAttachmentUuid(jobId)) throw new AttachmentTransportError("unsupported_version"); + const result = await send("GET", `${base(session)}/imports/${jobId}/status`, undefined, signal, 200); + if (!validStatus(result, jobId, "job_id")) throw attachmentWireRejected("ImportStatus", new AttachmentTransportError("unsupported_version")); + return { state: result.state, reason: result.reason }; + }, + }; + return v1; +} + +/** + * Legacy four-method shape, kept for existing doubles and adapters. The returned object also carries the full + * versioned surface under ATTACHMENT_TRANSPORT_V1, which the manager prefers. + */ +export function createAttachmentHttpTransport(options: AttachmentHttpTransportOptions): AttachmentTransport { + const v1 = createAttachmentHttpTransportV1(options); + const legacy: AttachmentTransport & { [ATTACHMENT_TRANSPORT_V1]: AttachmentTransportV1 } = { + bootstrap: (request, signal) => v1.bootstrap(request, attachmentIdempotencyKey("bootstrap", request.capture_session_id, 0), signal), + initialize: (session, request, signal) => v1.initialize(session, JSON.stringify(request), attachmentIdempotencyKey("upload", request.occurrence_id), signal), + upload: (session, upload, data, signal) => v1.putContent(session, upload, data, signal, attachmentNoPacer), + complete: async (session, upload, _request, signal) => { await v1.complete(session, upload, attachmentIdempotencyKey("complete", upload.upload_id), signal); }, + [ATTACHMENT_TRANSPORT_V1]: v1, + }; + return legacy; +} +export const attachmentNoPacer: AttachmentPacer = { take: async () => undefined }; + +/** Adapt a legacy four-method transport. Routes it cannot express fail closed. */ +export function attachmentLegacyTransport(transport: AttachmentTransport): AttachmentTransportV1 { + const requests = new WeakMap(); + const unsupported = async (): Promise => { throw new AttachmentTransportError("unsupported_version"); }; + return { + bootstrap: (body, _key, signal) => transport.bootstrap(body, signal), + renewSession: unsupported, + async initialize(session, body, _key, signal) { + let request: AttachmentUploadRequest; + try { request = JSON.parse(body); } catch { throw new AttachmentTransportError("unsupported_version"); } + const upload = await transport.initialize(session, request, signal); + if (object(upload)) requests.set(upload, request); + return upload; + }, + // No fresh capability can be minted through the legacy shape: renewal yields `expired` (design section 3). + renewUpload: async () => { throw new AttachmentTransportError("expired"); }, + async putContent(session, upload, source, signal) { + if (!(source instanceof Uint8Array) && !(typeof Blob !== "undefined" && source instanceof Blob)) { + // Ranged sources (file leases) are never materialized for a transport that cannot stream them. + if ((source as { size: number }).size > 8 * 1024 * 1024) throw new AttachmentTransportError("source_unavailable"); + } + return transport.upload(session, upload, await attachmentContentBytes(source, signal), signal); + }, + async complete(session, upload, _key, signal) { + const request = requests.get(upload); + if (!request) throw new AttachmentTransportError("unsupported_version"); + await transport.complete(session, upload, request, signal); + return "validating"; + }, + abort: async () => undefined, + uploadStatus: unsupported, + putManifest: unsupported, + submitImport: unsupported, + importStatus: unsupported, + }; +} +const LEGACY_WIRES = new WeakSet(); +/** @internal True for the adapter around a four-method transport, which cannot renew, manifest or import. */ +export function attachmentIsLegacyWire(wire: AttachmentTransportV1 | undefined): boolean { return !!wire && LEGACY_WIRES.has(wire); } +/** Pick the versioned surface of whatever transport shape the caller provided. A four-method shape warns once (loudly). */ +export function attachmentWireTransport(transport: AttachmentTransport | AttachmentTransportV1 | undefined): AttachmentTransportV1 | undefined { + if (!transport) return undefined; + const carried = (transport as { [ATTACHMENT_TRANSPORT_V1]?: AttachmentTransportV1 })[ATTACHMENT_TRANSPORT_V1]; + if (carried && typeof carried.putContent === "function") return carried; + if (typeof (transport as AttachmentTransportV1).putContent === "function") return transport as AttachmentTransportV1; + attachmentWarnLegacyTransport(); + const legacy = attachmentLegacyTransport(transport as AttachmentTransport); + LEGACY_WIRES.add(legacy); + return legacy; +} diff --git a/typescript/packages/fi-core/src/attachments/types.ts b/typescript/packages/fi-core/src/attachments/types.ts new file mode 100644 index 000000000..97e07dd0d --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/types.ts @@ -0,0 +1,346 @@ +import type { Span, SpanContext } from "@opentelemetry/api"; +import type { TraceConfigOptions } from "../trace/trace-config/types"; +import type { AttachmentCorrelation } from "./correlation"; + +export type AttachmentDirection = "input" | "output" | "mixed"; +export type AttachmentModality = "audio" | "image" | "video" | "document" | "file"; +export type AttachmentPurpose = "conversation_recording" | "turn_recording" | "model_input" | "model_output" | "tool_result" | "computer_screenshot" | "diagnostic" | "document"; +export type AttachmentSourceKind = "Bytes" | "Blob" | "RemoteUrl" | "LocalFile" | "OwnedRecordingFile" | "ProviderFile"; +/** No source is inferred from a string. Unsupported retrieval sources fail closed. */ +export type AttachmentSource = + | { kind: "Bytes"; data: Uint8Array } + | { kind: "Blob"; data: Blob } + | { kind: "RemoteUrl"; url: string } + | { kind: "LocalFile"; path: string } + | { kind: "OwnedRecordingFile"; lease: AttachmentOwnedFileRef } + | { kind: "ProviderFile"; provider: string; fileId: string }; +export type AttachmentAdmission = "queued" | "disabled" | "bootstrap_pending" | "policy_denied" | "credential_scope_unsupported" | "source_unavailable" | "unsampled" | "too_large" | "queue_full" | "unsupported_version" | "endpoint_unconfigured" | "scope_unresolved" | "correlation_unavailable" | "redaction_unavailable" | "recording_disabled" | "expired" | "deleted" | "incomplete"; +/** Terminal transfer outcomes (design 0.3). `validating` means transferred, NOT readable. `importing` is an accepted import job. */ +export type AttachmentCompletionStatus = "validating" | "importing" | "incomplete" | "expired" | "deleted" | "policy_denied" | "too_large" | "throttled" | + "conflict" | "not_found" | "credential_scope_unsupported" | "validation_failed" | "unsupported_version" | "source_unavailable" | "redaction_unavailable"; +/** + * `failed` is retired (design 0.3) and is no longer a member, so a leftover `status === "failed"` check is a compile error + * (TS2367) instead of a branch that silently never runs. Failures are `incomplete` or `unsupported_version`. + */ +export type AttachmentCompletion = Readonly<{ status: AttachmentAdmission | AttachmentCompletionStatus }>; +export interface AttachmentHandle { + readonly attachmentId: string; + readonly occurrenceId: string; + readonly admission: AttachmentAdmission; + /** Never rejects; validating is transferred, NOT readable. Use flush/shutdown for process-exit grace. */ + readonly completion: Promise; +} +/** Semantic ancestry can only add restrictions, never remove a directional restriction. */ +export interface AttachmentCapturePolicy { + captureContent?: boolean; + recordInputs?: boolean; + recordOutputs?: boolean; + muted?: boolean; + temporaryChat?: boolean; + requested?: boolean; + modelInput?: boolean; + containsInput?: boolean; + containsOutput?: boolean; + errorDetail?: boolean; + newRecording?: boolean; +} +export interface AttachmentOptions { + mimeType: string; + direction: AttachmentDirection; + purpose?: AttachmentPurpose; + conversationId?: string; + turnId?: string; + toolCallId?: string; + messageIndex?: number; + partIndex?: number; + role?: "user" | "assistant" | "tool" | "system" | "unknown"; + channel?: number; + capture?: AttachmentCapturePolicy; +} +export interface AttachmentConfig { + enabled?: boolean; + recordAudio?: boolean; + recordVideo?: boolean; + allowUnredactedMediaWithTextPii?: boolean; + /** Trusted control-plane origin, never inferred from the OTLP or source URL. */ + endpoint?: string; + maxDescriptors?: number; + maxOwnedBytes?: number; + workers?: number; + requestTimeoutMs?: number; + shutdownTimeoutMs?: number; + /** One content PUT (default 120 s), capped by the negotiated transfer_timeout_seconds. */ + contentTimeoutMs?: number; + /** One manifest PUT (default 10 s, max 30 s). */ + manifestTimeoutMs?: number; + /** Largest mutable buffer copied synchronously inside a caller's call (default 8 MiB, max 32 MiB). */ + syncCopyBytes?: number; + /** Unused preflight ticket lifetime (default 10 s, max 30 s). */ + ticketTtlMs?: number; + /** Consumed-stream idle lifetime (default 60 s, max 300 s). */ + streamIdleTtlMs?: number; + /** Session-end grace for adapter finalization (default and max 30 s). */ + finalizationTimeoutMs?: number; +} +export interface AttachmentReference { + attachment_id: string; + occurrence_id: string; + direction: AttachmentDirection; + modality: AttachmentModality; + mime_type: string; + purpose?: AttachmentPurpose; + message_index?: number; + part_index?: number; + role?: AttachmentOptions["role"]; + channel?: number; + conversation_id?: string; + turn_id?: string; + tool_call_id?: string; +} +export interface AttachmentServerPolicy { + version: string; + enabled: boolean; + allowed_sources: AttachmentSourceKind[]; + allowed_modalities: AttachmentModality[]; + require_media_redaction: boolean; + require_metadata_removal: boolean; + allow_unredacted_media_with_text_pii: boolean; + record_audio: boolean; + record_video: boolean; +} +export interface AttachmentSession { + schema_version: 1; + scope: { organization_id: string; workspace_id: string; project_id: string }; + capture_session_id: string; + policy: AttachmentServerPolicy; + limits: { max_upload_bytes: number; [name: string]: number }; + expires_at: string; + transfer_modes: ["authenticated_stream"]; +} +export interface AttachmentBootstrap { + schema_versions: [1]; + project_name: string; + project_type: string; + workspace_id?: string; + capture_session_id: string; + correlation_nonce: string; +} +export interface AttachmentUploadRequest { + schema_version: 1; + attachment_id: string; + occurrence_id: string; + association: { capture_session_id: string; correlation_nonce: string; parent: { trace_id: string; span_id: string } }; + policy_version: string; + deletion_generation: number; + mime_type: string; + modality: AttachmentModality; + direction: AttachmentDirection; + source_kind: AttachmentSourceKind; + declared_bytes: number; + purpose?: AttachmentPurpose; + message_index?: number; + part_index?: number; + recording_id?: string; + segment_index?: number; + conversation_id?: string; + turn_id?: string; + tool_call_id?: string; +} +export interface AttachmentUpload { + upload_id: string; + attachment_id: string; + occurrence_id: string; + state: "pending_upload"; + reserved_bytes: number; + expires_at: string; + transfer_mode: "authenticated_stream"; + write_capability: string; +} +/** Transport implementations must abort promptly, bound responses and suppress instrumentation. */ +export interface AttachmentTransport { + bootstrap(request: AttachmentBootstrap, signal: AbortSignal): Promise; + initialize(session: AttachmentSession, request: AttachmentUploadRequest, signal: AbortSignal): Promise; + upload(session: AttachmentSession, upload: AttachmentUpload, data: Uint8Array | Blob, signal: AbortSignal): Promise; + complete(session: AttachmentSession, upload: AttachmentUpload, request: AttachmentUploadRequest, signal: AbortSignal): Promise; +} +export interface AttachmentManagerOptions extends AttachmentConfig { + projectName?: string; + projectType?: string; + workspaceId?: string; + traceConfig?: TraceConfigOptions; + /** Node uses process.env via the provider; browsers do not inspect any ambient credentials. */ + environment?: () => Readonly>; + /** The legacy four-method shape or the full versioned wire surface (design section 9). */ + transport?: AttachmentTransport | AttachmentTransportV1; + /** @internal Test clock. `now` is monotonic milliseconds; `wall` is epoch milliseconds. */ + clock?: AttachmentClock; +} +export interface AttachmentFlushResult { readonly unfinished: number; readonly dropped: number } +export type AttachmentSpan = Span | SpanContext; + +// --------------------------------------------------------------------------- +// Design sections 0-9 (sdk-streaming-design.md). Additive; earlier names are unchanged. +// --------------------------------------------------------------------------- +export interface AttachmentClock { now(): number; wall(): number } +/** + * Runtime state (design 0.2). `deleted` and `policy_denied` remain only for an explicit `invalidate()`. + * The old ready value `"queued"` is retired for `"ready"`; it is deliberately absent, so `state !== "queued"` is a compile error (TS2367). + */ +export type AttachmentRuntimeState = "disabled" | "endpoint_unconfigured" | "credential_scope_unsupported" | "scope_unresolved" | "bootstrap_pending" | + "ready" | "expired" | "unsupported_version" | "incomplete" | "deleted" | "policy_denied" | "source_unavailable"; +export interface AttachmentStatus { + readonly state: AttachmentRuntimeState; + readonly queued: number; readonly active: number; + readonly openTickets: number; readonly openStreams: number; readonly openRecordings: number; + readonly ownedBytes: number; readonly dropped: number; +} +export type AttachmentOperation = "observe_consumed" | "import_existing" | "create_recording"; +export type AttachmentRole = NonNullable; +export type AttachmentTarget = AttachmentSpan | AttachmentCorrelation | undefined; +export type AttachmentGapReason = "queue_full" | "bootstrap_pending" | "source_unavailable" | "policy_denied" | "truncated" | "incomplete"; +export interface AttachmentPreflightOptions { + mimeType: string; + direction: AttachmentDirection; + /** Upper bound on the bytes the ticket may later carry; > 0. */ + maxBytes: number; + operation?: AttachmentOperation; + sourceKind?: AttachmentSourceKind; + purpose?: AttachmentPurpose; + capture?: AttachmentCapturePolicy; + ttlMs?: number; +} +/** `manager.deferredCorrelation(options)`: how long pre-bind captures may wait for their span. */ +export interface AttachmentDeferredCorrelationOptions { + /** Default 60 000 ms, max 300 000 ms. Enforced lazily at manager entry points and at bind; no timer is armed. */ + bindTimeoutMs?: number; +} +export interface AttachmentFields { + messageIndex?: number; partIndex?: number; role?: AttachmentRole; channel?: number; + conversationId?: string; turnId?: string; toolCallId?: string; +} +export interface AttachmentStreamOptions extends AttachmentFields { + mimeType: string; direction: AttachmentDirection; maxBytes: number; + purpose?: AttachmentPurpose; capture?: AttachmentCapturePolicy; idleTtlMs?: number; +} +export interface AttachmentStream { + readonly reason: AttachmentAdmission; readonly attachmentId?: string; readonly occurrenceId?: string; + /** Synchronous, never throws, does no I/O; copies the chunk. */ + tryWrite(chunk: Uint8Array): boolean; + /** Non-blocking: enqueues the single upload. */ + finish(): AttachmentHandle; + abort(reason?: "incomplete" | "policy_denied"): void; +} +export interface AttachmentAdmissionTicket { + readonly admitted: boolean; + readonly reason: AttachmentAdmission; + readonly maxBytes: number; + readonly correlation?: AttachmentCorrelation; + attach(source: Extract, fields?: AttachmentFields): AttachmentHandle; + startStream(options: AttachmentStreamOptions): AttachmentStream; + release(): void; +} +export interface AttachmentPcmFormat { sampleRateHz: number; sampleFormat?: "s16le" } +export interface AttachmentTrackSpec { channels: readonly number[]; direction: AttachmentDirection; role: "user" | "assistant" | "mixed" | "unknown" } +export interface AttachmentRecordingOptions { + format: AttachmentPcmFormat; + layout: "mixed" | "stereo" | "separate_tracks"; + tracks: readonly AttachmentTrackSpec[]; + operation: "observe_consumed" | "create_recording"; + purpose?: "conversation_recording" | "turn_recording"; + capture?: AttachmentCapturePolicy; + conversationId?: string; turnId?: string; + windowSegments?: number; +} +export interface AttachmentRecordingStatus { + readonly state: "pending" | "recording" | "finishing" | "terminal"; + readonly reason: AttachmentAdmission | AttachmentCompletionStatus; + readonly sealed: number; readonly uploaded: number; readonly gapMs: number; readonly manifestRevision: number; readonly ownedBytes: number; +} +export interface AttachmentRecordingResult { + readonly recordingId: string; + readonly state: "published" | "partial" | "unpublished" | "denied"; + readonly reason?: string; + readonly segmentsPublished: number; + readonly gapMs: number; + readonly captureComplete: boolean; + /** 0 = never published. */ + readonly manifestRevision: number; +} +export interface AttachmentRecordingWriter { + /** Client UUID; never projected on spans. */ + readonly recordingId: string; + readonly reason: AttachmentAdmission; + tryWrite(pcm: Uint8Array, track?: number): boolean; + markGap(durationMs: number, reason?: AttachmentGapReason, track?: number): void; + markPlayout(startMs: number, durationMs: number, evidence: "generated" | "played" | "unknown" | "truncated"): void; + /** Bounded; never rejects. */ + finish(timeoutMs?: number): Promise; + abort(): void; + status(): AttachmentRecordingStatus; +} +/** Opaque file reference minted only by a recording file scope from this core copy. */ +declare const attachmentOwnedFileBrand: unique symbol; +export interface AttachmentOwnedFileRef { readonly [attachmentOwnedFileBrand]: true } +export interface AttachmentFileIdentity { dev: bigint; ino: bigint; size: number } +export interface AttachmentRecordingFileScope { + identify(relativePath: string): Promise; + source(relativePath: string, identity: AttachmentFileIdentity): { kind: "OwnedRecordingFile"; lease: AttachmentOwnedFileRef }; + close(): Promise; +} +export interface AttachmentOwnedRecordingOptions { + mimeType: string; direction: AttachmentDirection; wavTracks?: readonly AttachmentTrackSpec[]; + purpose?: "conversation_recording" | "turn_recording"; capture?: AttachmentCapturePolicy; conversationId?: string; +} +export interface AttachmentOwnedRecordingHandle { + /** Provisional for WAV until its header is read after admission. */ + readonly tier: "single" | "wav_recording" | "denied"; + readonly reason: AttachmentAdmission; + readonly recordingId?: string; + readonly completion: Promise; + /** Bounded wait; resolves `incomplete` at the deadline (and cancels). */ + wait(timeoutMs?: number): Promise; + cancel(): void; +} +export interface AttachmentCompletedRecordingRequest { provider: string; providerRef: string; direction?: AttachmentDirection } +export type AttachmentImportSourceRef = { kind: "ProviderFile"; connectionId: string; registeredOutputId: string } | { kind: "RemoteUrlEnvelope"; envelopeId: string }; +export interface AttachmentResolvedCompletedRecording { source: AttachmentImportSourceRef; declaredBytes: number; declaredDurationMs: number; mimeType: string; direction: AttachmentDirection } +export interface AttachmentCompletedRecordingResolver { + resolve(request: AttachmentCompletedRecordingRequest, signal: AbortSignal): Promise; +} +export interface AttachmentImportOptions { resolver: AttachmentCompletedRecordingResolver; capture?: AttachmentCapturePolicy; timeoutMs?: number } +export interface AttachmentImportHandle { + readonly reason: AttachmentAdmission | AttachmentCompletionStatus; + readonly recordingId?: string; + readonly jobId?: string; + readonly completion: Promise; + /** Exactly one status GET per call. */ + status(timeoutMs?: number): Promise<{ state: string; reason: string }>; +} +export interface AttachmentImportJob { jobId: string; recordingId: string; state: "importing" } +/** In-memory bytes, an immutable Blob, or a bounded range reader (owned files, WAV ranges). */ +export type AttachmentContentSource = Uint8Array | Blob | { readonly size: number; read(offset: number, size: number): Promise; check(): Promise }; +export interface AttachmentPacer { take(bytes: number, signal: AbortSignal): Promise } +/** The ONE versioned wire surface (design section 9). Every HTTP shape stays behind this. */ +export interface AttachmentTransportV1 { + bootstrap(body: AttachmentBootstrap, key: string, signal: AbortSignal): Promise; + renewSession(current: AttachmentSession, sequence: number, signal: AbortSignal): Promise; + initialize(session: AttachmentSession, body: string, key: string, signal: AbortSignal): Promise; + renewUpload(session: AttachmentSession, upload: AttachmentUpload, body: string, signal: AbortSignal): Promise; + putContent(session: AttachmentSession, upload: AttachmentUpload, source: AttachmentContentSource, signal: AbortSignal, pacer: AttachmentPacer): Promise; + complete(session: AttachmentSession, upload: AttachmentUpload, key: string, signal: AbortSignal): Promise<"validating">; + abort(session: AttachmentSession, upload: AttachmentUpload, key: string, signal: AbortSignal): Promise; + uploadStatus(session: AttachmentSession, upload: AttachmentUpload, signal: AbortSignal): Promise<{ state: string; reason: string }>; + putManifest(session: AttachmentSession, capability: string, recordingId: string, body: string, key: string, signal: AbortSignal): Promise; + submitImport(session: AttachmentSession, body: string, key: string, signal: AbortSignal): Promise; + importStatus(session: AttachmentSession, jobId: string, signal: AbortSignal): Promise<{ state: string; reason: string }>; +} +export interface AttachmentRecordingTrack { track_id: string; channels: number[]; direction: AttachmentDirection; role: AttachmentTrackSpec["role"] } +export interface AttachmentRecordingManifest { + schema_version: 1; recording_id: string; timebase: "session_relative_ms"; + sample_format: "s16le"; sample_rate_hz: number; channels: number; codec: "pcm"; container: "wav"; + layout: AttachmentRecordingOptions["layout"]; tracks: AttachmentRecordingTrack[]; + segments: { occurrence_id: string; index: number; start_ms: number; duration_ms: number; track_id?: string }[]; + gaps: { start_ms: number; duration_ms: number; reason: AttachmentGapReason; track_id?: string }[]; + playout: { start_ms: number; duration_ms: number; evidence: "generated" | "played" | "unknown" | "truncated" }[]; + capture_complete: boolean; duration_ms: number; +} diff --git a/typescript/packages/fi-core/src/attachments/warn.ts b/typescript/packages/fi-core/src/attachments/warn.ts new file mode 100644 index 000000000..7a483ac1b --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/warn.ts @@ -0,0 +1,40 @@ +/** + * One-time, process-visible diagnostics for integration mistakes that would otherwise only show up as attachments + * quietly not arriving. Browser-safe: no `node:*` import. Messages carry fixed text and shape names only, never values, + * URLs, capabilities or credentials. + */ +export const ATTACHMENT_WARNING_LEGACY_TRANSPORT = "FI_ATTACHMENTS_LEGACY_TRANSPORT"; +export const ATTACHMENT_WARNING_WIRE_REJECTED = "FI_ATTACHMENTS_WIRE_REJECTED"; +const MIGRATION = "See ATTACHMENTS-MIGRATION.md in @traceai/fi-core."; +const seen = new Set(); + +type Emitter = (message: string, options: { type: string; code: string }) => void; +/** Emits once per key per process: `process.emitWarning` on Node (honours --no-deprecation/--no-warnings), else `console.warn`. */ +export function attachmentWarnOnce(key: string, code: string, message: string, deprecation = false): void { + if (seen.has(key)) return; + seen.add(key); + try { + const proc = (globalThis as { process?: { emitWarning?: Emitter } }).process; + if (proc && typeof proc.emitWarning === "function") { proc.emitWarning(`${message} ${MIGRATION}`, { type: deprecation ? "DeprecationWarning" : "Warning", code }); return; } + const out = (globalThis as { console?: { warn?: (...args: unknown[]) => void } }).console; + if (out && typeof out.warn === "function") out.warn(`[${code}] ${message} ${MIGRATION}`); + } catch { /* Diagnostics never break application code (e.g. --throw-deprecation). */ } +} + +/** A four-method `AttachmentTransport` was adapted: say exactly what it cannot do, once. */ +export function attachmentWarnLegacyTransport(): void { + attachmentWarnOnce("legacy-transport", ATTACHMENT_WARNING_LEGACY_TRANSPORT, + "A legacy four-method AttachmentTransport (bootstrap/initialize/upload/complete) was passed to AttachmentManager. " + + "It cannot renew sessions, PUT recording manifests, submit imports or read upload status: attachments stop with `expired` " + + "after the first session lifetime, and startRecording, WAV-tier attachOwnedRecording and importCompletedRecording are denied with " + + "`unsupported_version`. Pass createAttachmentHttpTransportV1(...) or an AttachmentTransportV1; a wrapper must forward the " + + "ATTACHMENT_TRANSPORT_V1 member of the transport it wraps.", true); +} + +/** A response failed its F01 shape check. Fails closed as `unsupported_version` and names the shape once. */ +export function attachmentWireRejected(shape: string, error: E): E { + attachmentWarnOnce(`wire:${shape}`, ATTACHMENT_WARNING_WIRE_REJECTED, + `An attachment API response failed the F01 ${shape} check, so the affected attachments report \`unsupported_version\`. ` + + "Check the server, or the test double, against spec/attachments/v1 (every status body carries `reason`)."); + return error; +} diff --git a/typescript/packages/fi-core/src/attachments/wire.ts b/typescript/packages/fi-core/src/attachments/wire.ts new file mode 100644 index 000000000..50f2ad58d --- /dev/null +++ b/typescript/packages/fi-core/src/attachments/wire.ts @@ -0,0 +1,132 @@ +/** + * Hand validators for every F01 shape the SDK sends or parses (design 9, T-TR-01). They mirror + * contract.schema.json plus the cross-field rules of validate.py, with no schema engine in the runtime. + */ +import { validAttachmentManifest } from "./recording"; +import { attachmentStrictJson } from "./transport-v1"; +import type { AttachmentRecordingManifest } from "./types"; + +type V = (value: unknown) => boolean; +const obj = (v: unknown): v is Record => !!v && typeof v === "object" && !Array.isArray(v); +const UUID: V = v => typeof v === "string" && v.length === 36 && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(v) && v !== "00000000-0000-0000-0000-000000000000"; +const TRACE: V = v => (typeof v === "string" && /^[0-9a-f]{32}$/.test(v) && !/^0{32}$/.test(v)) || UUID(v); +const SPAN: V = v => typeof v === "string" && /^[0-9a-f]{16}$/.test(v) && !/^0{16}$/.test(v); +const TOKEN: V = v => typeof v === "string" && /^[a-zA-Z0-9_-]{1,128}$/.test(v); +const int = (min: number, max: number): V => v => typeof v === "number" && Number.isInteger(v) && v >= min && v <= max; +const SAFE = int(0, Number.MAX_SAFE_INTEGER); +const TIMESTAMP: V = v => typeof v === "string" && v.length <= 40 && + /^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(\.\d+)?([Zz]|[+-]\d{2}:\d{2})$/.test(v) && Number.isFinite(Date.parse(v)); +const oneOf = (...values: unknown[]): V => v => values.includes(v); +const DIRECTION = oneOf("input", "output", "mixed"); +const MODALITY = oneOf("audio", "image", "video", "document", "file"); +const AUDIO = ["audio/wav", "audio/ogg", "audio/mpeg", "audio/mp4", "audio/webm", "audio/flac", "audio/L16"]; +const MIME_BY: Record = { audio: AUDIO, image: ["image/png", "image/jpeg", "image/webp"], video: ["video/mp4", "video/webm"], + document: ["application/pdf"], file: ["application/octet-stream"] }; +const MIME = oneOf(...Object.values(MIME_BY).flat()); +const PURPOSE = oneOf("conversation_recording", "turn_recording", "model_input", "model_output", "tool_result", "computer_screenshot", "diagnostic", "document"); +const SOURCE = oneOf("Bytes", "Blob", "RemoteUrl", "LocalFile", "OwnedRecordingFile", "ProviderFile"); +const STATUS = oneOf("pending_upload", "importing", "validating", "ready", "failed", "rejected", "expired", "deleted"); +const REASON = oneOf("queued", "disabled", "bootstrap_pending", "policy_denied", "credential_scope_unsupported", "source_unavailable", "unsampled", "too_large", + "queue_full", "unsupported_version", "endpoint_unconfigured", "scope_unresolved", "correlation_unavailable", "redaction_unavailable", "recording_disabled", + "binding_pending", "not_found", "conflict", "throttled", "validation_failed", "expired", "deleted", "incomplete", "ready"); +/** Closed object: every required key present, no unknown key, every present key valid. */ +function closed(required: string[], properties: Record): V { + return v => obj(v) && required.every(key => Object.prototype.hasOwnProperty.call(v, key)) && + Object.keys(v).every(key => Object.prototype.hasOwnProperty.call(properties, key) && properties[key](v[key])); +} +const array = (item: V, min: number, max: number, unique = false): V => v => Array.isArray(v) && v.length >= min && v.length <= max && v.every(item) && + (!unique || new Set(v.map(x => JSON.stringify(x))).size === v.length); +const SCOPE = closed(["organization_id", "workspace_id", "project_id"], { organization_id: UUID, workspace_id: UUID, project_id: UUID }); +const PARENT = closed(["trace_id", "span_id"], { trace_id: TRACE, span_id: SPAN }); +const ASSOCIATION = closed(["capture_session_id", "correlation_nonce", "parent"], { capture_session_id: UUID, correlation_nonce: UUID, parent: PARENT }); +const mimeMatches = (v: Record) => (MIME_BY[v.modality as string] ?? []).includes(v.mime_type as string); +const REFERENCE_SHAPE = closed(["attachment_id", "occurrence_id", "direction", "modality", "mime_type"], { attachment_id: UUID, occurrence_id: UUID, + direction: DIRECTION, modality: MODALITY, mime_type: MIME, purpose: PURPOSE, message_index: int(0, 65535), part_index: int(0, 65535), + role: oneOf("user", "assistant", "tool", "system", "unknown"), channel: int(0, 31), conversation_id: TOKEN, turn_id: TOKEN, tool_call_id: TOKEN }); +const REFERENCE: V = v => REFERENCE_SHAPE(v) && mimeMatches(v as Record); +const REFERENCES: V = v => array(REFERENCE, 0, 64, true)(v) && new Set((v as { occurrence_id: string }[]).map(r => r.occurrence_id)).size === (v as unknown[]).length; +const UPLOAD_REQUEST_SHAPE = closed(["schema_version", "attachment_id", "occurrence_id", "association", "policy_version", "deletion_generation", "mime_type", + "modality", "direction", "source_kind", "declared_bytes"], { schema_version: oneOf(1), attachment_id: UUID, occurrence_id: UUID, association: ASSOCIATION, + policy_version: TOKEN, deletion_generation: SAFE, mime_type: MIME, modality: MODALITY, direction: DIRECTION, source_kind: SOURCE, purpose: PURPOSE, + declared_bytes: int(1, 52428800), message_index: int(0, 65535), part_index: int(0, 65535), recording_id: UUID, segment_index: int(0, 65535), + conversation_id: TOKEN, turn_id: TOKEN, tool_call_id: TOKEN }); +const UPLOAD_REQUEST: V = v => { + if (!UPLOAD_REQUEST_SHAPE(v)) return false; + const r = v as Record; + if (("recording_id" in r) !== ("segment_index" in r)) return false; + if (!mimeMatches(r)) return false; + if (r.modality === "image" && (r.declared_bytes as number) > 20971520) return false; + if ("recording_id" in r && (r.declared_bytes as number) > 8388608) return false; + return true; +}; +const POLICY = closed(["version", "enabled", "allowed_sources", "allowed_modalities", "require_media_redaction", "require_metadata_removal", + "allow_unredacted_media_with_text_pii", "record_audio", "record_video"], { version: TOKEN, enabled: v => typeof v === "boolean", + allowed_sources: array(SOURCE, 0, 6, true), allowed_modalities: array(MODALITY, 0, 5, true), require_media_redaction: v => typeof v === "boolean", + require_metadata_removal: v => typeof v === "boolean", allow_unredacted_media_with_text_pii: v => typeof v === "boolean", + record_audio: v => typeof v === "boolean", record_video: v => typeof v === "boolean" }); +const LIMIT_MAX: Record = { max_upload_bytes: 52428800, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, + recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, initializations_per_minute: 120, + links_per_minute: 1000, upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, transfer_timeout_seconds: 600, grant_ttl_seconds: 300, + revocation_check_seconds: 5 }; +const LIMITS = closed(Object.keys(LIMIT_MAX), Object.fromEntries(Object.entries(LIMIT_MAX).map(([k, max]) => [k, int(1, max)]))); +const IMPORT_SOURCE: V = v => closed(["kind", "envelope_id"], { kind: oneOf("RemoteUrlEnvelope"), envelope_id: UUID })(v) || + closed(["kind", "connection_id", "registered_output_id"], { kind: oneOf("ProviderFile"), connection_id: UUID, registered_output_id: UUID })(v); +const COMPLETED = closed(["schema_version", "association", "policy_version", "deletion_generation", "mime_type", "direction", "source_kind", "recording_id", + "modality", "declared_bytes", "declared_duration_ms"], { schema_version: oneOf(1), association: ASSOCIATION, policy_version: TOKEN, deletion_generation: SAFE, + mime_type: oneOf(...AUDIO), direction: DIRECTION, source_kind: oneOf("ProviderFile", "RemoteUrl"), recording_id: UUID, modality: oneOf("audio"), + declared_bytes: int(1, 1073741824), declared_duration_ms: int(1, 7200000) }); +const RECORDING_IMPORT: V = v => { + if (!closed(["mode", "recording", "source"], { mode: oneOf("completed_recording"), recording: COMPLETED, source: IMPORT_SOURCE })(v)) return false; + const r = v as { recording: { source_kind: string }; source: { kind: string } }; + return r.recording.source_kind === (r.source.kind === "RemoteUrlEnvelope" ? "RemoteUrl" : "ProviderFile"); +}; +const TRACK = closed(["track_id", "channels", "direction", "role"], { track_id: UUID, channels: array(int(0, 31), 1, 32, true), direction: DIRECTION, + role: oneOf("user", "assistant", "mixed", "unknown") }); +const MANIFEST_SHAPE = closed(["schema_version", "recording_id", "timebase", "sample_format", "sample_rate_hz", "channels", "codec", "container", "layout", + "tracks", "segments", "gaps", "playout", "capture_complete", "duration_ms"], { schema_version: oneOf(1), recording_id: UUID, timebase: oneOf("session_relative_ms"), + sample_format: oneOf("s16le", "s24le", "s32le", "f32le", "encoded"), sample_rate_hz: int(8000, 192000), channels: int(1, 32), + codec: oneOf("pcm", "opus", "mp3", "aac", "flac"), container: oneOf("wav", "ogg", "mp3", "mp4", "webm", "flac", "raw"), layout: oneOf("mixed", "stereo", "separate_tracks"), + tracks: array(TRACK, 1, 32), segments: array(closed(["occurrence_id", "index", "start_ms", "duration_ms"], { occurrence_id: UUID, index: int(0, 65535), + start_ms: int(0, 7200000), duration_ms: int(1, 30000), track_id: UUID }), 0, 65536), + gaps: array(closed(["start_ms", "duration_ms", "reason"], { start_ms: int(0, 7200000), duration_ms: int(1, 7200000), + reason: oneOf("queue_full", "bootstrap_pending", "source_unavailable", "policy_denied", "truncated", "incomplete"), track_id: UUID }), 0, 65536), + playout: array(closed(["start_ms", "duration_ms", "evidence"], { start_ms: int(0, 7200000), duration_ms: int(1, 7200000), + evidence: oneOf("generated", "played", "unknown", "truncated") }), 0, 65536), + capture_complete: v => typeof v === "boolean", duration_ms: int(0, 7200000) }); +const SPAN_ATTRIBUTES: V = v => { + if (!closed(["fi.attachment.schema_version", "fi.attachment.refs"], { "fi.attachment.schema_version": oneOf(1), + "fi.attachment.refs": x => typeof x === "string" && x.length <= 16384, "fi.attachment.capture_session_id": UUID, "fi.attachment.correlation_nonce": UUID })(v)) return false; + const a = v as Record; + if (("fi.attachment.capture_session_id" in a) !== ("fi.attachment.correlation_nonce" in a)) return false; + if (new TextEncoder().encode(a["fi.attachment.refs"]).byteLength > 16384) return false; + try { return REFERENCES(attachmentStrictJson(a["fi.attachment.refs"])); } catch { return false; } +}; +const VALIDATORS: Record = { + BootstrapRequest: closed(["schema_versions", "project_name", "project_type", "capture_session_id", "correlation_nonce"], { + schema_versions: array(int(1, 65535), 1, 8, true), project_name: v => typeof v === "string" && v.length >= 1 && v.length <= 255, + project_type: v => typeof v === "string" && v.length >= 1 && v.length <= 64, workspace_id: UUID, capture_session_id: UUID, correlation_nonce: UUID }), + BootstrapResponse: closed(["schema_version", "scope", "capture_session_id", "policy", "limits", "expires_at", "transfer_modes"], { schema_version: oneOf(1), + scope: SCOPE, capture_session_id: UUID, policy: POLICY, limits: LIMITS, expires_at: TIMESTAMP, + transfer_modes: v => Array.isArray(v) && v.length === 1 && v[0] === "authenticated_stream" }), + UploadRequest: UPLOAD_REQUEST, + UploadResponse: closed(["upload_id", "attachment_id", "occurrence_id", "state", "reserved_bytes", "expires_at", "transfer_mode", "write_capability"], { + upload_id: UUID, attachment_id: UUID, occurrence_id: UUID, state: oneOf("pending_upload"), reserved_bytes: int(1, 52428800), expires_at: TIMESTAMP, + transfer_mode: oneOf("authenticated_stream"), write_capability: v => typeof v === "string" && /^[a-zA-Z0-9_.-]{16,4096}$/.test(v) }), + CompleteRequest: closed(["policy_version", "deletion_generation"], { policy_version: TOKEN, deletion_generation: SAFE }), + StatusResponse: closed(["upload_id", "state", "reason"], { upload_id: UUID, state: STATUS, reason: REASON }), + RecordingManifest: v => MANIFEST_SHAPE(v) && validAttachmentManifest(v as AttachmentRecordingManifest), + RecordingImportRequest: RECORDING_IMPORT, + RecordingImportResponse: closed(["mode", "job_id", "recording_id", "state"], { mode: oneOf("completed_recording"), job_id: UUID, recording_id: UUID, state: oneOf("importing") }), + ImportStatusResponse: closed(["job_id", "state", "reason"], { job_id: UUID, state: STATUS, reason: REASON }), + Error: closed(["code", "request_id"], { code: REASON, request_id: UUID }), + SpanAttributes: SPAN_ATTRIBUTES, + References: REFERENCES, + Reference: REFERENCE, +}; +/** Shapes this SDK sends or parses. */ +export const ATTACHMENT_WIRE_SHAPES = Object.freeze(Object.keys(VALIDATORS)); +export function validAttachmentWire(name: string, value: unknown): boolean { + const validate = VALIDATORS[name]; + if (!validate) throw new TypeError("unsupported wire shape"); + try { return validate(value); } catch { return false; } +} diff --git a/typescript/packages/fi-core/src/index.ts b/typescript/packages/fi-core/src/index.ts index 24e6448d1..c1e0f021f 100644 --- a/typescript/packages/fi-core/src/index.ts +++ b/typescript/packages/fi-core/src/index.ts @@ -14,3 +14,5 @@ export { projectAttributes, tracerAttributeLimit, DROPPED_CONTENT_COUNT } from " export * from "./awsLambda"; export { rememberProviderResponse } from "./privacy-runtime"; + +export * from "./attachments"; diff --git a/typescript/packages/fi-core/src/otel.ts b/typescript/packages/fi-core/src/otel.ts index 4ced29544..a8cf4ca57 100644 --- a/typescript/packages/fi-core/src/otel.ts +++ b/typescript/packages/fi-core/src/otel.ts @@ -32,6 +32,9 @@ import { seedSecrets } from "./secrets"; import { DefaultBatchSpanProcessor, DefaultSimpleSpanProcessor, SpanProcessorRegistry } from "./spanProcessorRegistry"; import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; +import { createNodeAttachmentManager, NodeAttachmentManagerOptions } from "./attachments/node"; +import { AttachmentManager } from "./attachments/manager"; + const privacyDestinations = new WeakSet(); /** @internal True only for constructors in this core copy, never based on user properties. */ export function hasExportPrivacyBoundary(exporter: SpanExporter): boolean { return privacyDestinations.has(exporter); } @@ -456,6 +459,8 @@ export function createSpanExporter(options: ExportPrivacyOptions & { // --- TracerProvider --- export interface FITracerProviderOptions extends ExportPrivacyOptions { + /** Opt-in out-of-band media, separate from the trace collector destination. */ + attachments?: NodeAttachmentManagerOptions; /** Default false selects SimpleSpanProcessor; true selects BatchSpanProcessor. */ batch?: boolean; resource?: Resource; @@ -469,6 +474,8 @@ export interface FITracerProviderOptions extends ExportPrivacyOptions { } class FITracerProvider extends BasicTracerProvider { + /** The single attachment runtime shared by explicit calls and integration adapters. */ + readonly attachments: AttachmentManager; private readonly processorRegistry: SpanProcessorRegistry; private shutdownPromise?: Promise; private defaultProcessorAttached: boolean = false; @@ -521,6 +528,17 @@ class FITracerProvider extends BasicTracerProvider { this.endpoint = endpoint; this.headers = headers; this.transport = transport; + const attributes = config.resource?.attributes ?? {}; + this.attachments = createNodeAttachmentManager({ + ...config.attachments, + headers: config.attachments?.headers ?? headers, + projectName: config.attachments?.projectName ?? String(attributes[PROJECT_NAME] ?? ""), + projectType: config.attachments?.projectType ?? String(attributes[PROJECT_TYPE] ?? ""), + traceConfig: { ...config.attachments?.traceConfig, ...config.traceConfig, + ...Object.fromEntries(Object.entries(config.attachments?.traceConfig ?? {}).filter(([key, value]) => + (key.startsWith("hide") || key === "piiRedaction") && value === true)), + }, + }); // Log to confirm processor and exporter details if (verbose) { @@ -575,12 +593,15 @@ class FITracerProvider extends BasicTracerProvider { detailsMsg += "| Using a default SpanProcessor. `addSpanProcessor` will overwrite this default.\n"; } } + async forceFlush(): Promise { + await Promise.all([super.forceFlush(), this.attachments.flush()]); + } shutdown(): Promise { if (!this.shutdownPromise) { if (this.verbose) { diag.info("Shutting down FI TracerProvider..."); } - this.shutdownPromise = super.shutdown(); + this.shutdownPromise = Promise.all([super.shutdown(), this.attachments.shutdown()]).then(() => undefined); } return this.shutdownPromise; } @@ -611,6 +632,7 @@ class BatchSpanProcessor extends OTelBatchSpanProcessor { } export interface RegisterOptions extends ExportPrivacyOptions { + attachments?: NodeAttachmentManagerOptions; /** Preserve application service identity; FI project routing is separate. */ resource?: Resource; projectName?: string; @@ -738,6 +760,7 @@ if (!projectName) { batch: options.batch ?? false, spanProcessors: options.spanProcessors, traceConfig: options.traceConfig, + attachments: options.attachments, resourceAttributes: options.resourceAttributes, }); diff --git a/typescript/packages/fi-core/test/attachments-browser.cjs b/typescript/packages/fi-core/test/attachments-browser.cjs new file mode 100644 index 000000000..c7e5a2bfe --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-browser.cjs @@ -0,0 +1,56 @@ +// Run against an esbuild browser bundle with Playwright installed in the qualification tools directory. +const assert = require('node:assert/strict'); +const { chromium } = require(process.argv[2]); +const bundle = process.argv[3]; +const id = '10000000-0000-4000-8000-000000000001'; +const expires = new Date(Date.now() + 3600000).toISOString(); +const policy = { version: 'v1', enabled: true, allowed_sources: ['Bytes', 'Blob'], allowed_modalities: ['audio', 'image'], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }; +const limits = { max_upload_bytes: 52428800, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, + recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, + initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, + transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }; +(async () => { + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage(); const calls = []; let content; + await page.route('https://attachments.example/**', async route => { + const req = route.request(); const url = req.url(); calls.push(url); + const data = req.postDataBuffer(); let response; let status = 200; + if (url.endsWith('attachment-sessions')) response = { schema_version: 1, scope: { organization_id: id, workspace_id: id, project_id: id }, + capture_session_id: JSON.parse(data).capture_session_id, policy, limits, expires_at: expires, transfer_modes: ['authenticated_stream'] }; + else if (url.endsWith('/uploads')) { const r = JSON.parse(data); status = 201; response = { upload_id: id, attachment_id: r.attachment_id, + occurrence_id: r.occurrence_id, state: 'pending_upload', reserved_bytes: r.declared_bytes, expires_at: expires, transfer_mode: 'authenticated_stream', write_capability: 'synthetic-write-capability' }; } + else if (url.endsWith('/content')) { content = [...data]; response = { upload_id: id, state: 'pending_upload', reason: 'queued' }; } + else { status = 202; response = { upload_id: id, state: 'validating', reason: 'binding_pending' }; } + await route.fulfill({ status, headers: { 'content-type': 'application/json', 'access-control-allow-origin': '*' }, body: JSON.stringify(response) }); + }); + await page.addScriptTag({ path: bundle }); + const result = await page.evaluate(async () => { + const { AttachmentManager, createAttachmentHttpTransport } = Attachments; + const identity = { traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }; + const transport = createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic-write-capability' } }); + const manager = new AttachmentManager({ enabled: true, projectName: 'browser-fixture', projectType: 'observe', transport }); + if (!await manager.ready()) throw Error('bootstrap failed'); + const result = manager.attach(identity, { kind: 'Blob', data: new Blob([new Uint8Array([1, 2, 3])]) }, { direction: 'input', mimeType: 'image/png' }); + const completion = await result.completion; await manager.shutdown(); return completion; + }); + assert.deepEqual(result, { status: 'validating' }); assert.deepEqual(content, [1, 2, 3]); assert.equal(calls.length, 4); + const denied = await page.evaluate(async () => { + const { AttachmentManager, createAttachmentHttpTransport } = Attachments; + const transport = createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic-write-capability' } }); + const identity = { traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }; + const states = []; + for (const options of [{}, { enabled: true, traceConfig: { piiRedaction: true } }, { enabled: true, traceConfig: { hideInputImages: true } }]) { + const manager = new AttachmentManager({ projectName: 'browser-fixture', projectType: 'observe', transport, ...options }); + await manager.ready(); let touched = false; + const h = manager.attach(identity, { kind: 'Blob', get data() { touched = true; throw Error('denied-source'); } }, { direction: 'input', mimeType: 'image/png' }); + states.push([h.admission, touched]); await manager.shutdown(); + } + return states; + }); + assert.deepEqual(denied, [['disabled', false], ['redaction_unavailable', false], ['policy_denied', false]]); + assert.equal(calls.length, 6); // two bootstrap controls; ZERO denied transfers. + console.log(JSON.stringify({ browser: browser.version(), passed: 4, transferRequests: 3, deniedTransferRequests: 0 })); + } finally { await browser.close(); } +})().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/typescript/packages/fi-core/test/attachments-compat.test.cjs b/typescript/packages/fi-core/test/attachments-compat.test.cjs new file mode 100644 index 000000000..840c6be76 --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-compat.test.cjs @@ -0,0 +1,151 @@ +/* + * Stacked-lane compatibility: API changes that used to break callers SILENTLY (capture just stopped) must now be loud: + * a compile error, a one-time process warning with a stable code, or an explicit reason at the call. Synthetic data only. + * See ATTACHMENTS-MIGRATION.md. + */ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { core, clock, mono, pcm } = require('./attachments-v1.cjs'); +const { identity, ids, policy, limits, fake } = require('./attachments-fixtures.cjs'); +const { AttachmentManager } = core; +const tick = () => new Promise(resolve => setImmediate(resolve)); +const helpers = { v1: require.resolve('./attachments-v1.cjs'), fixtures: require.resolve('./attachments-fixtures.cjs') }; + +/** Runs `body` in a fresh Node process (warnings are once per process) and returns its stderr and last stdout line. */ +function isolated(body) { + const script = ` + const { core, managerV1, clock, mono, pcm } = require(${JSON.stringify(helpers.v1)}); + const { identity, ids, policy, limits, fake } = require(${JSON.stringify(helpers.fixtures)}); + const out = {}; const warnings = []; + process.on('warning', w => warnings.push({ name: w.name, code: w.code })); + (async () => { ${body} + await new Promise(r => setImmediate(r)); + console.log(JSON.stringify({ out, warnings })); + })().catch(e => { console.error(e && e.stack); process.exit(3); });`; + const r = spawnSync(process.execPath, ['-e', script], { env: process.env, encoding: 'utf8', timeout: 20000 }); + assert.equal(r.status, 0, r.stderr); + return { stderr: r.stderr, ...JSON.parse(r.stdout.trim().split('\n').pop()) }; +} +const legacySession = sessionPolicy => ({ + async bootstrap(r) { + return { schema_version: 1, scope: { organization_id: ids[0], workspace_id: ids[1], project_id: ids[2] }, capture_session_id: r.capture_session_id, + policy: { ...policy, ...sessionPolicy }, limits, expires_at: new Date(Date.now() + 3600000).toISOString(), transfer_modes: ['authenticated_stream'] }; + }, +}); + +test('C-01 a legacy four-method transport warns ONCE per process (DeprecationWarning FI_ATTACHMENTS_LEGACY_TRANSPORT) and names the fix', () => { + const r = isolated(` + for (let i = 0; i < 3; i++) { + const m = new core.AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: fake().transport }); + out['ready' + i] = await m.ready(); await m.shutdown(100); + }`); + assert.deepEqual(r.warnings, [{ name: 'DeprecationWarning', code: 'FI_ATTACHMENTS_LEGACY_TRANSPORT' }]); + assert.match(r.stderr, /FI_ATTACHMENTS_LEGACY_TRANSPORT/); + assert.match(r.stderr, /createAttachmentHttpTransportV1/); assert.match(r.stderr, /ATTACHMENT_TRANSPORT_V1/); assert.match(r.stderr, /ATTACHMENTS-MIGRATION\.md/); + assert.deepEqual(r.out, { ready0: true, ready1: true, ready2: true }, 'the legacy shape still works for single uploads'); +}); + +test('C-02 no warning for an AttachmentTransportV1, for createAttachmentHttpTransport (carries V1), or when attachments are OFF', () => { + const r = isolated(` + const http = core.createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: {}, client: async () => ({ status: 503, body: new Uint8Array() }) }); + const f = await managerV1(); await f.manager.shutdown(100); + const h = new core.AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: http }); await h.shutdown(100); + const off = new core.AttachmentManager({ transport: fake().transport }); await off.shutdown(100);`); + assert.deepEqual(r.warnings, []); + assert.doesNotMatch(r.stderr, /FI_ATTACHMENTS_/); +}); + +test('C-03 a wrapper that drops ATTACHMENT_TRANSPORT_V1 is detected as legacy (the coding-lane guard shape) and warns', () => { + const r = isolated(` + const http = core.createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: {}, client: async () => ({ status: 503, body: new Uint8Array() }) }); + const guarded = { bootstrap: (...a) => http.bootstrap(...a), initialize: (...a) => http.initialize(...a), upload: (...a) => http.upload(...a), complete: (...a) => http.complete(...a) }; + const m = new core.AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: guarded }); await m.shutdown(100); + const forwarded = { ...guarded, [core.ATTACHMENT_TRANSPORT_V1]: http[core.ATTACHMENT_TRANSPORT_V1] }; + out.forwardedIsV1 = typeof forwarded[core.ATTACHMENT_TRANSPORT_V1].putContent === 'function';`); + assert.deepEqual(r.warnings, [{ name: 'DeprecationWarning', code: 'FI_ATTACHMENTS_LEGACY_TRANSPORT' }]); + assert.equal(r.out.forwardedIsV1, true); +}); + +test('C-04 legacy wire: recording, WAV import and completed-recording import are denied up front with unsupported_version; single uploads still work', async () => { + const f = fake(legacySession({ allowed_sources: ['Bytes', 'Blob', 'OwnedRecordingFile', 'ProviderFile', 'RemoteUrl'], record_audio: true })); + const m = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: f.transport, recordAudio: true }); + assert.equal(await m.ready(), true); + const w = m.startRecording(identity, mono(8000)); + assert.equal(w.reason, 'unsupported_version'); assert.equal(w.tryWrite(pcm(160)), false); + assert.equal((await w.finish(100)).state, 'denied'); + let resolverCalls = 0; + const imp = await m.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_synthetic' }, { resolver: { async resolve() { resolverCalls++; return 'source_unavailable'; } } }); + assert.equal(imp.reason, 'unsupported_version'); assert.equal(resolverCalls, 0, 'the resolver is never called'); + assert.equal((await imp.completion).status, 'unsupported_version'); + assert.equal((await m.attach(identity, { kind: 'Bytes', data: new Uint8Array([1, 2]) }, { mimeType: 'image/png', direction: 'input' }).completion).status, 'validating'); + assert.deepEqual(f.calls, ['initialize', 'upload', 'complete'], 'only the single upload reached the transport (bootstrap is the override)'); + await m.flush(1000); // the finished job releases its bytes just after its completion settles + assert.equal(m.status().ownedBytes, 0); assert.equal(m.status().openRecordings, 0); + await m.shutdown(100); +}); + +test('C-05 deny-wins: on a legacy wire a policy denial still beats unsupported_version', async () => { + const f = fake(legacySession({ record_audio: false })); + const m = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: f.transport, traceConfig: { hideInputs: true } }); + assert.equal(await m.ready(), true); + assert.equal(m.startRecording(identity, mono(8000)).reason, 'policy_denied'); + assert.ok(['policy_denied', 'recording_disabled'].includes(m.startRecording(identity, { ...mono(8000), operation: 'create_recording' }).reason)); + await m.shutdown(100); + const off = new AttachmentManager({ transport: fake().transport }); + assert.equal(off.startRecording(identity, mono(8000)).reason, 'disabled'); +}); + +test('C-06 legacy wire: no renewal reaches the transport; the session runs to `expired` and capture says so', async () => { + const c = clock(Date.now()); let boots = 0; + const f = fake({ async bootstrap(r) { boots++; return { ...(await legacySession({}).bootstrap(r)), expires_at: new Date(c.wall() + 120000).toISOString() }; } }); + const m = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: f.transport, clock: c }); + assert.equal(await m.ready(), true); + c.advance(100000); m.status(); await m.flush(100); for (let i = 0; i < 10; i++) await tick(); + assert.equal(m.status().state, 'ready'); assert.equal(boots, 1); + c.advance(30000); + assert.equal(m.status().state, 'expired'); + assert.equal(m.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, { mimeType: 'image/png', direction: 'input' }).admission, 'expired'); + await m.shutdown(100); +}); + +test('C-07 a response failing its F01 shape (status body without `reason`) is unsupported_version AND warns once with the shape name', () => { + const r = isolated(` + let n = 0; + const session = sid => ({ schema_version: 1, scope: { organization_id: ids[0], workspace_id: ids[1], project_id: ids[2] }, capture_session_id: sid, + policy, limits, expires_at: new Date(Date.now() + 3600000).toISOString(), transfer_modes: ['authenticated_stream'] }); + const client = async req => { + const p = new URL(req.url).pathname; const json = (status, body) => ({ status, body: Buffer.from(JSON.stringify(body)) }); + if (p.endsWith('/attachment-sessions')) return json(200, session(JSON.parse(req.body).capture_session_id)); + if (p.endsWith('/uploads')) { const b = JSON.parse(req.body); return json(201, { upload_id: '50000000-0000-4000-8000-00000000000' + (++n), attachment_id: b.attachment_id, + occurrence_id: b.occurrence_id, state: 'pending_upload', reserved_bytes: b.declared_bytes, expires_at: new Date(Date.now() + 600000).toISOString(), + transfer_mode: 'authenticated_stream', write_capability: 'synthetic-capability-' + n + '-abcdef' }); } + if (p.endsWith('/content')) return json(200, { upload_id: p.split('/')[7], state: 'pending_upload' }); // no reason: the pre-F01 double shape + if (p.endsWith('/abort')) return json(200, { upload_id: p.split('/')[7], state: 'failed', reason: 'incomplete' }); + return { status: 404, body: new Uint8Array() }; + }; + const transport = core.createAttachmentHttpTransportV1({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic-original' }, client }); + const m = new core.AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport }); + out.ready = await m.ready(); + const a = m.attach(identity, { kind: 'Bytes', data: new Uint8Array([1, 2]) }, { mimeType: 'image/png', direction: 'input' }); + const b = m.attach(identity, { kind: 'Bytes', data: new Uint8Array([3, 4]) }, { mimeType: 'image/png', direction: 'input' }); + out.a = (await a.completion).status; out.b = (await b.completion).status; + await m.shutdown(100);`); + assert.equal(r.out.ready, true); + assert.equal(r.out.a, 'unsupported_version'); assert.equal(r.out.b, 'unsupported_version'); + assert.deepEqual(r.warnings, [{ name: 'Warning', code: 'FI_ATTACHMENTS_WIRE_REJECTED' }]); + assert.match(r.stderr, /F01 UploadStatus check/); + // Shape names only: no capability, credential, URL or body value reaches the warning. + assert.doesNotMatch(r.stderr, /synthetic-capability|synthetic-original|attachments\.example|50000000-/); +}); + +test('C-08 type-level: retired values are compile errors, not silent never-true checks (tsc --noEmit on test/types)', { timeout: 120000 }, t => { + let tsc; + try { tsc = require.resolve('typescript/bin/tsc', { paths: [path.resolve(__dirname, '..')] }); } + catch { t.skip('typescript is not installed next to this fi-core (packed-install run)'); return; } + const project = path.join(__dirname, 'types', 'tsconfig.json'); + const r = spawnSync(process.execPath, [tsc, '--noEmit', '-p', project], { encoding: 'utf8', timeout: 110000 }); + // Every `@ts-expect-error` in the file must be consumed: an unused one (TS2578) means the retired value compiles again. + assert.equal(r.status, 0, `${r.stdout}\n${r.stderr}`); +}); diff --git a/typescript/packages/fi-core/test/attachments-contract.cjs b/typescript/packages/fi-core/test/attachments-contract.cjs new file mode 100644 index 000000000..9fbee960d --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-contract.cjs @@ -0,0 +1,79 @@ +// Runtime safety replay of every F01 privacy row. Unsupported positive paths +// are reported separately: fail-closed rejection is NOT integrated qualification. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { AttachmentManager, AttachmentCorrelation, AttachmentTransportError, canonicalAttachmentTraceId, attachmentTraceUuid } = require('../dist/src/attachments'); +const { identity, policy, fake, source } = require('./attachments-fixtures.cjs'); +const directory = process.argv[2]; +if (!directory) throw Error('Pass the frozen spec/attachments/v1 directory'); +const privacy = JSON.parse(fs.readFileSync(path.join(directory, 'privacy-fixtures.json'))).cases; +const identities = JSON.parse(fs.readFileSync(path.join(directory, 'identity-fixtures.json'))).cases; +const keepalive = setInterval(() => {}, 1000); +(async () => { + const results = []; const preflightResults = []; + for (const fixture of privacy) { + const c = fixture.context; + const traceConfig = Object.fromEntries(['hide_inputs', 'hide_outputs', 'hide_input_messages', 'hide_output_messages', 'hide_input_images', 'hide_input_text', 'hide_output_text', 'hide_error_details', 'pii_redaction'].map(key => [key.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()), c[key]])); + const mimeType = { audio: 'audio/wav', image: 'image/png', video: 'video/mp4', document: 'application/pdf', file: 'application/octet-stream' }[c.modality]; + const options = { mimeType, direction: c.direction, capture: { captureContent: c.integration_capture_content, recordInputs: c.integration_record_inputs, + recordOutputs: c.integration_record_outputs, muted: c.muted, temporaryChat: c.temporary_chat, requested: c.call_requested, + modelInput: c.model_bound_image, containsInput: c.semantic_input, containsOutput: c.semantic_output, errorDetail: c.error_diagnostic, + newRecording: c.operation === 'create_recording' } }; + const f = fake(); const requests = []; + const transport = { ...f.transport, + async bootstrap(request) { + if (c.policy_known !== true) return new Promise(() => {}); + if (!c.credential_allowed) throw new AttachmentTransportError('credential_scope_unsupported'); + const result = await f.transport.bootstrap(request); + result.policy = { ...policy, enabled: c.server_allowed, + allowed_sources: c.source_authorized ? [c.source_kind] : [], allowed_modalities: c.modality_allowed ? [c.modality] : [], + require_media_redaction: c.server_require_media_redaction, require_metadata_removal: c.require_metadata_removal, + allow_unredacted_media_with_text_pii: c.server_allow_unredacted_media_with_text_pii, record_audio: c.server_record_audio, record_video: c.server_record_video }; + return result; + }, + async initialize(session, request, signal) { requests.push(request.occurrence_id); return f.transport.initialize(session, request, signal); }, + }; + const manager = new AttachmentManager({ enabled: c.enabled, projectName: 'fixture', projectType: 'observe', traceConfig, + recordAudio: c.record_audio, recordVideo: c.record_video, allowUnredactedMediaWithTextPii: c.allow_unredacted_media_with_text_pii, + transport: c.endpoint_configured ? transport : undefined, maxOwnedBytes: c.budget_available ? undefined : 1, + maxDescriptors: c.queue_available ? undefined : 1 }); + await manager.ready(1); + if (!c.generation_current) manager.invalidate('deleted'); + if (!c.queue_available) manager.attach(identity, source(), options); + const actualSource = c.browser_remote_fetch ? { kind: 'RemoteUrl', url: 'https://untrusted.example/source' } : + c.source_kind === 'Bytes' ? { kind: 'Bytes', data: c.source_available ? source().data : undefined } : { kind: c.source_kind }; + const context = !c.correlation_available ? new AttachmentCorrelation() : { ...identity, traceFlags: c.sampled ? 1 : 0 }; + const result = manager.attach(context, actualSource, options); + const unsupported = c.qualified_client_sanitizer || c.qualified_client_metadata_removal || ['RemoteUrl', 'OwnedRecordingFile', 'ProviderFile'].includes(c.source_kind); + if (fixture.expected.decision === 'deny' || unsupported) { + assert.notEqual(result.admission, 'queued', fixture.id); + assert.equal(requests.includes(result.occurrenceId), false, fixture.id); + } else assert.equal(result.admission, 'queued', fixture.id); + // T-PF-03: the same fixture through preflight. A denial never reads the source; a ticket whose denial + // depends on the source itself (availability, budget) is refused at commit without being uploaded. + let preflightRead = false; + const ticket = manager.preflight(context, { mimeType, direction: c.direction, maxBytes: 8, sourceKind: actualSource.kind, + operation: c.operation === 'create_recording' ? 'create_recording' : c.operation === 'import_existing' ? 'import_existing' : 'observe_consumed', capture: options.capture }); + let preflightOutcome = ticket.reason; + if (ticket.admitted && (fixture.expected.decision === 'deny' || unsupported)) { + preflightOutcome = ticket.attach({ kind: actualSource.kind, get data() { preflightRead = true; return actualSource.data; } }).admission; + assert.notEqual(preflightOutcome, 'queued', fixture.id); + } else if (!ticket.admitted) assert.equal(preflightRead, false, fixture.id); + preflightResults.push({ id: fixture.id, expected: fixture.expected.reason, actual: preflightOutcome }); + ticket.release(); + results.push({ id: fixture.id, expected: fixture.expected.reason, actual: result.admission, + qualification: fixture.expected.decision === 'allow' && unsupported ? 'unsupported_fail_closed' : 'safety_pass' }); + await manager.shutdown(10); + } + for (const fixture of identities) { + const hex = canonicalAttachmentTraceId(fixture.input); + assert.equal(hex !== undefined, fixture.valid, JSON.stringify(fixture.input)); + if (fixture.valid) assert.equal(attachmentTraceUuid(hex), fixture.canonical); + } + console.log(JSON.stringify({ privacyFixtures: results.length, identityFixtures: identities.length, + exactReasons: results.filter(result => result.expected === result.actual).length, + preflightFixtures: preflightResults.length, preflightExactReasons: preflightResults.filter(r => r.expected === r.actual).length, + unsupportedPositive: results.filter(result => result.qualification === 'unsupported_fail_closed'), + conservativeReasonDifferences: results.filter(result => result.expected !== result.actual && result.qualification !== 'unsupported_fail_closed') }, null, 2)); +})().catch(error => { console.error(error); process.exitCode = 1; }).finally(() => clearInterval(keepalive)); diff --git a/typescript/packages/fi-core/test/attachments-deferred.test.cjs b/typescript/packages/fi-core/test/attachments-deferred.test.cjs new file mode 100644 index 000000000..8adafc2b6 --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-deferred.test.cjs @@ -0,0 +1,360 @@ +/* + * Deferred-target preflight (late-bound wire spans), the consumed-stream reservation clamp and the shutdown reason. + * Synthetic data and in-memory doubles only (sdk-streaming-design sections 1, 6 and 8; rows T-DT, T-SR, T-SD). + */ +const { test, after } = require('node:test'); +const assert = require('node:assert/strict'); +const { core, managerV1, span, clock } = require('./attachments-v1.cjs'); +const { identity } = require('./attachments-fixtures.cjs'); +const { AttachmentManager, AttachmentDeferredCorrelation, AttachmentCorrelation } = core; +const live = setInterval(() => {}, 1000); // Explicit test owner, not an SDK keepalive. +after(() => clearInterval(live)); +const tick = () => new Promise(resolve => setImmediate(resolve)); +const MiB = 1024 * 1024; +const wav = { mimeType: 'audio/wav', direction: 'input', purpose: 'model_input' }; +const mp3 = { mimeType: 'audio/mpeg', direction: 'output', purpose: 'model_output' }; +const ctx2 = { traceId: 'fedcba9876543210fedcba9876543210', spanId: 'fedcba9876543210', traceFlags: 1 }; +const dashed = hex => `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +/** A Blob whose reads are counted: deferral must never read it before bind. */ +function spyBlob(bytes, type = 'audio/wav') { + const blob = new Blob([bytes], { type }); let reads = 0; + for (const name of ['arrayBuffer', 'stream', 'text', 'bytes']) { + const original = blob[name]; if (typeof original !== 'function') continue; + blob[name] = function (...args) { reads++; return original.apply(this, args); }; + } + const slice = blob.slice.bind(blob); blob.slice = (...args) => { reads++; return slice(...args); }; + return { blob, reads: () => reads }; +} + +// ---- T-DT deferred target --------------------------------------------------------------------------------------- +test('T-DT-01 Blob admitted before the span exists: nothing sent or read until bind; then uploaded with the bound parent and projected', async () => { + const f = await managerV1(); + const target = f.manager.deferredCorrelation(); + assert.ok(target instanceof AttachmentDeferredCorrelation); assert.ok(target instanceof AttachmentCorrelation); assert.equal(target.pending, true); + const { blob, reads } = spyBlob(new Uint8Array([1, 2, 3, 4, 5])); + const ticket = f.manager.preflight(target, { ...wav, maxBytes: blob.size, sourceKind: 'Blob' }); + assert.equal(ticket.reason, 'queued'); + const h = ticket.attach({ kind: 'Blob', data: blob }, { messageIndex: 0, partIndex: 1 }); + assert.equal(h.admission, 'queued'); + await f.manager.flush(50); await tick(); + assert.deepEqual(f.ops().filter(op => op !== 'bootstrap'), [], 'no request before bind'); + assert.equal(reads(), 0, 'the Blob is retained, never read, before bind'); + assert.equal(f.manager.status().ownedBytes, 5); + const s = span(ctx2); + assert.equal(target.bind(s), true); assert.equal(target.pending, false); + assert.equal((await h.completion).status, 'validating'); + assert.deepEqual([...f.contents.get(h.occurrenceId)], [1, 2, 3, 4, 5]); + const request = f.requests.find(r => r.occurrence_id === h.occurrenceId); + assert.equal(request.association.parent.trace_id, dashed(ctx2.traceId)); assert.equal(request.association.parent.span_id, ctx2.spanId); + const refs = JSON.parse(s.attrs['fi.attachment.refs']); + assert.equal(refs.length, 1); assert.equal(refs[0].occurrence_id, h.occurrenceId); assert.equal(refs[0].part_index, 1); + assert.equal(s.attrs['fi.attachment.schema_version'], 1); + assert.equal(f.manager.correlation(s), target, 'the bound target is the span\'s single correlation'); + await f.manager.flush(); assert.equal(f.manager.status().ownedBytes, 0); + await f.manager.shutdown(); +}); +test('T-DT-02 consumed stream finished before bind: bounded chunks held, uploaded at bind byte-identical', async () => { + const f = await managerV1(); + const target = f.manager.deferredCorrelation(); + const stream = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 64 }); + assert.equal(stream.reason, 'queued'); + const buffer = new Uint8Array([7, 8, 9]); assert.equal(stream.tryWrite(buffer), true); buffer.fill(0); assert.equal(stream.tryWrite(new Uint8Array([10])), true); + const h = stream.finish(); assert.equal(h.admission, 'queued'); assert.equal(h.occurrenceId, stream.occurrenceId); + await f.manager.flush(50); await tick(); + assert.equal(f.ops().includes('initialize'), false); + assert.equal(f.manager.status().ownedBytes, 4, 'the reservation shrank to the consumed length and is still held'); + assert.equal((await f.manager.flush(1)).unfinished, 1, 'a parked capture counts as unfinished; flush does not wait for it'); + const s = span(ctx2); target.bind(s); + assert.equal((await h.completion).status, 'validating'); + assert.deepEqual([...f.contents.get(h.occurrenceId)], [7, 8, 9, 10]); + assert.equal(JSON.parse(s.attrs['fi.attachment.refs'])[0].occurrence_id, h.occurrenceId); + await f.manager.shutdown(); +}); +test('T-DT-03 stream still open at bind finishes normally; an over-cap deferred stream uploads and projects nothing', async () => { + const f = await managerV1(); const target = f.manager.deferredCorrelation(); const s = span(ctx2); + const stream = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 4 }); + stream.tryWrite(new Uint8Array([1, 2])); target.bind(s); stream.tryWrite(new Uint8Array([3])); + const h = stream.finish(); assert.equal((await h.completion).status, 'validating'); assert.deepEqual([...f.contents.get(h.occurrenceId)], [1, 2, 3]); + const t2 = f.manager.deferredCorrelation(); const s2 = span(identity); + const over = f.manager.startAttachmentStream(t2, { ...mp3, maxBytes: 4 }); + over.tryWrite(new Uint8Array(3)); assert.equal(over.tryWrite(new Uint8Array(3)), false); assert.equal(over.reason, 'too_large'); + t2.bind(s2); assert.equal((await over.finish().completion).status, 'too_large'); + assert.equal(s2.attrs['fi.attachment.refs'], undefined); await f.manager.flush(); assert.equal(f.manager.status().ownedBytes, 0); + await f.manager.shutdown(); +}); +test('T-DT-04 bind timeout (fake clock): pending captures fail correlation_unavailable at the next entry point; no timer armed; nothing sent on a late bind', async () => { + const c = clock(); const f = await managerV1({}, { clock: c }); + const timers = []; const original = global.setTimeout; + global.setTimeout = (...args) => { timers.push(args[1]); return original(...args); }; + let target, h, stream, ticket; + try { + target = f.manager.deferredCorrelation({ bindTimeoutMs: 1000 }); + h = f.manager.preflight(target, { ...wav, maxBytes: 3 }).attach({ kind: 'Bytes', data: new Uint8Array([1, 2, 3]) }); + stream = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 64, idleTtlMs: 300000 }); stream.tryWrite(new Uint8Array(5)); + ticket = f.manager.preflight(target, { ...wav, maxBytes: 8, ttlMs: 30000 }); + } finally { global.setTimeout = original; } + assert.equal(timers.length, 0, 'deferral armed a timer'); + assert.equal(f.manager.status().ownedBytes, 3 + 64 + 8); + c.advance(1001); + const st = f.manager.status(); + assert.equal(st.ownedBytes, 0); assert.equal(st.openStreams, 0); assert.equal(st.openTickets, 0); + assert.equal((await h.completion).status, 'correlation_unavailable'); + assert.equal(stream.reason, 'correlation_unavailable'); + assert.equal(ticket.attach({ kind: 'Bytes', data: new Uint8Array(2) }).admission, 'correlation_unavailable'); + assert.equal(f.manager.preflight(target, { ...wav, maxBytes: 2 }).reason, 'correlation_unavailable', 'an expired unbound target admits nothing'); + const s = span(ctx2); target.bind(s); + await f.manager.flush(50); + assert.equal(f.ops().includes('initialize'), false); assert.equal(s.attrs['fi.attachment.refs'], undefined); + await f.manager.shutdown(1); +}); +test('T-DT-05 cancel(): pending captures and streams end correlation_unavailable, budget returned, later captures refused, bind refused', async () => { + const f = await managerV1(); const target = f.manager.deferredCorrelation(); + const h = f.manager.attach(target, { kind: 'Bytes', data: new Uint8Array([1]) }, wav); + const stream = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 16 }); stream.tryWrite(new Uint8Array(2)); + const ticket = f.manager.preflight(target, { ...wav, maxBytes: 4 }); + target.cancel(); + assert.equal((await h.completion).status, 'correlation_unavailable'); assert.equal(stream.reason, 'correlation_unavailable'); + assert.equal(f.manager.status().ownedBytes, 0); assert.equal(f.manager.status().openTickets, 0); + let read = false; assert.equal(ticket.attach({ kind: 'Bytes', get data() { read = true; return new Uint8Array(1); } }).admission, 'correlation_unavailable'); assert.equal(read, false); + assert.equal(f.manager.preflight(target, { ...wav, maxBytes: 1 }).reason, 'correlation_unavailable'); + assert.equal(target.bind(span(ctx2)), false); + await f.manager.flush(50); assert.equal(f.ops().includes('initialize'), false); + await f.manager.shutdown(); +}); +test('T-DT-06 manager shutdown before bind: correlation_unavailable, nothing uploaded, a later bind sends nothing', async () => { + const f = await managerV1(); const target = f.manager.deferredCorrelation(); + const h = f.manager.preflight(target, { ...wav, maxBytes: 2 }).attach({ kind: 'Bytes', data: new Uint8Array([1, 2]) }); + const s = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 8 }); s.tryWrite(new Uint8Array(1)); const sh = s.finish(); + const result = await f.manager.shutdown(200); + assert.equal((await h.completion).status, 'correlation_unavailable'); assert.equal((await sh.completion).status, 'correlation_unavailable'); + assert.equal(result.unfinished, 0); assert.equal(f.manager.status().ownedBytes, 0); + target.bind(span(ctx2)); await tick(); + assert.equal(f.ops().includes('initialize'), false); +}); +test('T-DT-07 policy is evaluated again at bind: environment kill switch, tightened trace config, invalidate', async () => { + // (a) A deny that arrives with no policy-epoch event (the environment is re-read): only the bind-time check sees it. + let env = {}; + const a = await managerV1({}, {}, { environment: () => env }); + const ta = a.manager.deferredCorrelation(); + const ha = a.manager.preflight(ta, { ...wav, maxBytes: 1 }).attach({ kind: 'Bytes', data: new Uint8Array([1]) }); + env = { FI_ATTACHMENTS_ENABLED: 'false' }; + const sa = span(ctx2); ta.bind(sa); + assert.equal((await ha.completion).status, 'disabled'); assert.equal(sa.attrs['fi.attachment.refs'], undefined); + await a.manager.flush(50); assert.equal(a.ops().includes('initialize'), false); assert.equal(a.manager.status().ownedBytes, 0); + // (b) Tightened trace config between commit and bind. + const b = await managerV1(); const tb = b.manager.deferredCorrelation(); + const hb = b.manager.startAttachmentStream(tb, { ...wav, maxBytes: 8 }); hb.tryWrite(new Uint8Array(1)); const hbh = hb.finish(); + b.manager.updatePolicy({ traceConfig: { hideInputs: true } }); + assert.equal((await hbh.completion).status, 'policy_denied'); tb.bind(span(ctx2)); await b.manager.flush(50); + assert.equal(b.ops().includes('initialize'), false); + // (c) Revocation. + const c = await managerV1(); const tc = c.manager.deferredCorrelation(); + const hc = c.manager.attach(tc, { kind: 'Bytes', data: new Uint8Array([1]) }, wav); + c.manager.invalidate('deleted'); assert.equal((await hc.completion).status, 'deleted'); + for (const f of [a, b, c]) await f.manager.shutdown(1); +}); +test('T-DT-08 unsampled or non-recording span at bind: unsampled, nothing uploaded', async () => { + const f = await managerV1(); const target = f.manager.deferredCorrelation(); + const h = f.manager.attach(target, { kind: 'Bytes', data: new Uint8Array([1]) }, wav); + target.bind({ ...ctx2, traceFlags: 0 }); + assert.equal((await h.completion).status, 'unsampled'); + const t2 = f.manager.deferredCorrelation(); const h2 = f.manager.attach(t2, { kind: 'Bytes', data: new Uint8Array([1]) }, wav); + const ended = span(ctx2); ended.end(); t2.bind(ended); + assert.equal((await h2.completion).status, 'unsampled'); + await f.manager.flush(50); assert.equal(f.ops().includes('initialize'), false); assert.equal(f.manager.status().ownedBytes, 0); + await f.manager.shutdown(); +}); +test('T-DT-09 pre-bind bytes are capped at half the owned budget; bound captures keep their headroom', async () => { + const f = await managerV1({}, {}, { maxOwnedBytes: 1000 }); + const target = f.manager.deferredCorrelation(); + const held = f.manager.preflight(target, { ...wav, maxBytes: 400 }); assert.equal(held.reason, 'queued'); + assert.equal(f.manager.preflight(target, { ...wav, maxBytes: 101 }).reason, 'queue_full', 'over the pre-bind cap'); + assert.equal(f.manager.startAttachmentStream(f.manager.deferredCorrelation(), { ...mp3, maxBytes: 101 }).reason, 'queue_full'); + assert.equal(f.manager.attach(target, { kind: 'Bytes', data: new Uint8Array(101) }, wav).admission, 'queue_full'); + assert.equal(f.manager.preflight(identity, { ...wav, maxBytes: 500 }).reason, 'queued', 'a bound capture still fits'); + held.release(); + assert.equal(f.manager.preflight(target, { ...wav, maxBytes: 100 }).reason, 'queued'); + await f.manager.shutdown(1); +}); +test('T-DT-10 one reference list per span: a span that already has a correlation keeps it and the target forwards to it', async () => { + const f = await managerV1(); const s = span(ctx2); + const first = f.manager.attach(s, { kind: 'Bytes', data: new Uint8Array([1]) }, { ...wav, partIndex: 0 }); + const target = f.manager.deferredCorrelation(); + const late = f.manager.attach(target, { kind: 'Bytes', data: new Uint8Array([2]) }, { ...wav, partIndex: 1 }); + target.bind(s); + await first.completion; assert.equal((await late.completion).status, 'validating'); + const refs = JSON.parse(s.attrs['fi.attachment.refs']); + assert.deepEqual(refs.map(r => r.occurrence_id).sort(), [first.occurrenceId, late.occurrenceId].sort()); + assert.equal(target.references().length, 2); + const after = f.manager.preflight(target, { ...wav, maxBytes: 1 }); assert.equal(after.reason, 'queued'); + after.attach({ kind: 'Bytes', data: new Uint8Array([3]) }, { partIndex: 2 }); + assert.equal(JSON.parse(s.attrs['fi.attachment.refs']).length, 3); + await f.manager.shutdown(); +}); +test('T-DT-11 scope: another manager\'s target, owned files, recordings and imports are not deferred (correlation_unavailable)', async () => { + const f = await managerV1(); const g = await managerV1(); + const foreign = g.manager.deferredCorrelation(); + assert.equal(f.manager.preflight(foreign, { ...wav, maxBytes: 1 }).reason, 'correlation_unavailable'); + const target = f.manager.deferredCorrelation(); + assert.equal(f.manager.preflight(target, { ...wav, maxBytes: 1, sourceKind: 'OwnedRecordingFile' }).reason, 'correlation_unavailable'); + assert.equal(f.manager.startRecording(target, { format: { sampleRateHz: 8000 }, layout: 'mixed', tracks: [{ channels: [0], direction: 'mixed', role: 'mixed' }], operation: 'observe_consumed' }).reason, 'correlation_unavailable'); + let resolved = false; + const imported = await f.manager.importCompletedRecording(target, { provider: 'livekit', providerRef: 'egress-1' }, { resolver: { resolve: async () => { resolved = true; } } }); + assert.equal(imported.reason, 'correlation_unavailable'); assert.equal(resolved, false); + assert.equal(f.manager.status().ownedBytes, 0); + await f.manager.shutdown(); await g.manager.shutdown(); +}); +test('T-DT-12 denials before admission never touch the source; a plain unbound correlation is still correlation_unavailable', async () => { + const f = await managerV1({}, {}, { traceConfig: { hideInputs: true } }); + const target = f.manager.deferredCorrelation(); let read = false; + const t = f.manager.preflight(target, { ...wav, maxBytes: 4 }); assert.equal(t.reason, 'policy_denied'); + assert.equal(t.attach({ kind: 'Bytes', get data() { read = true; return new Uint8Array(1); } }).admission, 'policy_denied'); assert.equal(read, false); + const g = await managerV1(); + assert.equal(g.manager.preflight(new AttachmentCorrelation(), { ...wav, maxBytes: 1 }).reason, 'correlation_unavailable'); + assert.equal(g.manager.preflight(undefined, { ...wav, maxBytes: 1 }).reason, 'correlation_unavailable'); + await f.manager.shutdown(1); await g.manager.shutdown(1); +}); +test('T-DT-13 disabled: inert deferred target, preflight disabled, bind schedules nothing, no timers/finalizers/listeners', async () => { + const saved = { st: global.setTimeout, fr: global.FinalizationRegistry }; let timers = 0; let registry = 0; + const listenersBefore = process.eventNames().map(n => process.listenerCount(n)).reduce((a, b) => a + b, 0); + try { + global.setTimeout = (...args) => { timers++; return saved.st(...args); }; + global.FinalizationRegistry = class extends saved.fr { constructor(...a) { super(...a); registry++; } }; + for (const m of [new AttachmentManager({ environment: () => ({}) }), new core.DisabledAttachmentManager()]) { + const target = m.deferredCorrelation({ bindTimeoutMs: 10 }); + assert.equal(m.preflight(target, { ...wav, maxBytes: 4 }).reason, 'disabled'); + assert.equal(m.startAttachmentStream(target, { ...mp3, maxBytes: 4 }).reason, 'disabled'); + const s = span(ctx2); assert.equal(target.bind(s), true); assert.equal(s.writes, 0); + target.cancel(); + } + } finally { global.setTimeout = saved.st; global.FinalizationRegistry = saved.fr; } + assert.equal(timers, 0); assert.equal(registry, 0); + assert.equal(process.eventNames().map(n => process.listenerCount(n)).reduce((a, b) => a + b, 0), listenersBefore); +}); +test('T-DT-14 bind is synchronous and never throws; the transport runs only from the queue', async () => { + let sync = true; let calledSync = false; + const f = await managerV1(base => ({ initialize: async (...args) => { if (sync) calledSync = true; return base.initialize(...args); } })); + const target = f.manager.deferredCorrelation(); + const h = f.manager.attach(target, { kind: 'Bytes', data: new Uint8Array([1]) }, wav); + const throwing = { spanContext: () => ctx2, isRecording: () => true, setAttribute() { throw Error('host'); } }; + assert.doesNotThrow(() => target.bind(throwing)); sync = false; + assert.equal(calledSync, false); assert.equal((await h.completion).status, 'validating'); + await f.manager.shutdown(); +}); + +// Sampling at bind covers every capture on the target, not only those parked before bind (design 1.4 step 10). +test('T-DT-15 stream open at bind, bound to an unsampled or non-recording span: unsampled, budget returned, nothing uploaded', async () => { + const f = await managerV1(); + for (const bindTo of [{ ...ctx2, traceFlags: 0 }, (() => { const ended = span(ctx2); ended.end(); return ended; })()]) { + const target = f.manager.deferredCorrelation(); + const stream = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 64 }); assert.equal(stream.tryWrite(new Uint8Array([1, 2, 3])), true); + assert.equal(target.bind(bindTo), true); + assert.equal(stream.reason, 'unsampled', 'the open stream is refused at bind'); assert.equal(f.manager.status().ownedBytes, 0); + assert.equal(stream.tryWrite(new Uint8Array([4])), false); + assert.equal((await stream.finish().completion).status, 'unsampled'); + } + await f.manager.flush(50); assert.equal(f.ops().includes('initialize'), false); assert.equal(f.manager.status().openStreams, 0); + await f.manager.shutdown(); +}); +test('T-DT-16 ticket preflighted before bind, committed after bind to an unsampled span: unsampled, source never read, nothing uploaded', async () => { + const f = await managerV1(); const target = f.manager.deferredCorrelation(); + const ticket = f.manager.preflight(target, { ...wav, maxBytes: 4 }); assert.equal(ticket.reason, 'queued'); + const blobTicket = f.manager.preflight(target, { ...wav, maxBytes: 5, sourceKind: 'Blob' }); assert.equal(blobTicket.reason, 'queued'); + target.bind({ ...ctx2, traceFlags: 0 }); + assert.equal(f.manager.status().openTickets, 0, 'reservations on the target are released at bind'); assert.equal(f.manager.status().ownedBytes, 0); + let read = false; + const h = ticket.attach({ kind: 'Bytes', get data() { read = true; return new Uint8Array([1, 2]); } }); + assert.equal(h.admission, 'unsampled'); assert.equal((await h.completion).status, 'unsampled'); assert.equal(read, false); + const { blob, reads } = spyBlob(new Uint8Array([1, 2, 3, 4, 5])); + assert.equal(blobTicket.attach({ kind: 'Blob', data: blob }).admission, 'unsampled'); assert.equal(reads(), 0); + await f.manager.flush(50); assert.equal(f.ops().includes('initialize'), false); + await f.manager.shutdown(); +}); +test('T-DT-17 late bind with no entry point since the deadline: bind returns false, open streams and tickets end correlation_unavailable', async () => { + const c = clock(); const f = await managerV1({}, { clock: c }); + const target = f.manager.deferredCorrelation({ bindTimeoutMs: 1000 }); + const stream = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 64, idleTtlMs: 300000 }); stream.tryWrite(new Uint8Array([1, 2])); + const ticket = f.manager.preflight(target, { ...wav, maxBytes: 4, ttlMs: 30000 }); + c.advance(5000); // nothing touches the manager between the deadline and the bind + const s = span(ctx2); + assert.equal(target.bind(s), false, 'a late bind binds nothing'); assert.equal(target.snapshot(), undefined); assert.equal(target.pending, false); + assert.equal(stream.reason, 'correlation_unavailable'); assert.equal(stream.tryWrite(new Uint8Array([3])), false); + assert.equal((await stream.finish().completion).status, 'correlation_unavailable'); + let read = false; + assert.equal(ticket.attach({ kind: 'Bytes', get data() { read = true; return new Uint8Array([9]); } }).admission, 'correlation_unavailable'); assert.equal(read, false); + assert.equal(f.manager.status().ownedBytes, 0); assert.equal(s.writes, 0); + await f.manager.flush(50); assert.equal(f.ops().includes('initialize'), false); + // Control: a bind inside the deadline still uploads a ticket committed after it and a stream finished after it. + const t2 = f.manager.deferredCorrelation({ bindTimeoutMs: 1000 }); + const s2 = f.manager.startAttachmentStream(t2, { ...mp3, maxBytes: 8, idleTtlMs: 300000 }); s2.tryWrite(new Uint8Array([5])); + const k2 = f.manager.preflight(t2, { ...wav, maxBytes: 4, ttlMs: 30000 }); + c.advance(999); assert.equal(t2.bind(span(ctx2)), true); + assert.equal((await s2.finish().completion).status, 'validating'); + assert.equal((await k2.attach({ kind: 'Bytes', data: new Uint8Array([6]) }).completion).status, 'validating'); + await f.manager.shutdown(1); +}); +test('T-DT-18 backstops: with the bind notification lost, commit and finish still refuse an unsampled span; deferredBound refuses a late target', async () => { + // (a) The sampled check also runs at ticket commit and stream finish, not only in deferredBound. + const f = await managerV1(); f.manager.deferredBound = () => undefined; + const target = f.manager.deferredCorrelation(); + const stream = f.manager.startAttachmentStream(target, { ...mp3, maxBytes: 8 }); stream.tryWrite(new Uint8Array([1])); + const ticket = f.manager.preflight(target, { ...wav, maxBytes: 4 }); + assert.equal(target.bind({ ...ctx2, traceFlags: 0 }), true); + assert.equal((await stream.finish().completion).status, 'unsampled'); + assert.equal(ticket.attach({ kind: 'Bytes', data: new Uint8Array([1]) }).admission, 'unsampled'); + await f.manager.flush(50); assert.equal(f.ops().includes('initialize'), false); assert.equal(f.manager.status().ownedBytes, 0); + await f.manager.shutdown(1); + // (b) deferredBound fails a late target's open streams and tickets even if the pre-bind deadline check is bypassed. + const c = clock(); const g = await managerV1({}, { clock: c }); g.manager.deferredExpired = () => false; + const late = g.manager.deferredCorrelation({ bindTimeoutMs: 1000 }); + const ls = g.manager.startAttachmentStream(late, { ...mp3, maxBytes: 8, idleTtlMs: 300000 }); ls.tryWrite(new Uint8Array([1])); + const lt = g.manager.preflight(late, { ...wav, maxBytes: 4, ttlMs: 30000 }); + c.advance(5000); late.bind(span(ctx2)); + assert.equal(ls.reason, 'correlation_unavailable'); + assert.equal((await lt.attach({ kind: 'Bytes', data: new Uint8Array([1]) }).completion).status, 'expired', 'its reservation was released at bind'); + await g.manager.flush(50); assert.equal(g.ops().includes('initialize'), false); assert.equal(g.manager.status().ownedBytes, 0); + await g.manager.shutdown(1); +}); + +// ---- T-SR consumed-stream reservation = min(requested, negotiated single-upload cap) --------------------------------- +test('T-SR-01 server max_upload_bytes below 4 MiB: a 4 MiB stream request is admitted at the server cap', async () => { + const f = await managerV1({}, { limits: { max_upload_bytes: 1 * MiB } }); + const stream = f.manager.startAttachmentStream(identity, { ...mp3, maxBytes: 4 * MiB }); + assert.equal(stream.reason, 'queued'); assert.equal(f.manager.status().ownedBytes, 1 * MiB, 'reservation = min(4 MiB, max_upload_bytes)'); + assert.equal(stream.tryWrite(new Uint8Array(512 * 1024)), true); + const h = stream.finish(); assert.equal((await h.completion).status, 'validating'); assert.equal(f.contents.get(h.occurrenceId).length, 512 * 1024); + const over = f.manager.startAttachmentStream(identity, { ...mp3, maxBytes: 4 * MiB }); + assert.equal(over.tryWrite(new Uint8Array(MiB)), true); assert.equal(over.tryWrite(new Uint8Array(1)), false); + assert.equal(over.reason, 'too_large', 'a body over the server cap is discarded whole'); + // The explicit ticket API keeps exact semantics: the caller asked for more than the cap. + assert.equal(f.manager.preflight(identity, { ...mp3, maxBytes: 4 * MiB }).reason, 'too_large'); + // A request below the cap is unchanged. + await f.manager.flush(); + const small = f.manager.startAttachmentStream(identity, { ...mp3, maxBytes: 1000 }); assert.equal(f.manager.status().ownedBytes, 1000); small.abort(); + await f.manager.shutdown(); +}); +test('T-SR-02 the clamp never admits what the lattice denies, and needs a session', async () => { + const pending = await managerV1({ bootstrap: () => new Promise(() => {}) }, { ready: false }); + assert.equal(pending.manager.startAttachmentStream(identity, { ...mp3, maxBytes: 4 * MiB }).reason, 'bootstrap_pending'); + const denied = await managerV1({}, { limits: { max_upload_bytes: MiB } }, { traceConfig: { hideOutputs: true } }); + assert.equal(denied.manager.startAttachmentStream(identity, { ...mp3, maxBytes: 4 * MiB }).reason, 'policy_denied'); + const bad = await managerV1({}, { limits: { max_upload_bytes: MiB } }); + assert.equal(bad.manager.startAttachmentStream(identity, { ...mp3, maxBytes: 0 }).reason, 'too_large'); + assert.equal(bad.manager.startAttachmentStream(identity, { ...mp3, maxBytes: 1.5 }).reason, 'too_large'); + for (const f of [pending, denied, bad]) await f.manager.shutdown(1); +}); + +// ---- T-SD shutdown keeps an earlier invalidate reason --------------------------------------------------------------- +test('T-SD-01 invalidate(policy_denied) then shutdown(): state and admissions stay policy_denied, not incomplete', async () => { + for (const reason of ['policy_denied', 'deleted', 'expired']) { + const f = await managerV1(); + f.manager.invalidate(reason); + assert.equal(f.manager.status().state, reason); + await f.manager.shutdown(100); + assert.equal(f.manager.status().state, reason, `shutdown overwrote ${reason}`); + assert.equal(f.manager.preflight(identity, { ...wav, maxBytes: 1 }).reason, reason); + assert.equal(f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array(1) }, wav).admission, 'incomplete', 'attach after shutdown is incomplete, as before'); + } + const plain = await managerV1(); await plain.manager.shutdown(100); + assert.equal(plain.manager.status().state, 'incomplete', 'a plain shutdown still reports incomplete'); +}); diff --git a/typescript/packages/fi-core/test/attachments-design.test.cjs b/typescript/packages/fi-core/test/attachments-design.test.cjs new file mode 100644 index 000000000..0f2b162cf --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-design.test.cjs @@ -0,0 +1,837 @@ +/* Design qualification (sdk-streaming-design.md section 10). Synthetic data and in-memory doubles only. */ +const { test, after } = require('node:test'); +const assert = require('node:assert/strict'); +const { core, v1, managerV1, span, pcm, writeSeconds, readAll, clock, mono } = require('./attachments-v1.cjs'); +const { identity, ids, policy, limits, source } = require('./attachments-fixtures.cjs'); +const { AttachmentManager, AttachmentTransportError, attachmentWavHeader, attachmentSegmentFrames, attachmentFramesToMs, validAttachmentManifest, + validAttachmentWire, createAttachmentHttpTransport, createAttachmentHttpTransportV1, attachmentHttpError, attachmentIdempotencyKey, AttachmentCorrelation } = core; +const live = setInterval(() => {}, 1000); // Explicit test owner, not an SDK keepalive. +after(() => clearInterval(live)); +const hex = value => value.replace(/-/g, ''); +const tick = () => new Promise(resolve => setImmediate(resolve)); +const options = { mimeType: 'image/png', direction: 'input' }; + +// ---- 10.1 Preflight ----------------------------------------------------------------------------------- +test('T-PF-01 disabled manager: preflight denies without touching any source', async () => { + const m = new AttachmentManager({ transport: v1().transport }); + const t = m.preflight(identity, { mimeType: 'image/png', direction: 'input', maxBytes: 10 }); + assert.equal(t.reason, 'disabled'); assert.equal(t.admitted, false); + let read = false; const h = t.attach({ kind: 'Bytes', get data() { read = true; return new Uint8Array(1); } }); + assert.equal(h.admission, 'disabled'); assert.equal(read, false); +}); +test('T-PF-02 bootstrap pending: denied and nothing reserved', async () => { + const f = await managerV1({ bootstrap: () => new Promise(() => {}) }, { ready: false }); + const t = f.manager.preflight(identity, { mimeType: 'image/png', direction: 'input', maxBytes: 100 }); + assert.equal(t.reason, 'bootstrap_pending'); assert.equal(f.manager.status().ownedBytes, 0); assert.equal(f.manager.status().openTickets, 0); + await f.manager.shutdown(1); +}); +test('T-PF-04 create_recording needs client AND server record_audio; T-PF-05 import is not recording', async () => { + const off = await managerV1({}, {}, { recordAudio: false }); + assert.equal(off.manager.preflight(identity, { mimeType: 'audio/wav', direction: 'mixed', maxBytes: 10, operation: 'create_recording' }).reason, 'recording_disabled'); + assert.equal(off.manager.preflight(identity, { mimeType: 'audio/wav', direction: 'mixed', maxBytes: 10, operation: 'import_existing' }).reason, 'queued'); + const server = await managerV1({}, { policy: { record_audio: false } }, { recordAudio: true }); + assert.equal(server.manager.preflight(identity, { mimeType: 'audio/wav', direction: 'mixed', maxBytes: 10, operation: 'create_recording' }).reason, 'recording_disabled'); + const both = await managerV1({}, {}, { recordAudio: true }); + assert.equal(both.manager.preflight(identity, { mimeType: 'audio/wav', direction: 'mixed', maxBytes: 10, operation: 'create_recording' }).reason, 'queued'); + for (const f of [off, server, both]) await f.manager.shutdown(1); +}); +test('T-PF-06 mixed direction with hideInputs; T-PF-07 computer_screenshot output with hideInputImages', async () => { + const a = await managerV1({}, {}, { traceConfig: { hideInputs: true } }); + assert.equal(a.manager.preflight(identity, { mimeType: 'audio/wav', direction: 'mixed', maxBytes: 10 }).reason, 'policy_denied'); + const b = await managerV1({}, {}, { traceConfig: { hideInputImages: true } }); + assert.equal(b.manager.preflight(identity, { mimeType: 'image/png', direction: 'output', purpose: 'computer_screenshot', maxBytes: 10 }).reason, 'policy_denied'); + assert.equal(b.manager.preflight(identity, { mimeType: 'image/png', direction: 'output', purpose: 'model_output', maxBytes: 10 }).reason, 'queued'); + await a.manager.shutdown(1); await b.manager.shutdown(1); +}); +test('T-PF-08 descriptors or bytes exhausted -> queue_full', async () => { + const f = await managerV1({}, {}, { maxDescriptors: 1, maxOwnedBytes: 100 }); + const first = f.manager.preflight(identity, { ...options, maxBytes: 10 }); assert.equal(first.reason, 'queued'); + assert.equal(f.manager.preflight(identity, { ...options, maxBytes: 10 }).reason, 'queue_full'); + first.release(); + assert.equal(f.manager.preflight(identity, { ...options, maxBytes: 101 }).reason, 'too_large'); + const g = await managerV1({}, {}, { maxOwnedBytes: 100 }); + const held = g.manager.preflight(identity, { ...options, maxBytes: 90 }); assert.equal(held.reason, 'queued'); + assert.equal(g.manager.preflight(identity, { ...options, maxBytes: 20 }).reason, 'queue_full'); + held.release(); assert.equal(g.manager.status().ownedBytes, 0); + await f.manager.shutdown(1); await g.manager.shutdown(1); +}); +test('T-PF-11 ticket used twice -> conflict and the source is untouched', async () => { + const f = await managerV1(); + const t = f.manager.preflight(identity, { ...options, maxBytes: 8 }); + assert.equal(t.attach({ kind: 'Bytes', data: new Uint8Array([1, 2, 3]) }).admission, 'queued'); + let read = false; + assert.equal(t.attach({ kind: 'Bytes', get data() { read = true; return new Uint8Array(1); } }).admission, 'conflict'); assert.equal(read, false); + await f.manager.shutdown(); +}); +test('T-PF-12 ticket TTL with a fake clock: expired, budget returned lazily, no timer armed', async () => { + const c = clock(); const f = await managerV1({}, { clock: c }); + const timers = []; const original = global.setTimeout; + global.setTimeout = (...args) => { timers.push(args[1]); return original(...args); }; + let t; + try { t = f.manager.preflight(identity, { ...options, maxBytes: 64, ttlMs: 100 }); } finally { global.setTimeout = original; } + assert.equal(timers.length, 0, 'preflight armed a timer'); + assert.equal(f.manager.status().ownedBytes, 64); + c.advance(150); + assert.equal(f.manager.status().ownedBytes, 0); assert.equal(f.manager.status().openTickets, 0); + assert.equal(t.attach({ kind: 'Bytes', data: new Uint8Array(4) }).admission, 'expired'); + await f.manager.shutdown(1); +}); +test('T-PF-13 policy tightened between preflight and attach: new denial, zero reads', async () => { + const f = await managerV1(); + const t = f.manager.preflight(identity, { ...options, maxBytes: 8 }); + f.manager.updatePolicy({ traceConfig: { hideInputs: true } }); + let read = false; + assert.equal(t.attach({ kind: 'Bytes', get data() { read = true; return new Uint8Array(1); } }).admission, 'policy_denied'); + assert.equal(read, false); assert.equal(f.manager.status().ownedBytes, 0); await f.manager.shutdown(1); +}); +test('T-PF-14 actual length above the ticket -> too_large, no copy; T-PF-15 mutable buffer above syncCopyBytes', async () => { + const f = await managerV1({}, {}, { syncCopyBytes: 16 }); + const t = f.manager.preflight(identity, { ...options, maxBytes: 4 }); + assert.equal(t.attach({ kind: 'Bytes', data: new Uint8Array(5) }).admission, 'too_large'); + assert.equal(f.manager.status().ownedBytes, 0); + const big = f.manager.preflight(identity, { ...options, maxBytes: 64 }); + assert.equal(big.attach({ kind: 'Bytes', data: new Uint8Array(32) }).admission, 'too_large'); + const blob = f.manager.preflight(identity, { ...options, maxBytes: 64, sourceKind: 'Blob' }); + const h = blob.attach({ kind: 'Blob', data: new Blob([new Uint8Array(32)]) }); + assert.equal(h.admission, 'queued'); assert.equal((await h.completion).status, 'validating'); + await f.manager.shutdown(); +}); +test('preflight pattern: denied ticket means no decode happened (adapter shape from design 1.5)', async () => { + const f = await managerV1({}, {}, { traceConfig: { hideInputImages: true } }); + let decoded = 0; const decode = () => { decoded++; return new Uint8Array(4); }; + const t = f.manager.preflight(identity, { mimeType: 'image/png', direction: 'input', purpose: 'model_input', maxBytes: 4 }); + if (t.admitted) t.attach({ kind: 'Bytes', data: decode() }); + assert.equal(decoded, 0); await f.manager.shutdown(1); +}); + +// ---- 10.2 Streaming recordings ---------------------------------------------------------------------------- +test('T-ST-01..04 worked vectors: segment frames, bytes and milliseconds', () => { + assert.equal(attachmentSegmentFrames(24000, 2), 720000); assert.equal(44 + 720000 * 2, 1440044); + assert.equal(attachmentSegmentFrames(48000, 4), 1440000); assert.equal(44 + 1440000 * 4, 5760044); + assert.equal(attachmentSegmentFrames(48000, 16), 524256); assert.equal(44 + 524256 * 16, 8388140); assert.equal(attachmentFramesToMs(524256, 48000), 10922); + assert.equal(attachmentFramesToMs(10000, 44100, true), 227); + const h = attachmentWavHeader(1, 24000, 1440000); const v = new DataView(h.buffer); + assert.equal(Buffer.from(h.subarray(0, 4)).toString(), 'RIFF'); assert.equal(v.getUint32(4, true), 1440036); assert.equal(Buffer.from(h.subarray(8, 16)).toString(), 'WAVEfmt '); + assert.deepEqual([v.getUint32(16, true), v.getUint16(20, true), v.getUint16(22, true), v.getUint32(24, true), v.getUint32(28, true), v.getUint16(32, true), v.getUint16(34, true)], [16, 1, 1, 24000, 48000, 2, 16]); + assert.equal(Buffer.from(h.subarray(36, 40)).toString(), 'data'); assert.equal(v.getUint32(40, true), 1440000); +}); +test('T-ST-01 24 kHz mono 30 s: one 1 440 044-byte segment with an exact header; T-ST-20 carrier stamp, no references', async () => { + const f = await managerV1(); const s = span(); + const w = f.manager.startRecording(s, mono(24000)); assert.equal(w.reason, 'queued'); + assert.equal(s.attrs['fi.attachment.refs'], '[]'); assert.equal(s.attrs['fi.attachment.schema_version'], 1); + assert.equal(s.attrs['fi.attachment.capture_session_id'], f.manager.sessionId); assert.equal(s.attrs['fi.attachment.correlation_nonce'], f.manager.nonce); + assert.equal(validAttachmentWire('SpanAttributes', s.attrs), true); + await writeSeconds(f.manager, w, 24000, 30); + const result = await w.finish(10000); + assert.equal(result.state, 'published'); assert.equal(result.captureComplete, true); + const sizes = f.calls.filter(c => c.op === 'putContent').map(c => c.size); assert.deepEqual(sizes, [1440044]); + const body = [...f.contents.values()][0]; assert.deepEqual([...body.subarray(0, 44)], [...attachmentWavHeader(1, 24000, 1440000)]); + const manifest = f.manifests.at(-1).body; + assert.deepEqual(manifest.segments.map(s => [s.index, s.start_ms, s.duration_ms]), [[0, 0, 30000]]); + assert.equal(manifest.duration_ms, 30000); assert.equal(validAttachmentWire('RecordingManifest', manifest), true); + assert.equal(s.attrs['fi.attachment.refs'], '[]', 'segments are never projected as references'); + const request = f.requests[0]; + assert.equal(request.recording_id, w.recordingId); assert.equal(request.segment_index, 0); assert.equal(request.declared_bytes, 1440044); + assert.equal(validAttachmentWire('UploadRequest', request), true); + await f.manager.shutdown(); +}); +test('T-ST-02 24 kHz mono 61 s: 1 440 044 / 1 440 044 / 48 044, duration 61 000, capture_complete', async () => { + const f = await managerV1(); + const w = f.manager.startRecording(identity, mono(24000)); + await writeSeconds(f.manager, w, 24000, 61); + const result = await w.finish(10000); + assert.deepEqual(f.calls.filter(c => c.op === 'putContent').map(c => c.size), [1440044, 1440044, 48044]); + const manifest = f.manifests.at(-1).body; + assert.equal(manifest.duration_ms, 61000); assert.equal(manifest.capture_complete, true); assert.deepEqual(manifest.gaps, []); + assert.deepEqual(manifest.segments.map(s => s.duration_ms), [30000, 30000, 1000]); assert.equal(result.state, 'published'); + await f.manager.shutdown(); +}); +test('T-ST-03 44.1 kHz mono 10 000 frames: 20 044 bytes, duration 227', async () => { + const f = await managerV1(); + const w = f.manager.startRecording(identity, mono(44100)); + assert.equal(w.tryWrite(pcm(10000)), true); + const result = await w.finish(10000); + assert.deepEqual(f.calls.filter(c => c.op === 'putContent').map(c => c.size), [20044]); + assert.equal(f.manifests.at(-1).body.duration_ms, 227); assert.equal(f.manifests.at(-1).body.segments[0].duration_ms, 227); assert.equal(result.captureComplete, true); + await f.manager.shutdown(); +}); +test('T-ST-05 negotiated segment_duration_ms 10 000 and segment_bytes 1 MiB are honoured', async () => { + const f = await managerV1({}, { limits: { segment_duration_ms: 10000, segment_bytes: 1048576 } }); + const w = f.manager.startRecording(identity, mono(24000)); + await writeSeconds(f.manager, w, 24000, 25, { flushEvery: 5 }); + await w.finish(10000); + const sizes = f.calls.filter(c => c.op === 'putContent').map(c => c.size); + assert.deepEqual(sizes, [480044, 480044, 240044]); assert.ok(sizes.every(s => s <= 1048576)); + assert.ok(f.manifests.at(-1).body.segments.every(s => s.duration_ms <= 10000)); + await f.manager.shutdown(); +}); +const twoTracks = { format: { sampleRateHz: 16000 }, layout: 'separate_tracks', operation: 'observe_consumed', + tracks: [{ channels: [0], direction: 'input', role: 'user' }, { channels: [1], direction: 'output', role: 'assistant' }] }; +test('T-ST-06 separate tracks with one track starved for 5 s; T-ST-07 track_id rules', async () => { + const f = await managerV1(); + const w = f.manager.startRecording(identity, twoTracks); + for (let i = 0; i < 250; i++) { w.tryWrite(pcm(320), 0); w.tryWrite(pcm(320), 1); } // 5 s each + for (let i = 0; i < 250; i++) w.tryWrite(pcm(320), 0); // track 1 starved for 5 s + const result = await w.finish(10000); + const m = f.manifests.at(-1).body; + assert.equal(validAttachmentManifest(m), true); assert.equal(validAttachmentWire('RecordingManifest', m), true); + assert.ok(m.segments.every(s => s.track_id) && m.gaps.every(g => g.track_id)); + const user = m.tracks[0].track_id; const bot = m.tracks[1].track_id; + assert.deepEqual(m.segments.filter(s => s.track_id === user).map(s => [s.start_ms, s.duration_ms]), [[0, 10000]]); + assert.deepEqual(m.segments.filter(s => s.track_id === bot).map(s => [s.start_ms, s.duration_ms]), [[0, 5000]]); + assert.deepEqual(m.gaps.map(g => [g.track_id === bot, g.start_ms, g.duration_ms, g.reason]), [[true, 5000, 5000, 'source_unavailable']]); + assert.equal(result.state, 'partial'); assert.equal(m.capture_complete, false); + const mutated = JSON.parse(JSON.stringify(m)); delete mutated.gaps[0].track_id; + assert.equal(validAttachmentManifest(mutated), false, 'F01 separate-track-needs-track-identity'); + await f.manager.shutdown(); +}); +test('T-ST-08 stereo: two tracks [0] and [1], channels 2; a [0],[0] mutation fails locally', async () => { + const f = await managerV1(); + const w = f.manager.startRecording(identity, { ...twoTracks, layout: 'stereo' }); + assert.equal(w.reason, 'queued'); + for (let i = 0; i < 50; i++) assert.equal(w.tryWrite(pcm(320, 2)), true); + await w.finish(10000); + const m = f.manifests.at(-1).body; + assert.equal(m.channels, 2); assert.deepEqual(m.tracks.map(t => t.channels), [[0], [1]]); assert.ok(m.segments.every(s => !('track_id' in s))); + assert.equal(f.requests[0].direction, 'mixed'); assert.equal(validAttachmentManifest(m), true); + const mutated = JSON.parse(JSON.stringify(m)); mutated.tracks[1].channels = [0]; assert.equal(validAttachmentManifest(mutated), false); + const mixedMutation = JSON.parse(JSON.stringify(m)); mixedMutation.segments[0].track_id = m.tracks[0].track_id; assert.equal(validAttachmentManifest(mixedMutation), false); + await f.manager.shutdown(); +}); +test('T-ST-09 budget exhausted mid-recording: tryWrite false, queue_full gap of the exact duration, no exception', async () => { + const f = await managerV1({}, {}, { maxOwnedBytes: 32000 }); // 1 s of 16 kHz mono + const w = f.manager.startRecording(identity, mono(16000)); + const a = await writeSeconds(f.manager, w, 16000, 1, { flushEvery: 0 }); assert.equal(a.rejected, 0); + const b = await writeSeconds(f.manager, w, 16000, 1, { flushEvery: 0 }); assert.equal(b.accepted, 0); + await w.finish(10000); + const m = f.manifests.at(-1).body; + assert.deepEqual(m.gaps.map(g => [g.start_ms, g.duration_ms, g.reason]), [[1000, 1000, 'queue_full']]); + await f.manager.shutdown(); +}); +test('T-ST-10 window full (blocked transport): queue_full gaps, at most windowSegments sealed segments held', async () => { + const f = await managerV1(base => ({ putContent: (_s, _u, _src, signal) => new Promise((_, reject) => signal.addEventListener('abort', () => reject(new AttachmentTransportError('incomplete')), { once: true })) }), + { limits: { segment_duration_ms: 1000 } }); + const w = f.manager.startRecording(identity, mono(16000)); + const r = await writeSeconds(f.manager, w, 16000, 5, { flushEvery: 0 }); + assert.ok(r.rejected > 0); + assert.ok(f.manager.status().ownedBytes <= 2 * (32000 + 44) + 262144, `owned ${f.manager.status().ownedBytes}`); + const result = await w.finish(500); assert.ok(['partial', 'unpublished'].includes(result.state)); + await f.manager.shutdown(100); +}); +test('T-ST-11 bootstrap pending at start, then ready: leading bootstrap_pending gap and no retained bytes before ready', async () => { + let release; const gate = new Promise(resolve => { release = resolve; }); + const f = await managerV1(base => ({ bootstrap: async (body, key) => { await gate; return base.bootstrap(body, key); } }), { ready: false }); + const w = f.manager.startRecording(identity, mono(16000)); + assert.equal(w.reason, 'bootstrap_pending'); + await writeSeconds(f.manager, w, 16000, 1, { flushEvery: 0 }); + assert.equal(f.manager.status().ownedBytes, 0); + release(); assert.equal(await f.manager.ready(), true); + const r = await writeSeconds(f.manager, w, 16000, 1, { flushEvery: 0 }); assert.equal(r.rejected, 0); + assert.equal(w.reason, 'queued'); + await w.finish(10000); + const m = f.manifests.at(-1).body; + assert.deepEqual(m.gaps.map(g => [g.start_ms, g.duration_ms, g.reason]), [[0, 1000, 'bootstrap_pending']]); + assert.deepEqual(m.segments.map(s => [s.start_ms, s.duration_ms]), [[1000, 1000]]); + await f.manager.shutdown(); +}); +test('T-ST-12 policy tightened mid-recording: writer terminal, bytes freed, final manifest ends with policy_denied', async () => { + const f = await managerV1({}, { limits: { segment_duration_ms: 1000 } }); + const w = f.manager.startRecording(identity, mono(16000)); + await writeSeconds(f.manager, w, 16000, 2, { flushEvery: 1 }); + f.manager.updatePolicy({ traceConfig: { hideInputs: true } }); + assert.equal(w.tryWrite(pcm(320)), false); assert.equal(w.tryWrite(pcm(320)), false); + await f.manager.flush(); assert.equal(f.manager.status().ownedBytes, 0); + await w.finish(10000); + const m = f.manifests.at(-1).body; + assert.equal(m.segments.length, 2); assert.equal(m.gaps.at(-1).reason, 'policy_denied'); assert.equal(m.capture_complete, false); + await f.manager.shutdown(); +}); +test('T-ST-13 recording cap (recording_duration_ms 60 000): truncated gap to the end, capture_complete false', async () => { + const f = await managerV1({}, { limits: { recording_duration_ms: 60000 } }); + const w = f.manager.startRecording(identity, mono(8000)); + await writeSeconds(f.manager, w, 8000, 61); + await w.finish(10000); + const m = f.manifests.at(-1).body; + assert.deepEqual(m.gaps.map(g => [g.start_ms, g.duration_ms, g.reason]), [[60000, 1000, 'truncated']]); assert.equal(m.capture_complete, false); + await f.manager.shutdown(); +}); +test('T-ST-14 a segment failing after initialize is aborted and its index reused; a 409 on reuse closes partial', async () => { + let failures = 0; + const f = await managerV1(base => ({ putContent: async (s, u, src, sig) => { + if (base._first === undefined) base._first = u.occurrence_id; + if (u.occurrence_id === base._first) { failures++; throw new AttachmentTransportError('incomplete', true, 0, 500); } + return base.putContent(s, u, src, sig); + } }), { limits: { segment_duration_ms: 1000 } }); + const w = f.manager.startRecording(identity, mono(16000)); + await writeSeconds(f.manager, w, 16000, 3, { flushEvery: 1 }); + await w.finish(10000); + assert.equal(failures, 3); assert.equal(f.calls.filter(c => c.op === 'abort').length, 1); + assert.deepEqual(f.requests.map(r => r.segment_index), [0, 0, 1]); + const m = f.manifests.at(-1).body; + assert.deepEqual(m.gaps.map(g => [g.start_ms, g.duration_ms, g.reason]), [[0, 1000, 'incomplete']]); + assert.deepEqual(m.segments.map(s => [s.index, s.start_ms]), [[0, 1000], [1, 2000]]); + await f.manager.shutdown(); + // Reuse rejected: seg0 ok, seg1 fails, seg2 (reusing index 1) gets 409 -> partial with the published prefix. + let n = 0; + const g = await managerV1(base => ({ + putContent: async (s, u, src, sig) => { if (u.occurrence_id === base._second) throw new AttachmentTransportError('incomplete', true, 0, 500); return base.putContent(s, u, src, sig); }, + initialize: async (s, body, key, sig) => { + n++; const r = JSON.parse(body); + if (n === 2) base._second = r.occurrence_id; + if (n === 3) throw new AttachmentTransportError('conflict', false, 0, 409); + return base.initialize(s, body, key, sig); + } }), { limits: { segment_duration_ms: 1000 } }); + const w2 = g.manager.startRecording(identity, mono(16000)); + await writeSeconds(g.manager, w2, 16000, 3, { flushEvery: 1 }); + const result = await w2.finish(10000); + assert.equal(result.state, 'partial'); + const last = g.manifests.at(-1).body; + assert.deepEqual(last.segments.map(s => s.index), [0]); assert.equal(last.gaps.at(-1).reason, 'incomplete'); assert.equal(last.duration_ms, 3000); + await g.manager.shutdown(); +}); +test('T-ST-15 intermediate manifests are prefix-monotonic across 10 PUTs; T-ST-16 latest completed capability only', async () => { + const f = await managerV1({}, { limits: { segment_duration_ms: 1000 } }); + const w = f.manager.startRecording(identity, mono(8000)); + await writeSeconds(f.manager, w, 8000, 40, { flushEvery: 1 }); + await w.finish(10000); + assert.ok(f.manifests.length >= 10, `manifests ${f.manifests.length}`); + for (let i = 1; i < f.manifests.length; i++) { + const a = f.manifests[i - 1].body; const b = f.manifests[i].body; + const has = (list, item) => list.some(x => JSON.stringify(x) === JSON.stringify(item)); + assert.ok(a.segments.every(x => has(b.segments, x)) && a.gaps.every(x => has(b.gaps, x)), `manifest ${i} dropped an entry`); + assert.ok(b.duration_ms >= a.duration_ms); + } + assert.deepEqual(f.manifests.map(m => m.key), f.manifests.map((_, i) => `mf-${hex(w.recordingId)}-${i + 1}`)); + for (const m of f.manifests) { + const top = m.body.segments.at(-1); + assert.equal(m.capability, f.completed.get(top.occurrence_id), 'manifest must use the latest completed segment capability'); + } + assert.equal(f.manifests.at(-1).body.capture_complete, true); + await f.manager.shutdown(); +}); +test('T-ST-17 capture_complete only with zero gaps (F01 unmarked-capture-gap, complete-without-any-media)', () => { + const rid = '40000000-0000-4000-8000-000000000001'; const track = '40000000-0000-4000-8000-000000000002'; + const base = { schema_version: 1, recording_id: rid, timebase: 'session_relative_ms', sample_format: 's16le', sample_rate_hz: 16000, channels: 1, codec: 'pcm', container: 'wav', + layout: 'mixed', tracks: [{ track_id: track, channels: [0], direction: 'mixed', role: 'mixed' }], segments: [], gaps: [{ start_ms: 0, duration_ms: 10, reason: 'incomplete' }], playout: [], capture_complete: true, duration_ms: 10 }; + assert.equal(validAttachmentManifest(base), false); assert.equal(validAttachmentManifest({ ...base, capture_complete: false }), true); +}); +test('T-ST-18 tryWrite never performs I/O on the caller (10 000 calls, blocked transport)', async () => { + const f = await managerV1(() => ({ putContent: (_s, _u, _src, signal) => new Promise((_, reject) => signal.addEventListener('abort', () => reject(new AttachmentTransportError('incomplete')), { once: true })) })); + const w = f.manager.startRecording(identity, mono(16000)); + const before = f.calls.length; const times = []; + const chunk = pcm(320); + for (let i = 0; i < 10000; i++) { const t = process.hrtime.bigint(); w.tryWrite(chunk); times.push(Number(process.hrtime.bigint() - t) / 1e6); } + assert.equal(f.calls.length, before, 'tryWrite performed transport I/O synchronously'); + times.sort((a, b) => a - b); console.log(`# T-ST-18 p99 tryWrite ms ${times[Math.floor(times.length * 0.99)].toFixed(4)} (recorded, not asserted)`); + w.abort(); await f.manager.shutdown(100); +}); +test('T-ST-19 finish(1000) with a blocked transport returns within 1.25 s', async () => { + const f = await managerV1(() => ({ initialize: () => new Promise(() => {}) }), { limits: { segment_duration_ms: 1000 } }); + const w = f.manager.startRecording(identity, mono(16000)); + await writeSeconds(f.manager, w, 16000, 2, { flushEvery: 0 }); + const start = Date.now(); const result = await w.finish(1000); const elapsed = Date.now() - start; + assert.ok(elapsed <= 1250, `finish took ${elapsed} ms`); assert.ok(['partial', 'unpublished'].includes(result.state)); + await f.manager.shutdown(100); +}); +test('recording abort is synchronous and PUTs no manifest', async () => { + const f = await managerV1(); + const w = f.manager.startRecording(identity, mono(16000)); + await writeSeconds(f.manager, w, 16000, 1, { flushEvery: 0 }); + w.abort(); assert.equal(f.manager.status().ownedBytes, 0); + assert.equal((await w.finish()).state, 'unpublished'); assert.equal(f.manifests.length, 0); + await f.manager.shutdown(); +}); + +// ---- 10.3 Renewal ------------------------------------------------------------------------------------- +function httpDouble(handler) { + const requests = []; + const client = async request => { + const body = request.body === undefined ? undefined : typeof request.body === 'string' ? request.body : Buffer.from(await readAll(request.body)); + const entry = { url: new URL(request.url).pathname, method: request.method, headers: request.headers, body }; requests.push(entry); + const out = await handler(entry, requests); + return { status: out.status, body: Buffer.from(out.body === undefined ? '' : typeof out.body === 'string' ? out.body : JSON.stringify(out.body)), retryAfter: out.retryAfter }; + }; + return { requests, client }; +} +const sessionBody = (sid, extra = {}) => ({ schema_version: 1, scope: { organization_id: ids[0], workspace_id: ids[1], project_id: ids[2] }, capture_session_id: sid, + policy: { ...policy, allowed_sources: ['Bytes', 'Blob', 'OwnedRecordingFile', 'ProviderFile', 'RemoteUrl'], record_audio: true }, limits, + expires_at: new Date(Date.now() + 3600000).toISOString(), transfer_modes: ['authenticated_stream'], ...extra }); +let capCounter = 0; +function standardRoutes(entry, overrides = {}) { + const path = entry.url; + if (path.endsWith('/attachment-sessions')) return overrides.session ? overrides.session(entry) : { status: 200, body: sessionBody(JSON.parse(entry.body).capture_session_id) }; + if (path.endsWith('/uploads')) { + if (overrides.upload) { const out = overrides.upload(entry); if (out) return out; } + const r = JSON.parse(entry.body); + return { status: 201, body: { upload_id: '50000000-0000-4000-8000-000000000001', attachment_id: r.attachment_id, occurrence_id: r.occurrence_id, state: 'pending_upload', + reserved_bytes: r.declared_bytes, expires_at: new Date(Date.now() + 600000 + (++capCounter) * 1000).toISOString(), transfer_mode: 'authenticated_stream', write_capability: `synthetic-capability-${capCounter}-abcdef` } }; + } + if (path.endsWith('/content')) return overrides.content ? overrides.content(entry) : { status: 200, body: { upload_id: path.split('/')[7], state: 'pending_upload', reason: 'queued' } }; + if (path.endsWith('/complete')) return { status: 202, body: { upload_id: path.split('/')[7], state: 'validating', reason: 'binding_pending' } }; + if (path.endsWith('/abort')) return { status: 200, body: { upload_id: path.split('/')[7], state: 'failed', reason: 'incomplete' } }; + if (path.endsWith('/manifest')) return { status: 204 }; + if (path.endsWith('/imports')) { const b = JSON.parse(entry.body); return overrides.imports ? overrides.imports(entry) : { status: 202, body: { mode: 'completed_recording', job_id: '60000000-0000-4000-8000-000000000001', recording_id: b.recording.recording_id, state: 'importing' } }; } + if (path.includes('/imports/') && path.endsWith('/status')) return { status: 200, body: { job_id: path.split('/')[7], state: 'importing', reason: 'binding_pending' } }; + return { status: 404 }; +} +async function httpManager(overrides = {}, managerOptions = {}) { + const d = httpDouble(entry => standardRoutes(entry, overrides)); + const transport = createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic-original', 'x-api-key': 'synthetic-original-key' }, client: d.client }); + const manager = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport, recordAudio: true, ...managerOptions }); + await manager.ready(); + return { manager, ...d }; +} +test('T-RN-01 session near expiry renews with the same id and nonce under key bs-{sid}-1', async () => { + const f = await httpManager({ session: entry => ({ status: 200, body: sessionBody(JSON.parse(entry.body).capture_session_id, { expires_at: new Date(Date.now() + 50000).toISOString() }) }) }); + f.manager.preflight(identity, { ...options, maxBytes: 4 }).release(); + for (let i = 0; i < 20 && f.requests.filter(r => r.url.endsWith('/attachment-sessions')).length < 2; i++) await tick(); + const boots = f.requests.filter(r => r.url.endsWith('/attachment-sessions')); + assert.equal(boots.length, 2); + assert.equal(boots[0].headers['idempotency-key'], `bs-${hex(f.manager.sessionId)}-0`); assert.equal(boots[1].headers['idempotency-key'], `bs-${hex(f.manager.sessionId)}-1`); + assert.equal(boots[1].body, boots[0].body); assert.equal(JSON.parse(boots[1].body).correlation_nonce, f.manager.nonce); + assert.equal(boots[1].headers.authorization, 'Bearer synthetic-original'); + await f.manager.shutdown(); +}); +test('T-RN-02 renewal with a different scope invalidates the manager (scope_unresolved); T-RN-03 tighter policy drops queued work', async () => { + let renew; + const f = await managerV1(base => ({ renewSession: () => new Promise(resolve => { renew = resolve; }) }), { lifetimeMs: 50000 }); + f.manager.preflight(identity, { ...options, maxBytes: 4 }).release(); await tick(); + renew({ ...sessionBody(f.manager.sessionId), scope: { organization_id: ids[0], workspace_id: ids[1], project_id: '70000000-0000-4000-8000-000000000001' } }); + for (let i = 0; i < 10; i++) await tick(); + assert.equal(f.manager.status().state, 'scope_unresolved'); + assert.equal(f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array(2) }, options).admission, 'scope_unresolved'); + let renew2; let unblock; + const block = new Promise(resolve => { unblock = resolve; }); + const g = await managerV1(base => ({ renewSession: () => new Promise(resolve => { renew2 = resolve; }), initialize: async (...a) => { await block; return base.initialize(...a); } }), { lifetimeMs: 50000 }); + const handles = [1, 2, 3].map(() => g.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array(2) }, options)); + await tick(); + renew2({ ...sessionBody(g.manager.sessionId), policy: { ...sessionBody(g.manager.sessionId).policy, version: 'v2', allowed_modalities: ['audio'] } }); + for (let i = 0; i < 10; i++) await tick(); + unblock(); + assert.deepEqual((await Promise.all(handles.map(h => h.completion))).map(c => c.status), ['policy_denied', 'policy_denied', 'policy_denied']); + assert.equal(g.manager.status().ownedBytes, 0); + assert.equal(g.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array(2) }, { mimeType: 'audio/wav', direction: 'mixed' }).admission, 'queued'); + await f.manager.shutdown(1); await g.manager.shutdown(); +}); +test('T-RN-04 renewal fails until expiry: state expired, recordings get incomplete gaps, initialized uploads continue', async () => { + const c = clock(Date.now()); let unblock; const block = new Promise(resolve => { unblock = resolve; }); + const f = await managerV1(base => ({ renewSession: async () => { throw new AttachmentTransportError('incomplete', true); }, + putContent: async (...a) => { await block; return base.putContent(...a); } }), { clock: c, lifetimeMs: 30000 }, { clock: c }); + const h = f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array(4) }, options); + const w = f.manager.startRecording(identity, mono(16000)); + assert.equal(w.tryWrite(pcm(16000)), true); + for (let i = 0; i < 20 && !f.calls.some(x => x.op === 'putContent' || x.op === 'initialize'); i++) await tick(); + c.advance(30000); + assert.equal(f.manager.status().state, 'expired'); + assert.equal(f.manager.preflight(identity, { ...options, maxBytes: 4 }).reason, 'expired'); + assert.equal(w.tryWrite(pcm(16000)), false); + unblock(); assert.equal((await h.completion).status, 'validating'); + const result = await w.finish(2000); assert.notEqual(result.state, 'published'); + await f.manager.shutdown(100); +}); +test('T-RN-05 capability near expiry before content: replay with the byte-identical body under up-{occ}; same upload_id', async () => { + let inits = 0; + const f = await httpManager({ upload: entry => { + inits++; const r = JSON.parse(entry.body); + const expires = inits === 1 ? Date.now() + 3000 : Date.now() + 600000; + return { status: 201, body: { upload_id: '50000000-0000-4000-8000-000000000001', attachment_id: r.attachment_id, occurrence_id: r.occurrence_id, state: 'pending_upload', + reserved_bytes: r.declared_bytes, expires_at: new Date(expires).toISOString(), transfer_mode: 'authenticated_stream', write_capability: `synthetic-capability-renewed-${inits}` } }; + } }); + const h = f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1, 2, 3]) }, options); + assert.equal((await h.completion).status, 'validating'); + const uploads = f.requests.filter(r => r.url.endsWith('/uploads')); + assert.equal(uploads.length, 2); assert.equal(uploads[1].body, uploads[0].body); + assert.equal(uploads[0].headers['idempotency-key'], `up-${hex(h.occurrenceId)}`); assert.equal(uploads[1].headers['idempotency-key'], uploads[0].headers['idempotency-key']); + assert.equal(uploads[1].headers.authorization, 'Bearer synthetic-original'); + assert.equal(f.requests.find(r => r.url.endsWith('/content')).headers.authorization, 'Bearer synthetic-capability-renewed-2'); + await f.manager.shutdown(); +}); +test('T-RN-06 replay returning the same capability: upload expired and aborted, no new upload id', async () => { + const f = await httpManager({ upload: entry => { + const r = JSON.parse(entry.body); + return { status: 201, body: { upload_id: '50000000-0000-4000-8000-000000000001', attachment_id: r.attachment_id, occurrence_id: r.occurrence_id, state: 'pending_upload', + reserved_bytes: r.declared_bytes, expires_at: new Date(Date.now() + 3000).toISOString(), transfer_mode: 'authenticated_stream', write_capability: 'synthetic-capability-same-0000' } }; + } }); + const h = f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, options); + assert.equal((await h.completion).status, 'expired'); + assert.equal(f.requests.filter(r => r.url.endsWith('/abort')).length, 1); + assert.equal(new Set(f.requests.filter(r => r.url.endsWith('/uploads')).map(r => JSON.parse(r.body).occurrence_id)).size, 1); + await f.manager.shutdown(); +}); +test('T-RN-07 401 on content: one renewal then one retry; a second 401 is credential_scope_unsupported', async () => { + let puts = 0; + const ok = await httpManager({ content: entry => (++puts === 1 ? { status: 401 } : { status: 200, body: { upload_id: entry.url.split('/')[7], state: 'pending_upload', reason: 'queued' } }) }); + assert.equal((await ok.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, options).completion).status, 'validating'); + assert.equal(ok.requests.filter(r => r.url.endsWith('/uploads')).length, 2); + const bad = await httpManager({ content: () => ({ status: 401 }) }); + assert.equal((await bad.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, options).completion).status, 'credential_scope_unsupported'); + assert.equal(bad.requests.filter(r => r.url.endsWith('/content')).length, 2); + await ok.manager.shutdown(); await bad.manager.shutdown(); +}); +test('T-RN-08 the write capability never appears in status, handles, span attributes or errors', async () => { + const f = await httpManager({ content: () => ({ status: 500 }) }); + const s = span(); + const h = f.manager.attach(s, { kind: 'Bytes', data: new Uint8Array([1]) }, options); + const completion = await h.completion; + const caps = f.requests.filter(r => r.url.endsWith('/content')).map(r => r.headers.authorization.slice(7)); + const haystack = JSON.stringify([f.manager.status(), h, completion, s.attrs, String(new AttachmentTransportError(completion.status))]); + for (const cap of caps) assert.equal(haystack.includes(cap), false); + await f.manager.shutdown(); +}); +test('T-RN-09 recording-only workload (tryWrite + flush, never preflight) renews before expiry and keeps its tail', async () => { + // Pipecat shape: one long recording, 20 ms writes, a flush every second, no other entry point (design 3.1). + const c = clock(Date.now()); + const f = await managerV1({}, { clock: c, lifetimeMs: 120000, limits: { segment_duration_ms: 10000 } }); + const w = f.manager.startRecording(identity, mono(8000)); + assert.equal(w.reason, 'queued'); + let rejectedAfter = 0; const stateAt = {}; + for (let ms = 0; ms < 150000; ms += 20) { + const ok = w.tryWrite(pcm(160, 1, ms)); + if (ms >= 120000 && !ok) rejectedAfter++; + c.advance(20); + if (ms % 1000 === 0) { await f.manager.flush(5000); for (let i = 0; i < 5; i++) await tick(); } + if (ms === 100000 || ms === 116000 || ms === 130000) stateAt[ms] = f.manager.status().state; + } + const result = await w.finish(10000); + const manifest = f.manifests.at(-1).body; + assert.ok(f.calls.filter(x => x.op === 'renewSession').length >= 1); + assert.deepEqual(stateAt, { 100000: 'ready', 116000: 'ready', 130000: 'ready' }); + assert.equal(rejectedAfter, 0); + assert.equal(result.state, 'published'); assert.equal(result.captureComplete, true); + assert.equal(manifest.segments.length, 15); assert.deepEqual(manifest.gaps, []); + await f.manager.shutdown(1000); +}); +test('T-RN-10 each non-admission entry point is a renewal trigger on its own: recording write, flush, status, worker wake', async () => { + const renewals = f => f.calls.filter(x => x.op === 'renewSession').length; + const settle = async f => { for (let i = 0; i < 20 && !renewals(f); i++) await tick(); }; + const fresh = async (overrides = {}) => { const c = clock(Date.now()); return { c, f: await managerV1(overrides, { clock: c, lifetimeMs: 120000, limits: { segment_duration_ms: 10000 } }) }; }; + // Recording write: a sub-segment write seals nothing and enqueues nothing; only the deferred host check can renew. + { const { c, f } = await fresh(); + const w = f.manager.startRecording(identity, mono(8000)); + assert.equal(w.tryWrite(pcm(160)), true); + for (let i = 0; i < 5; i++) await tick(); + assert.equal(renewals(f), 0); + c.advance(70000); + const before = f.calls.length; + assert.equal(w.tryWrite(pcm(160, 1, 1)), true); + assert.equal(f.calls.length, before, 'tryWrite performs no transport call on the caller stack'); + await settle(f); assert.equal(renewals(f), 1, 'recording write'); + w.abort(); await f.manager.shutdown(100); } + // flush() and status() with no other work. + for (const [name, call] of [['flush', m => m.flush(100)], ['status', m => m.status()]]) { + const { c, f } = await fresh(); + c.advance(70000); assert.equal(renewals(f), 0); + await call(f.manager); await settle(f); + assert.equal(renewals(f), 1, name); + await f.manager.shutdown(100); + } + // Worker wake: an upload admitted before the window finishes after it; its job end wakes the pump. + { let unblock; const block = new Promise(resolve => { unblock = resolve; }); + const { c, f } = await fresh(base => ({ initialize: async (...a) => { await block; return base.initialize(...a); } })); + const h = f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array(4) }, options); + for (let i = 0; i < 5; i++) await tick(); + c.advance(70000); assert.equal(renewals(f), 0); + unblock(); assert.equal((await h.completion).status, 'validating'); + await settle(f); assert.equal(renewals(f), 1, 'worker wake'); + await f.manager.shutdown(100); } +}); + +// ---- 10.5 Completed import ------------------------------------------------------------------------------- +const providerResolved = { source: { kind: 'ProviderFile', connectionId: '80000000-0000-4000-8000-000000000001', registeredOutputId: '80000000-0000-4000-8000-000000000002' }, + declaredBytes: 172800044, declaredDurationMs: 3600000, mimeType: 'audio/wav', direction: 'mixed' }; +const resolver = value => ({ calls: 0, async resolve() { this.calls++; return typeof value === 'function' ? value() : value; } }); +test('T-IM-01 provider import: RecordingImportRequest body, key im-{rid}, original credentials; T-IM-09 no capability; T-IM-07 one status GET', async () => { + const f = await httpManager(); const s = span(); const r = resolver(providerResolved); + const h = await f.manager.importCompletedRecording(s, { provider: 'livekit_egress', providerRef: 'EG_synthetic1' }, { resolver: r }); + assert.equal((await h.completion).status, 'importing'); assert.equal(h.jobId, '60000000-0000-4000-8000-000000000001'); + const post = f.requests.find(x => x.url.endsWith('/imports')); const body = JSON.parse(post.body); + assert.equal(validAttachmentWire('RecordingImportRequest', body), true); + assert.equal(post.headers['idempotency-key'], `im-${hex(body.recording.recording_id)}`); assert.equal(post.headers.authorization, 'Bearer synthetic-original'); + assert.equal(body.recording.declared_bytes, 172800044); assert.equal(body.recording.source_kind, 'ProviderFile'); assert.equal(body.recording.deletion_generation, 0); + assert.equal(s.attrs['fi.attachment.refs'], '[]'); assert.equal(s.attrs['fi.attachment.capture_session_id'], f.manager.sessionId); + const before = f.requests.length; assert.deepEqual(await h.status(), { state: 'importing', reason: 'binding_pending' }); assert.equal(f.requests.length, before + 1); + assert.equal(f.requests.at(-1).headers.authorization, 'Bearer synthetic-original'); + // T-IM-08: no URL, path, header or credential in the body. + assert.equal(/https?:|\/tmp|synthetic-original|credential/.test(post.body), false); + await f.manager.shutdown(); +}); +test('T-IM-02 denied preflight never calls the resolver; T-IM-03 timeout or throw -> source_unavailable', async () => { + const f = await managerV1({}, {}, { traceConfig: { hideOutputs: true } }); const r = resolver(providerResolved); + assert.equal((await f.manager.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_1' }, { resolver: r })).reason, 'policy_denied'); assert.equal(r.calls, 0); + const g = await managerV1(); + const slow = { resolve: () => new Promise(() => {}) }; + const start = Date.now(); const h = await g.manager.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_1' }, { resolver: slow, timeoutMs: 200 }); + assert.equal(h.reason, 'source_unavailable'); assert.ok(Date.now() - start < 1000); + const thrower = { resolve: async () => { throw Error('synthetic-secret-detail'); } }; + const t = await g.manager.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_1' }, { resolver: thrower }); + assert.equal(t.reason, 'source_unavailable'); assert.equal(JSON.stringify(t).includes('synthetic-secret-detail'), false); + await f.manager.shutdown(1); await g.manager.shutdown(1); +}); +test('T-IM-04 out-of-range size or duration -> too_large before submit; T-IM-05 narrowing mixed -> policy_denied; T-IM-06 envelope', async () => { + const f = await managerV1(); + for (const bad of [{ declaredBytes: 1073741825 }, { declaredDurationMs: 7200001 }]) { + assert.equal((await f.manager.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_1' }, { resolver: resolver({ ...providerResolved, ...bad }) })).reason, 'too_large'); + } + assert.equal((await f.manager.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_1', direction: 'mixed' }, { resolver: resolver({ ...providerResolved, direction: 'output' }) })).reason, 'policy_denied'); + const widened = await f.manager.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_1', direction: 'output' }, { resolver: resolver({ ...providerResolved, direction: 'mixed' }) }); + assert.equal((await widened.completion).status, 'importing'); assert.equal(f.imports.at(-1).body.recording.direction, 'mixed'); + const envelope = await f.manager.importCompletedRecording(identity, { provider: 'trusted_importer', providerRef: 'ENV_1' }, { resolver: resolver({ ...providerResolved, source: { kind: 'RemoteUrlEnvelope', envelopeId: '80000000-0000-4000-8000-000000000003' } }) }); + assert.equal((await envelope.completion).status, 'importing'); + assert.deepEqual(f.imports.at(-1).body.source, { kind: 'RemoteUrlEnvelope', envelope_id: '80000000-0000-4000-8000-000000000003' }); assert.equal(f.imports.at(-1).body.recording.source_kind, 'RemoteUrl'); + assert.equal(f.calls.filter(c => c.op === 'submitImport').length, 2); + await f.manager.shutdown(); +}); +test('T-IM-10 429 with Retry-After 1 three times -> throttled within the deadline', async () => { + const f = await httpManager({ imports: () => ({ status: 429, retryAfter: '1' }) }); + const start = Date.now(); + const h = await f.manager.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_1' }, { resolver: resolver(providerResolved) }); + assert.equal((await h.completion).status, 'throttled'); assert.equal(f.requests.filter(r => r.url.endsWith('/imports')).length, 3); + assert.ok(Date.now() - start < 5000); + await f.manager.shutdown(); +}); + +// ---- 10.6 Consumed streams ------------------------------------------------------------------------------- +test('T-CS-01 app-owned reader: chunks observed after the app reads; no extra reads, no clone or tee', async () => { + const f = await managerV1(); const data = [new Uint8Array([1, 2]), new Uint8Array([3, 4, 5])]; + let pulls = 0; + const body = new ReadableStream({ pull(controller) { pulls++; const next = data.shift(); if (next) controller.enqueue(next); else controller.close(); } }); + const stream = f.manager.startAttachmentStream(identity, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 64 }); + const reader = body.getReader(); // the APPLICATION obtained the reader + const read = reader.read.bind(reader); let appReads = 0; + reader.read = async () => { appReads++; const r = await read(); if (!r.done) stream.tryWrite(r.value); else stream.finish(); return r; }; + while (!(await reader.read()).done) { /* app consumes */ } + assert.ok(pulls <= appReads + 1); + const h = stream.finish(); assert.equal((await h.completion).status, 'validating'); + assert.deepEqual([...f.contents.get(h.occurrenceId)], [1, 2, 3, 4, 5]); + await f.manager.shutdown(); +}); +test('T-CS-02 over the cap: all discarded, too_large, nothing uploaded; T-CS-03 app cancel -> incomplete, budget released', async () => { + const f = await managerV1(); const s = span(); + const stream = f.manager.startAttachmentStream(s, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 4 }); + assert.equal(stream.tryWrite(new Uint8Array(3)), true); assert.equal(stream.tryWrite(new Uint8Array(3)), false); + assert.equal(stream.reason, 'too_large'); assert.equal((await stream.finish().completion).status, 'too_large'); + assert.equal(s.attrs['fi.attachment.refs'], undefined, 'an over-cap stream projects nothing'); + const other = f.manager.startAttachmentStream(identity, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 64 }); + other.tryWrite(new Uint8Array(8)); other.abort(); + assert.equal(other.reason, 'incomplete'); assert.equal(f.manager.status().ownedBytes, 0); assert.equal(f.manager.status().openStreams, 0); + await f.manager.flush(); assert.equal(f.ops().includes('initialize'), false); + await f.manager.shutdown(); +}); +test('T-CS-04 abandoned stream aborted at the next entry point after idle TTL (fake clock, no timer)', async () => { + const c = clock(); const f = await managerV1({}, { clock: c }); + const stream = f.manager.startAttachmentStream(identity, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 64, idleTtlMs: 1000 }); + stream.tryWrite(new Uint8Array(4)); c.advance(1001); + assert.equal(f.manager.status().openStreams, 0); assert.equal(stream.reason, 'incomplete'); assert.equal(f.manager.status().ownedBytes, 0); + await f.manager.shutdown(1); +}); +test('T-CS-06 BYOB buffer reused after tryWrite: uploaded bytes equal the bytes at write time', async () => { + const f = await managerV1(); + const stream = f.manager.startAttachmentStream(identity, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 64 }); + const buffer = new Uint8Array([9, 9, 9]); stream.tryWrite(buffer); buffer.fill(0); stream.tryWrite(buffer.subarray(0, 1)); + const h = stream.finish(); assert.equal((await h.completion).status, 'validating'); + assert.deepEqual([...f.contents.get(h.occurrenceId)], [9, 9, 9, 0]); + await f.manager.shutdown(); +}); + +// ---- 10.7 Channel and role ----------------------------------------------------------------------------------- +test('T-CH-01 UploadRequest never carries role or channel; T-CH-02 the live reference does', async () => { + const f = await managerV1(); const s = span(); + const h = f.manager.attach(s, { kind: 'Bytes', data: new Uint8Array([1]) }, { mimeType: 'audio/wav', direction: 'output', role: 'assistant', channel: 1, messageIndex: 2 }); + await h.completion; + assert.equal('role' in f.requests[0] || 'channel' in f.requests[0], false); assert.equal(validAttachmentWire('UploadRequest', f.requests[0]), true); + const refs = JSON.parse(s.attrs['fi.attachment.refs']); assert.equal(refs[0].role, 'assistant'); assert.equal(refs[0].channel, 1); + assert.equal(validAttachmentWire('SpanAttributes', s.attrs), true); + await f.manager.shutdown(); +}); +test('T-CH-03 span ended before attach: no projection, the upload still proceeds; T-CH-05 carrier stamp only on admitted targets', async () => { + const f = await managerV1(); const ended = span(); + f.manager.correlation(ended); // snapshot taken while the span was live (adapters bind at span start) + ended.end(); + const h = f.manager.attach(ended, { kind: 'Bytes', data: new Uint8Array([1]) }, options); + assert.equal((await h.completion).status, 'validating'); assert.equal(ended.writes, 0); + const never = span(); never.end(); // never observed live: not recording at admission (design 1.4 step 10) + assert.equal(f.manager.attach(never, { kind: 'Bytes', data: new Uint8Array([1]) }, options).admission, 'unsampled'); + const bystander = span(); f.manager.correlation(bystander); assert.equal(bystander.writes, 0, 'non-target spans are never stamped'); + const denied = span(); f.manager.attach(denied, { kind: 'Bytes', data: new Uint8Array([1]) }, { ...options, capture: { captureContent: false } }); + assert.equal(denied.writes, 0, 'a denied capture stamps nothing'); + const aborted = span(); const st = f.manager.startAttachmentStream(aborted, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 8 }); st.tryWrite(new Uint8Array(1)); st.abort(); + assert.equal(aborted.writes, 0, 'an aborted stream projects nothing'); + for (const make of [s => f.manager.attach(s, { kind: 'Bytes', data: new Uint8Array([1]) }, options), + s => { const x = f.manager.startAttachmentStream(s, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 8 }); x.tryWrite(new Uint8Array(1)); x.finish(); }, + s => f.manager.startRecording(s, mono(16000)).abort()]) { + const target = span(); make(target); + assert.equal(validAttachmentWire('SpanAttributes', target.attrs), true, JSON.stringify(target.attrs)); + } + await f.manager.flush(); await f.manager.shutdown(); +}); +test('T-CH-06 explicit workspace with a different response scope -> scope_unresolved', async () => { + const f = await managerV1({}, { ready: false }, { workspaceId: '90000000-0000-4000-8000-000000000001' }); + assert.equal(await f.manager.ready(), false); assert.equal(f.manager.status().state, 'scope_unresolved'); + assert.equal(f.manager.attach(identity, source(), { mimeType: 'audio/wav', direction: 'mixed' }).admission, 'scope_unresolved'); + await f.manager.shutdown(1); +}); + +// ---- 10.8 Lifecycle ---------------------------------------------------------------------------------------- +test('T-LC-02/03/04/05 disabled null object: no timers, sockets, listeners, finalizers or span writes; every call immediate', async () => { + const counts = { timers: 0, intervals: 0, registry: 0 }; + const saved = { st: global.setTimeout, si: global.setInterval, fr: global.FinalizationRegistry }; + global.setTimeout = (...a) => { counts.timers++; return saved.st(...a); }; + global.setInterval = (...a) => { counts.intervals++; return saved.si(...a); }; + global.FinalizationRegistry = class extends saved.fr { constructor(...a) { counts.registry++; super(...a); } }; + const listenersBefore = process.eventNames().map(n => process.listenerCount(n)).reduce((a, b) => a + b, 0); + try { + const { createNodeAttachmentManager } = require('./attachments-v1.cjs').nodeCore; + const m = createNodeAttachmentManager({ environment: () => ({}) }); + assert.equal(m.constructor.name, 'DisabledAttachmentManager'); + const s = span(); const start = process.hrtime.bigint(); + assert.equal(await m.ready(), false); assert.equal(m.status().state, 'disabled'); + let touched = false; + const handle = m.attach(s, { kind: 'Bytes', get data() { touched = true; return new Uint8Array(1); } }, options); + assert.equal(handle.admission, 'disabled'); assert.deepEqual(await handle.completion, { status: 'disabled' }); + assert.equal(m.preflight(s, { ...options, maxBytes: 1 }).reason, 'disabled'); + const w = m.startRecording(s, mono(16000)); assert.equal(w.tryWrite(pcm(10)), false); assert.equal((await w.finish()).state, 'denied'); + const st = m.startAttachmentStream(s, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 8 }); assert.equal(st.tryWrite(new Uint8Array(1)), false); + let resolved = false; + assert.equal((await m.importCompletedRecording(s, { provider: 'x', providerRef: 'y' }, { resolver: { resolve: async () => { resolved = true; return 'source_unavailable'; } } })).reason, 'disabled'); + assert.equal(m.attachOwnedRecording(s, { kind: 'OwnedRecordingFile', lease: {} }, { mimeType: 'audio/wav', direction: 'mixed' }).reason, 'disabled'); + m.updatePolicy({ enabled: true }); assert.equal(m.status().state, 'disabled'); + assert.deepEqual(await m.flush(), { unfinished: 0, dropped: 0 }); assert.deepEqual(await m.shutdown(), { unfinished: 0, dropped: 0 }); + const elapsed = Number(process.hrtime.bigint() - start) / 1e6; + assert.equal(touched, false); assert.equal(resolved, false); assert.equal(s.writes, 0); + assert.ok(elapsed < 50, `disabled calls took ${elapsed} ms`); + } finally { global.setTimeout = saved.st; global.setInterval = saved.si; global.FinalizationRegistry = saved.fr; } + const listenersAfter = process.eventNames().map(n => process.listenerCount(n)).reduce((a, b) => a + b, 0); + assert.deepEqual(counts, { timers: 0, intervals: 0, registry: 0 }); assert.equal(listenersAfter, listenersBefore); +}); +test('T-LC-12 status().state is "ready" after bootstrap', async () => { + const f = await managerV1(); assert.equal(f.manager.status().state, 'ready'); await f.manager.shutdown(); +}); +test('T-LC-15 shutdown is idempotent and concurrent; T-LC-16 returns within timeout + 250 ms with a transport that ignores abort', async () => { + const f = await managerV1(() => ({ putContent: () => new Promise(() => {}) })); + f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array(4) }, options); + await tick(); await tick(); + const start = Date.now(); const a = f.manager.shutdown(500); const b = f.manager.shutdown(500); + assert.equal(a, b); const result = await a; const elapsed = Date.now() - start; + assert.ok(elapsed <= 750, `shutdown took ${elapsed} ms`); assert.ok(result.unfinished >= 1); +}); + +// ---- 10.9 Transport and wire ------------------------------------------------------------------------------- +test('T-TR-02 status -> reason table, Error.code refinement only within class', () => { + const body = code => Buffer.from(JSON.stringify({ code, request_id: '10000000-0000-4000-8000-000000000009' })); + const rows = [[400, 'unsupported_version', false], [401, 'credential_scope_unsupported', false], [403, 'policy_denied', false], [404, 'not_found', false], + [409, 'conflict', false], [410, 'expired', false], [413, 'too_large', false], [429, 'throttled', true], [408, 'incomplete', true], [500, 'incomplete', true], + [502, 'incomplete', true], [503, 'incomplete', true], [504, 'incomplete', true], [418, 'incomplete', false], [302, 'incomplete', false]]; + for (const [status, code, retryable] of rows) { const e = attachmentHttpError(status); assert.equal(e.code, code, String(status)); assert.equal(e.retryable, retryable, String(status)); } + assert.equal(attachmentHttpError(409, body('deleted')).code, 'deleted'); assert.equal(attachmentHttpError(403, body('redaction_unavailable')).code, 'redaction_unavailable'); + assert.equal(attachmentHttpError(401, body('expired')).code, 'expired'); assert.equal(attachmentHttpError(404, body('scope_unresolved')).code, 'scope_unresolved'); + assert.equal(attachmentHttpError(403, body('deleted')).code, 'policy_denied', 'out-of-class code ignored'); + assert.equal(attachmentHttpError(500, body('deleted')).code, 'incomplete'); + assert.equal(attachmentHttpError(409, Buffer.from('{"code":"deleted","code":"conflict","request_id":"10000000-0000-4000-8000-000000000009"}')).code, 'conflict', 'duplicate members ignored'); + assert.equal(attachmentHttpError(429, undefined, '3').retryAfterMs, 3000); assert.equal(attachmentHttpError(429, undefined, '60').retryAfterMs, 5000); + assert.equal(attachmentHttpError(429, undefined, 'Wed, 21 Oct 2015 07:28:00 GMT').retryAfterMs, 1000); assert.equal(attachmentHttpError(429).retryAfterMs, 1000); +}); +test('T-TR-03 idempotency keys on every route and byte-identical replays', async () => { + let inits = 0; + const f = await httpManager({ upload: entry => (++inits === 1 ? { status: 503 } : undefined), content: () => ({ status: 409 }) }); + const h = f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, options); + assert.equal((await h.completion).status, 'conflict'); + const uploads = f.requests.filter(r => r.url.endsWith('/uploads')); + assert.equal(uploads.length, 2); assert.equal(uploads[0].body, uploads[1].body); assert.equal(uploads[0].headers['idempotency-key'], `up-${hex(h.occurrenceId)}`); + const abort = f.requests.find(r => r.url.endsWith('/abort')); + assert.equal(abort.headers['idempotency-key'], `ab-${hex('50000000-0000-4000-8000-000000000001')}`); assert.match(abort.headers.authorization, /^Bearer synthetic-capability-/); + assert.equal(attachmentIdempotencyKey('complete', '50000000-0000-4000-8000-000000000001'), 'cp-50000000000040008000000000000001'); + const g = await httpManager({}, {}); + await g.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, options).completion; + const complete = g.requests.find(r => r.url.endsWith('/complete')); + assert.equal(complete.headers['idempotency-key'], `cp-${hex('50000000-0000-4000-8000-000000000001')}`); assert.deepEqual(JSON.parse(complete.body), { policy_version: 'v1', deletion_generation: 0 }); + const w = g.manager.startRecording(identity, mono(8000)); w.tryWrite(pcm(8000)); await w.finish(5000); + const manifest = g.requests.find(r => r.url.endsWith('/manifest')); + assert.equal(manifest.method, 'PUT'); assert.equal(manifest.headers['idempotency-key'], `mf-${hex(w.recordingId)}-1`); assert.match(manifest.headers.authorization, /^Bearer synthetic-capability-/); + assert.equal(manifest.headers['x-api-key'], undefined, 'capability routes never carry original credentials'); + await f.manager.shutdown(); await g.manager.shutdown(); +}); +test('T-TR-05 content PUT pacing honours upload_bytes_per_second', async () => { + const d = httpDouble(entry => standardRoutes(entry, { session: e => ({ status: 200, body: sessionBody(JSON.parse(e.body).capture_session_id, { limits: { ...limits, upload_bytes_per_second: 400000 } }) }) })); + const transport = createAttachmentHttpTransportV1({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic-original' }, client: async request => { + if (request.pacer && request.body && typeof request.body !== 'string') { const size = request.body.byteLength ?? request.body.size; for (let o = 0; o < size; o += 262144) await request.pacer.take(Math.min(262144, size - o), request.signal); } + return d.client(request); + } }); + const m = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport }); + await m.ready(); const start = Date.now(); + assert.equal((await m.attach(identity, { kind: 'Bytes', data: new Uint8Array(800000) }, { mimeType: 'audio/mpeg', direction: 'output' }).completion).status, 'validating'); + const elapsed = Date.now() - start; assert.ok(elapsed >= 700, `paced upload took ${elapsed} ms`); + const put = d.requests.find(r => r.url.endsWith('/content')); assert.equal(put.body.length, 800000); + await m.shutdown(); +}); +test('T-TR-06 oversized or duplicate-member responses -> unsupported_version; T-TR-07 missing capability rejected; T-TR-08 validating, never ready', async () => { + const big = httpDouble(entry => (entry.url.endsWith('/attachment-sessions') ? { status: 200, body: 'x'.repeat(70000) } : { status: 404 })); + const t1 = createAttachmentHttpTransportV1({ endpoint: 'https://attachments.example', headers: {}, client: big.client }); + await assert.rejects(t1.bootstrap({ capture_session_id: ids[0] }, 'bs-x-0', new AbortController().signal), e => e.code === 'unsupported_version'); + const dup = httpDouble(() => ({ status: 200, body: '{"schema_version":1,"schema_version":1}' })); + const t2 = createAttachmentHttpTransportV1({ endpoint: 'https://attachments.example', headers: {}, client: dup.client }); + await assert.rejects(t2.bootstrap({ capture_session_id: ids[0] }, 'bs-x-0', new AbortController().signal), e => e.code === 'unsupported_version'); + const f = await httpManager({ upload: entry => { const r = JSON.parse(entry.body); return { status: 201, body: { upload_id: '50000000-0000-4000-8000-000000000001', attachment_id: r.attachment_id, + occurrence_id: r.occurrence_id, state: 'pending_upload', reserved_bytes: r.declared_bytes, expires_at: new Date(Date.now() + 600000).toISOString(), transfer_mode: 'authenticated_stream' } }; } }); + assert.equal((await f.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, options).completion).status, 'unsupported_version'); + assert.equal(f.requests.some(r => r.url.endsWith('/content')), false); + const g = await httpManager(); + assert.equal((await g.manager.attach(identity, { kind: 'Bytes', data: new Uint8Array([1]) }, options).completion).status, 'validating'); + await f.manager.shutdown(); await g.manager.shutdown(); +}); +test('legacy four-method transports keep working through the adapter (stacked lanes)', async () => { + const { fake } = require('./attachments-fixtures.cjs'); const f = fake(); + const m = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: f.transport }); + assert.equal(await m.ready(), true); assert.equal(m.status().state, 'ready'); + assert.equal((await m.attach(identity, source(), { mimeType: 'audio/wav', direction: 'mixed' }).completion).status, 'validating'); + assert.deepEqual(f.calls, ['bootstrap', 'initialize', 'upload', 'complete']); + // Legacy transports cannot PUT manifests: the recording is denied at start (loud), not buffered and uploaded unbindable. + const w = m.startRecording(identity, mono(8000)); + assert.equal(w.reason, 'unsupported_version'); assert.equal(w.tryWrite(pcm(800)), false); + const r = await w.finish(2000); + assert.equal(r.state, 'denied'); assert.equal(r.segmentsPublished, 0); + assert.deepEqual(f.calls, ['bootstrap', 'initialize', 'upload', 'complete'], 'no segment was uploaded'); + assert.equal(m.status().ownedBytes, 0); + await m.shutdown(); +}); +test('T-ST-16 BR-7 fallback: capability rejected after complete -> init, content, manifest, complete for the remaining segments', async () => { + const f = await managerV1(base => ({ putManifest: async (s, capability, rid, body, key, signal) => { + if ([...f.completed.values()].includes(capability)) throw new AttachmentTransportError('policy_denied', false, 0, 403); + return base.putManifest(s, capability, rid, body, key, signal); + } }), { limits: { segment_duration_ms: 1000 } }); + const w = f.manager.startRecording(identity, mono(8000)); + await writeSeconds(f.manager, w, 8000, 8, { flushEvery: 1 }); + const result = await w.finish(10000); + assert.ok(f.manifests.length >= 2, `manifests ${f.manifests.length}`); + for (const m of f.manifests) { + const top = m.body.segments.at(-1); + assert.equal(f.completed.has(top.occurrence_id), true); + const order = f.calls.map(c => c.op + ':' + (c.capability || '')); + assert.ok(order.indexOf('putManifest:' + m.capability) < order.indexOf('complete:' + m.capability), 'manifest must precede that segment complete'); + } + assert.equal(f.manifests.at(-1).body.capture_complete, true); assert.equal(result.state, 'published'); + for (let i = 1; i < f.manifests.length; i++) assert.ok(f.manifests[i].body.segments.length >= f.manifests[i - 1].body.segments.length); + await f.manager.shutdown(); + // Fallback also rejected: nothing published. + const g = await managerV1(() => ({ putManifest: async () => { throw new AttachmentTransportError('policy_denied', false, 0, 403); } }), { limits: { segment_duration_ms: 1000 } }); + const w2 = g.manager.startRecording(identity, mono(8000)); + await writeSeconds(g.manager, w2, 8000, 3, { flushEvery: 1 }); + assert.equal((await w2.finish(10000)).state, 'unpublished'); + await g.manager.shutdown(); +}); +test('T-CS-05 garbage-collected abandoned stream and ticket release their budget (best effort, forced GC)', () => { + const { spawnSync } = require('node:child_process'); + const script = ` + const { managerV1 } = require(${JSON.stringify(require.resolve('./attachments-v1.cjs'))}); + const identity = { traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }; + (async () => { + const f = await managerV1(); + (() => { const s = f.manager.startAttachmentStream(identity, { mimeType: 'audio/mpeg', direction: 'output', maxBytes: 4096 }); s.tryWrite(new Uint8Array(10)); + f.manager.preflight(identity, { mimeType: 'image/png', direction: 'input', maxBytes: 2048 }); })(); + for (let i = 0; i < 50 && f.manager.status().ownedBytes; i++) { global.gc(); await new Promise(r => setTimeout(r, 20)); } + const st = f.manager.status(); console.log(JSON.stringify({ owned: st.ownedBytes, streams: st.openStreams, tickets: st.openTickets })); + await f.manager.shutdown(100); + })();`; + const r = spawnSync(process.execPath, ['--expose-gc', '-e', script], { env: process.env, encoding: 'utf8', timeout: 20000 }); + assert.equal(r.status, 0, r.stderr); + assert.deepEqual(JSON.parse(r.stdout.trim().split('\n').pop()), { owned: 0, streams: 0, tickets: 0 }); +}); diff --git a/typescript/packages/fi-core/test/attachments-exit-probe.cjs b/typescript/packages/fi-core/test/attachments-exit-probe.cjs new file mode 100644 index 000000000..ead56e30e --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-exit-probe.cjs @@ -0,0 +1,49 @@ +// One exit-probe case per process (design 8.5). Prints DONE and returns; the process must then exit by itself. +const path = require('node:path'); +const { createRequire } = require('node:module'); +const load = createRequire(process.env.FI_CORE_PACKAGE_JSON || path.resolve(__dirname, '../package.json')); +const { register, ProjectType } = load('@traceai/fi-core'); +const { v1, pcm } = require('./attachments-v1.cjs'); +const identity = { traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }; +const blocked = () => new Promise(() => {}); // ignores its abort signal entirely +const provider = (transport, extra = {}) => register({ projectName: 'probe', projectType: ProjectType.OBSERVE, setGlobalTracerProvider: false, spanProcessors: [], + attachments: { enabled: true, environment: () => ({}), recordAudio: true, transport, ...extra } }); +(async () => { + const name = process.argv[2]; + if (name === 'disabled') { + const p = register({ projectName: 'probe', projectType: ProjectType.OBSERVE, setGlobalTracerProvider: false, spanProcessors: [] }); + p.getTracer('probe').startSpan('s').end(); + await p.shutdown(); console.log(`DONE ${p.attachments.status().state}`); return; + } + if (name === 'enabled-idle') { const p = provider(v1().transport); await p.attachments.ready(); await p.shutdown(); console.log('DONE'); return; } + if (name === 'enabled-blocked') { + const p = provider(v1({ putContent: blocked }).transport, { shutdownTimeoutMs: 1000 }); + await p.attachments.ready(); p.attachments.attach(identity, { kind: 'Bytes', data: new Uint8Array(8) }, { mimeType: 'audio/wav', direction: 'mixed' }); + await new Promise(resolve => setImmediate(resolve)); + const result = await p.attachments.shutdown(1000); await p.shutdown(); + if (result.unfinished < 1) throw Error('expected unfinished'); + console.log(`DONE unfinished=${result.unfinished}`); return; + } + if (name === 'enabled-no-shutdown') { + const p = provider(v1({ putContent: blocked }).transport); await p.attachments.ready(); + p.attachments.attach(identity, { kind: 'Bytes', data: new Uint8Array(8) }, { mimeType: 'audio/wav', direction: 'mixed' }); + console.log('DONE'); return; // no shutdown: unref'd handles must not hold the process + } + if (name === 'mid-recording') { + const d = v1(); const p = provider(d.transport, { shutdownTimeoutMs: 10000 }); await p.attachments.ready(); + const w = p.attachments.startRecording(identity, { format: { sampleRateHz: 16000 }, layout: 'mixed', tracks: [{ channels: [0], direction: 'mixed', role: 'mixed' }], operation: 'observe_consumed' }); + for (let i = 0; i < 250; i++) w.tryWrite(pcm(320)); + await p.shutdown(); + const result = await w.finish(); + if (!(d.manifests.length >= 1 || result.state === 'partial')) throw Error(`no manifest: ${result.state}`); + console.log(`DONE ${result.state} manifests=${d.manifests.length}`); return; + } + if (name === 'import-pending') { + const p = provider(v1().transport); await p.attachments.ready(); + const h = await p.attachments.importCompletedRecording(identity, { provider: 'livekit_egress', providerRef: 'EG_probe' }, { resolver: { resolve: () => new Promise(() => {}) }, timeoutMs: 1000 }); + await p.shutdown(); + if (h.reason !== 'source_unavailable') throw Error(h.reason); + console.log(`DONE ${h.reason}`); return; + } + throw Error(`unknown case ${name}`); +})().catch(error => { console.error(String(error && error.message)); process.exitCode = 1; }); diff --git a/typescript/packages/fi-core/test/attachments-exit.test.cjs b/typescript/packages/fi-core/test/attachments-exit.test.cjs new file mode 100644 index 000000000..5e1f2ccf0 --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-exit.test.cjs @@ -0,0 +1,17 @@ +/* Process-exit regression probes (design 8.5, T-LC-06..11). Each case runs in a separate Node process. */ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const packageJson = process.env.FI_CORE_PACKAGE_JSON || path.resolve(__dirname, '../package.json'); +const probe = path.join(__dirname, 'attachments-exit-probe.cjs'); +for (const [name, limit] of [['disabled', 10000], ['enabled-idle', 10000], ['enabled-blocked', 10000], ['enabled-no-shutdown', 10000], ['mid-recording', 15000], ['import-pending', 10000]]) { + test(`exit probe ${name} exits 0 under ${limit} ms`, () => { + const start = Date.now(); + const result = spawnSync(process.execPath, [probe, name], { env: { ...process.env, FI_CORE_PACKAGE_JSON: packageJson }, timeout: limit, encoding: 'utf8' }); + const elapsed = Date.now() - start; + assert.equal(result.signal, null, `${name} was killed (kept alive): ${result.stderr}`); + assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /DONE/); assert.ok(elapsed < limit); + console.log(`# probe ${name} ${elapsed} ms ${result.stdout.trim().split('\n').pop()}`); + }); +} diff --git a/typescript/packages/fi-core/test/attachments-files.test.cjs b/typescript/packages/fi-core/test/attachments-files.test.cjs new file mode 100644 index 000000000..2b7f0c4ee --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-files.test.cjs @@ -0,0 +1,138 @@ +/* Owned recording files (design section 4, T-LS-*). Synthetic files under a private temp root only. */ +const { test, after } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { nodeCore, core, managerV1, readAll } = require('./attachments-v1.cjs'); +const { identity } = require('./attachments-fixtures.cjs'); +const { createRecordingFileScope } = nodeCore; +const { attachmentWavHeader, validAttachmentManifest, AttachmentTransportError } = core; +const live = setInterval(() => {}, 1000); +after(() => clearInterval(live)); +const MiB = 1024 * 1024; +function root() { return fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'owned-rec-'))); } +function pattern(size, seed = 1) { const b = Buffer.alloc(size); for (let i = 0; i < size; i++) b[i] = (i * 13 + seed + (i >> 11)) & 0xff; return b; } +function wavFile(file, { rate = 24000, channels = 2, frames, bits = 16, format = 1, badLength = false }) { + const data = pattern(frames * channels * (bits / 8), 7); + const header = Buffer.from(attachmentWavHeader(channels, rate, badLength ? data.length + 100 : data.length)); + header.writeUInt16LE(format, 20); header.writeUInt16LE(bits, 34); header.writeUInt16LE(channels * bits / 8, 32); + fs.writeFileSync(file, Buffer.concat([header, data])); + return data; +} +const stereo = [{ channels: [0], direction: 'input', role: 'user' }, { channels: [1], direction: 'output', role: 'assistant' }]; + +test('T-LS-02 symlink leaf and T-LS-03 symlinked intermediate are source_unavailable; foreign refs fail closed', async () => { + const dir = root(); fs.writeFileSync(path.join(dir, 'real.ogg'), pattern(100)); + fs.symlinkSync(path.join(dir, 'real.ogg'), path.join(dir, 'link.ogg')); + fs.mkdirSync(path.join(dir, 'real-dir')); fs.writeFileSync(path.join(dir, 'real-dir', 'a.ogg'), pattern(100)); fs.symlinkSync(path.join(dir, 'real-dir'), path.join(dir, 'dir-link')); + const scope = await createRecordingFileScope(dir, { enabled: true }); + await assert.rejects(scope.identify('link.ogg'), e => e.code === 'source_unavailable'); + await assert.rejects(scope.identify('dir-link/a.ogg'), e => e.code === 'source_unavailable'); + await assert.rejects(scope.identify('../etc/passwd'), e => e.code === 'source_unavailable'); + await assert.rejects(createRecordingFileScope(path.join(dir, 'dir-link'), { enabled: true }), e => e.code === 'source_unavailable'); + const f = await managerV1(); + const h = f.manager.attachOwnedRecording(identity, { kind: 'OwnedRecordingFile', lease: {} }, { mimeType: 'audio/ogg', direction: 'mixed' }); + assert.equal(h.reason, 'source_unavailable'); assert.equal(h.tier, 'denied'); + await scope.close(); await f.manager.shutdown(); +}); +test('T-LS-06 12 MiB OGG: tier single, streamed from the lease, owned memory never above one slice; T-CH-04 mixed, no channel', async () => { + const dir = root(); const data = pattern(12 * MiB); fs.writeFileSync(path.join(dir, 'call.ogg'), data); + let peak = 0; let manager; + const f = await managerV1(base => ({ putContent: async (s, u, src, sig) => { + const out = Buffer.alloc(src.size); + for (let o = 0; o < src.size; o += 262144) { await src.check(); const p = await src.read(o, Math.min(262144, src.size - o)); Buffer.from(p).copy(out, o); peak = Math.max(peak, manager.status().ownedBytes); } + f.contents.set(u.occurrence_id, out); + } })); + manager = f.manager; + const scope = await createRecordingFileScope(dir, { enabled: true }); + const id = await scope.identify('call.ogg'); assert.equal(id.size, 12 * MiB); + const h = f.manager.attachOwnedRecording(identity, scope.source('call.ogg', id), { mimeType: 'audio/ogg', direction: 'mixed' }); + assert.equal(h.tier, 'single'); assert.equal((await h.wait(30000)).status, 'validating'); + assert.ok(peak <= 262144, `owned peak ${peak}`); + assert.ok(Buffer.from(f.contents.values().next().value).equals(data)); + const request = f.requests[0]; + assert.equal(request.direction, 'mixed'); assert.equal(request.source_kind, 'OwnedRecordingFile'); assert.equal('channel' in request || 'role' in request, false); + assert.equal(request.recording_id, undefined); + await scope.close(); await f.manager.shutdown(); +}); +test('T-LS-07 70 MiB OGG: too_large with zero bytes read', async () => { + const dir = root(); const file = path.join(dir, 'big.ogg'); const fd = fs.openSync(file, 'w'); fs.ftruncateSync(fd, 70 * MiB); fs.closeSync(fd); + const f = await managerV1(); + const scope = await createRecordingFileScope(dir, { enabled: true }); + const h = f.manager.attachOwnedRecording(identity, scope.source('big.ogg', await scope.identify('big.ogg')), { mimeType: 'audio/ogg', direction: 'mixed' }); + assert.equal(h.tier, 'denied'); assert.equal(h.reason, 'too_large'); await f.manager.flush(); + assert.equal(f.ops().includes('initialize'), false); + await scope.close(); await f.manager.shutdown(); +}); +test('T-LS-08 90 MiB 24 kHz stereo WAV with wavTracks: segmented from lease ranges, stereo manifest, bytes identical', async () => { + const dir = root(); const frames = Math.floor(90 * MiB / 4); + const data = wavFile(path.join(dir, 'call.wav'), { rate: 24000, channels: 2, frames }); + const f = await managerV1(); + const scope = await createRecordingFileScope(dir, { enabled: true }); + const h = f.manager.attachOwnedRecording(identity, scope.source('call.wav', await scope.identify('call.wav')), { mimeType: 'audio/wav', direction: 'mixed', wavTracks: stereo }); + assert.equal(h.tier, 'wav_recording'); + assert.equal((await h.wait(30000)).status, 'validating'); assert.ok(h.recordingId); + const m = f.manifests.at(-1).body; + assert.equal(m.layout, 'stereo'); assert.deepEqual(m.tracks.map(t => [t.channels, t.role]), [[[0], 'user'], [[1], 'assistant']]); + assert.equal(m.capture_complete, true); assert.equal(m.duration_ms, Math.ceil(frames * 1000 / 24000)); assert.equal(validAttachmentManifest(m), true); + const joined = Buffer.concat(m.segments.map(s => f.contents.get(s.occurrence_id).subarray(44))); + assert.ok(joined.equals(data), 'decoded segment frames must equal the source frames'); + assert.ok(f.requests.every(r => r.declared_bytes <= 8 * 1024 * 1024 && r.recording_id === h.recordingId)); + await scope.close(); await f.manager.shutdown(); +}); +test('T-LS-09 bad data length, non-PCM and 24-bit WAV: single when within the cap, too_large above it', async () => { + const dir = root(); + wavFile(path.join(dir, 'bad.wav'), { frames: 1000, badLength: true }); + wavFile(path.join(dir, 'float.wav'), { frames: 1000, format: 3 }); + wavFile(path.join(dir, 'b24.wav'), { frames: 1000, bits: 24 }); + const f = await managerV1(); + const scope = await createRecordingFileScope(dir, { enabled: true }); + for (const name of ['bad.wav', 'float.wav', 'b24.wav']) { + const h = f.manager.attachOwnedRecording(identity, scope.source(name, await scope.identify(name)), { mimeType: 'audio/wav', direction: 'mixed', wavTracks: stereo }); + assert.equal((await h.wait(10000)).status, 'validating', name); assert.equal(h.tier, 'single', name); + } + const big = path.join(dir, 'big.wav'); fs.writeFileSync(big, Buffer.from(attachmentWavHeader(2, 24000, 100))); const fd = fs.openSync(big, 'r+'); fs.ftruncateSync(fd, 60 * MiB); fs.closeSync(fd); + const h = f.manager.attachOwnedRecording(identity, scope.source('big.wav', await scope.identify('big.wav')), { mimeType: 'audio/wav', direction: 'mixed', wavTracks: stereo }); + assert.equal((await h.wait(10000)).status, 'too_large'); assert.equal(h.tier, 'denied'); + await scope.close(); await f.manager.shutdown(); +}); +test('T-LS-01 file changed during transfer -> source_unavailable and abort; T-LS-04 the app file position is unchanged', async () => { + const dir = root(); const file = path.join(dir, 'grow.ogg'); fs.writeFileSync(file, pattern(MiB)); + const appFd = fs.openSync(file, 'r'); const probe = Buffer.alloc(10); fs.readSync(appFd, probe, 0, 10, null); + const f = await managerV1(base => ({ putContent: async (s, u, src) => { + for (let o = 0; o < src.size; o += 262144) { await src.check(); await src.read(o, 262144); if (o === 0) fs.appendFileSync(file, 'x'); } + await src.check(); + } })); + const scope = await createRecordingFileScope(dir, { enabled: true }); + const h = f.manager.attachOwnedRecording(identity, scope.source('grow.ogg', await scope.identify('grow.ogg')), { mimeType: 'audio/ogg', direction: 'mixed' }); + assert.equal((await h.wait(10000)).status, 'source_unavailable'); + assert.equal(f.ops().filter(op => op === 'abort').length, 1); + const next = Buffer.alloc(1); fs.readSync(appFd, next, 0, 1, null); assert.equal(next[0], pattern(11)[10]); + fs.closeSync(appFd); await scope.close(); await f.manager.shutdown(); +}); +test('T-LS-05 intermediate directory swapped to a symlink after identify: identity bind fails closed (documented residual race)', async () => { + const dir = root(); fs.mkdirSync(path.join(dir, 'session')); fs.writeFileSync(path.join(dir, 'session', 'r.ogg'), pattern(1000)); + fs.mkdirSync(path.join(dir, 'other')); fs.writeFileSync(path.join(dir, 'other', 'r.ogg'), pattern(1000, 9)); + const scope = await createRecordingFileScope(dir, { enabled: true }); + const id = await scope.identify('session/r.ogg'); const src = scope.source('session/r.ogg', id); + fs.renameSync(path.join(dir, 'session'), path.join(dir, 'session-real')); fs.symlinkSync(path.join(dir, 'other'), path.join(dir, 'session')); + const f = await managerV1(); + const h = f.manager.attachOwnedRecording(identity, src, { mimeType: 'audio/ogg', direction: 'mixed' }); + assert.equal((await h.wait(10000)).status, 'source_unavailable'); assert.equal(f.ops().includes('initialize'), false); + // Same inode reached through a symlinked directory is still refused (lstat of every component). + fs.unlinkSync(path.join(dir, 'session')); fs.symlinkSync(path.join(dir, 'session-real'), path.join(dir, 'session')); + const h2 = f.manager.attachOwnedRecording(identity, src, { mimeType: 'audio/ogg', direction: 'mixed' }); + assert.equal((await h2.wait(10000)).status, 'source_unavailable'); + await scope.close(); await f.manager.shutdown(); +}); +test('T-LS-10 finalization deadline with a blocked transport: incomplete within 1.25 s; lease and scope closed', async () => { + const dir = root(); fs.writeFileSync(path.join(dir, 'r.ogg'), pattern(600000)); + const f = await managerV1(() => ({ putContent: (_s, _u, _src, signal) => new Promise((_, reject) => signal.addEventListener('abort', () => reject(new AttachmentTransportError('incomplete')), { once: true })) })); + const scope = await createRecordingFileScope(dir, { enabled: true }); + const h = f.manager.attachOwnedRecording(identity, scope.source('r.ogg', await scope.identify('r.ogg')), { mimeType: 'audio/ogg', direction: 'mixed' }); + const start = Date.now(); const result = await h.wait(1000); const elapsed = Date.now() - start; + assert.equal(result.status, 'incomplete'); assert.ok(elapsed <= 1250, `wait took ${elapsed} ms`); + const closing = Date.now(); await scope.close(); assert.ok(Date.now() - closing < 500); + await f.manager.shutdown(500); assert.equal(f.manager.status().ownedBytes, 0); +}); diff --git a/typescript/packages/fi-core/test/attachments-fixtures.cjs b/typescript/packages/fi-core/test/attachments-fixtures.cjs new file mode 100644 index 000000000..98666639b --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-fixtures.cjs @@ -0,0 +1,24 @@ +const identity = { traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }; +const ids = ['10000000-0000-4000-8000-000000000001', '10000000-0000-4000-8000-000000000002', '10000000-0000-4000-8000-000000000003']; +const policy = { version: 'v1', enabled: true, allowed_sources: ['Bytes', 'Blob'], allowed_modalities: ['audio', 'image', 'video', 'document', 'file'], + require_media_redaction: false, require_metadata_removal: false, allow_unredacted_media_with_text_pii: false, record_audio: false, record_video: false }; +const limits = { max_upload_bytes: 52428800, segment_bytes: 8388608, segment_duration_ms: 30000, recording_bytes: 1073741824, + recording_duration_ms: 7200000, active_transfers: 16, active_recordings: 4, reserved_bytes: 1073741824, validation_jobs: 1024, + initializations_per_minute: 120, links_per_minute: 1000, upload_bytes_per_second: 1073741824, playback_bytes_per_second: 1073741824, + transfer_timeout_seconds: 600, grant_ttl_seconds: 300, revocation_check_seconds: 5 }; +const options = { mimeType: 'audio/wav', direction: 'mixed' }; +const source = () => ({ kind: 'Bytes', data: new Uint8Array([82, 73, 70, 70, 1, 2, 3, 4]) }); +// One hour: well outside the renewal window, so call-count assertions see no session renewal. +const expiry = () => new Date(Date.now() + 3600000).toISOString(); +function fake(overrides = {}) { + const calls = []; let body; + const transport = { + async bootstrap(r) { calls.push('bootstrap'); return { schema_version: 1, scope: { organization_id: ids[0], workspace_id: ids[1], project_id: ids[2] }, capture_session_id: r.capture_session_id, policy, limits, expires_at: expiry(), transfer_modes: ['authenticated_stream'] }; }, + async initialize(_s, r) { calls.push('initialize'); return { upload_id: ids[0], attachment_id: r.attachment_id, occurrence_id: r.occurrence_id, state: 'pending_upload', reserved_bytes: r.declared_bytes, expires_at: expiry(), transfer_mode: 'authenticated_stream', write_capability: 'synthetic-write-capability' }; }, + async upload(_s, _u, data) { calls.push('upload'); body = Array.from(data instanceof Uint8Array ? data : new Uint8Array(await data.arrayBuffer())); }, + async complete() { calls.push('complete'); }, ...overrides, + }; + return { transport, calls, body: () => body }; +} + +module.exports = { identity, ids, policy, limits, options, source, expiry, fake }; diff --git a/typescript/packages/fi-core/test/attachments-tls.test.cjs b/typescript/packages/fi-core/test/attachments-tls.test.cjs new file mode 100644 index 000000000..48f1733d8 --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-tls.test.cjs @@ -0,0 +1,65 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const https = require('node:https'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { execFileSync, spawn } = require('node:child_process'); +const { once } = require('node:events'); +const { ids, fake } = require('./attachments-fixtures.cjs'); + +test('Node TLS transfer uses upload capability, bounds readers and does not keep host alive', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'attachment-tls-')); + const key = path.join(dir, 'test.key'); const cert = path.join(dir, 'test.crt'); + execFileSync('openssl', ['req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', key, '-out', cert, '-days', '1', '-subj', '/CN=localhost', '-addext', 'subjectAltName=IP:127.0.0.1,DNS:localhost'], { stdio: 'ignore' }); + const bodies = []; const headers = []; let stalled = false; + const server = https.createServer({ key: fs.readFileSync(key), cert: fs.readFileSync(cert) }, async (req, res) => { + headers.push(req.headers); const chunks = []; + for await (const chunk of req) chunks.push(chunk); + const body = Buffer.concat(chunks); res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('attachment-sessions')) res.end(JSON.stringify(await fake().transport.bootstrap(JSON.parse(body)))); + else if (req.url.endsWith('/uploads')) { res.statusCode = 201; res.end(JSON.stringify(await fake().transport.initialize(null, JSON.parse(body)))); } + else if (req.url.endsWith('/content')) { + bodies.push(body); assert.equal(Number(req.headers['content-length']), body.length); + if (stalled) return; + res.end(JSON.stringify({ upload_id: ids[0], state: 'pending_upload', reason: 'queued' })); + } else { res.statusCode = 202; res.end(JSON.stringify({ upload_id: ids[0], state: 'validating', reason: 'binding_pending' })); } + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + // T-TR-04: ambient proxy settings are a canary; the attachment client must never connect to them. + let proxied = 0; const canary = require('node:net').createServer(socket => { proxied++; socket.destroy(); }); + canary.listen(0, '127.0.0.1'); await once(canary, 'listening'); + const proxy = `http://127.0.0.1:${canary.address().port}`; + const modulePath = path.resolve(path.dirname(process.env.FI_CORE_PACKAGE_JSON || path.resolve(__dirname, '../package.json')), 'dist/src/attachments/node.js'); + async function child(stall) { + const script = ` + const { createNodeAttachmentManager } = require(${JSON.stringify(modulePath)}); + const manager = createNodeAttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', + endpoint: 'https://127.0.0.1:${server.address().port}', headers: { authorization: 'Bearer synthetic-bootstrap', 'x-api-key': 'synthetic-initial-only' } }); + const live = ${stall} ? setTimeout(() => {}, 500) : undefined; + (async () => { + if (!await manager.ready()) throw Error('ready'); + const handle = manager.attach({ traceId: '0123456789abcdef0123456789abcdef', spanId: '0123456789abcdef', traceFlags: 1 }, + { kind: 'Blob', data: new Blob([new Uint8Array(700000).fill(7)]) }, { direction: 'mixed', mimeType: 'audio/wav' }); + if (${stall}) { console.log('TRANSFER_STARTED'); return; } + await manager.flush(); console.log(JSON.stringify(await handle.completion)); await manager.shutdown(); + if (live) clearTimeout(live); + })().catch(() => process.exitCode = 1); + `; + const processChild = spawn(process.execPath, ['-e', script], { env: { ...process.env, NODE_EXTRA_CA_CERTS: cert, HTTPS_PROXY: proxy, https_proxy: proxy, HTTP_PROXY: proxy, NODE_USE_ENV_PROXY: '1' }, stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; let stderr = ''; processChild.stdout.on('data', c => { stdout += c; }); processChild.stderr.on('data', c => { stderr += c; }); + const deadline = setTimeout(() => processChild.kill('SIGKILL'), 5000); + const [code, signal] = await once(processChild, 'exit'); clearTimeout(deadline); + assert.equal(signal, null, 'background transfer kept the process alive'); assert.equal(code, 0, stderr); + return stdout; + } + try { + assert.match(await child(false), /"status":"validating"/); + assert.equal(bodies[0].length, 700000); assert.ok(bodies[0].every(byte => byte === 7)); + assert.equal(headers[2].authorization, 'Bearer synthetic-write-capability'); assert.equal(headers[2]['x-api-key'], undefined); + assert.equal(headers[3].authorization, 'Bearer synthetic-write-capability'); + stalled = true; + assert.match(await child(true), /TRANSFER_STARTED/); assert.equal(bodies.length, 2); + assert.equal(proxied, 0, 'environment proxy was used'); + } finally { canary.close(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); fs.rmSync(dir, { recursive: true, force: true }); } +}); diff --git a/typescript/packages/fi-core/test/attachments-v1.cjs b/typescript/packages/fi-core/test/attachments-v1.cjs new file mode 100644 index 000000000..5320e96dc --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-v1.cjs @@ -0,0 +1,93 @@ +// Synthetic doubles for the versioned wire surface (design section 9). No network, no real credentials. +const path = require('node:path'); +const { createRequire } = require('node:module'); +const packageJson = process.env.FI_CORE_PACKAGE_JSON || path.resolve(__dirname, '../package.json'); +const load = createRequire(packageJson); +const core = load('@traceai/fi-core/attachments'); +const nodeCore = load('@traceai/fi-core/attachments/node'); +const { ids, policy, limits, identity } = require('./attachments-fixtures.cjs'); + +async function readAll(source) { + if (source instanceof Uint8Array) return Buffer.from(source); + if (typeof Blob !== 'undefined' && source instanceof Blob) return Buffer.from(await source.arrayBuffer()); + const out = Buffer.alloc(source.size); + for (let offset = 0; offset < source.size; offset += 262144) { + await source.check(); + const part = await source.read(offset, Math.min(262144, source.size - offset)); + Buffer.from(part.buffer, part.byteOffset, part.byteLength).copy(out, offset); + } + await source.check(); + return out; +} +function clock(start = 1_700_000_000_000) { + let t = 0; + return { now: () => t, wall: () => start + t, advance(ms) { t += ms; }, set(ms) { t = ms; } }; +} +/** A complete in-memory AttachmentTransportV1. `overrides(base)` may replace any method. */ +function v1(overrides = {}, options = {}) { + const calls = []; let n = 0; + const id = () => `30000000-0000-4000-8000-${(++n).toString(16).padStart(12, '0')}`; + const cap = () => `synthetic-capability-${(++n).toString(36)}-abcdefgh`; + const wall = options.clock ? options.clock.wall : () => Date.now(); + const contents = new Map(); const manifests = []; const imports = []; const requests = []; const completed = new Map(); + const session = sid => ({ schema_version: 1, scope: { organization_id: ids[0], workspace_id: ids[1], project_id: ids[2] }, capture_session_id: sid, + policy: { ...policy, allowed_sources: ['Bytes', 'Blob', 'OwnedRecordingFile', 'ProviderFile', 'RemoteUrl'], record_audio: true, ...(options.policy || {}) }, + limits: { ...limits, ...(options.limits || {}) }, expires_at: new Date(wall() + (options.lifetimeMs ?? 3600000)).toISOString(), transfer_modes: ['authenticated_stream'] }); + const base = { + async bootstrap(body, key) { calls.push({ op: 'bootstrap', key, body }); return session(body.capture_session_id); }, + async renewSession(current, sequence) { calls.push({ op: 'renewSession', sequence }); return session(current.capture_session_id); }, + async initialize(_s, body, key) { + const request = JSON.parse(body); calls.push({ op: 'initialize', key, request, body }); requests.push(request); + return { upload_id: id(), attachment_id: request.attachment_id, occurrence_id: request.occurrence_id, state: 'pending_upload', reserved_bytes: request.declared_bytes, + expires_at: new Date(wall() + (options.uploadLifetimeMs ?? 600000)).toISOString(), transfer_mode: 'authenticated_stream', write_capability: cap() }; + }, + async renewUpload(_s, upload, body) { + calls.push({ op: 'renewUpload', body }); + return { ...upload, write_capability: cap(), expires_at: new Date(Date.parse(upload.expires_at) + 600000).toISOString() }; + }, + async putContent(_s, upload, source) { + const bytes = await readAll(source); + calls.push({ op: 'putContent', upload: upload.upload_id, occurrence: upload.occurrence_id, size: bytes.length, capability: upload.write_capability }); + contents.set(upload.occurrence_id, bytes); + }, + async complete(_s, upload, key) { calls.push({ op: 'complete', key, capability: upload.write_capability, occurrence: upload.occurrence_id }); completed.set(upload.occurrence_id, upload.write_capability); return 'validating'; }, + async abort(_s, upload, key) { calls.push({ op: 'abort', key, occurrence: upload.occurrence_id }); }, + async uploadStatus() { calls.push({ op: 'uploadStatus' }); return { state: 'validating', reason: 'binding_pending' }; }, + async putManifest(_s, capability, recordingId, body, key) { calls.push({ op: 'putManifest', key, capability, recordingId }); manifests.push({ capability, key, raw: body, body: JSON.parse(body) }); }, + async submitImport(_s, body, key) { const parsed = JSON.parse(body); calls.push({ op: 'submitImport', key, body: parsed }); imports.push({ raw: body, body: parsed, key }); return { jobId: id(), recordingId: parsed.recording.recording_id, state: 'importing' }; }, + async importStatus(_s, jobId) { calls.push({ op: 'importStatus', jobId }); return { state: 'importing', reason: 'binding_pending' }; }, + }; + const transport = { ...base, ...(typeof overrides === 'function' ? overrides(base) : overrides) }; + return { transport, base, calls, contents, manifests, imports, requests, completed, ops: () => calls.map(c => c.op) }; +} +async function managerV1(overrides, options = {}, managerOptions = {}) { + const double = v1(overrides, options); + const manager = new core.AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', environment: () => ({}), transport: double.transport, + recordAudio: true, ...(options.clock ? { clock: options.clock } : {}), ...managerOptions }); + if (options.ready !== false) { if (!await manager.ready()) throw Error('ready'); } + return { manager, ...double }; +} +function span(context = identity) { + const attrs = {}; let recording = true; let writes = 0; + return { attrs, get writes() { return writes; }, end() { recording = false; }, + spanContext: () => context, isRecording: () => recording, setAttribute(k, v) { writes++; attrs[k] = v; return this; } }; +} +/** Deterministic s16le PCM: `frames` frames of `channels` channels, offset by `seed`. */ +function pcm(frames, channels = 1, seed = 0) { + const out = new Uint8Array(frames * channels * 2); + for (let i = 0; i < out.length; i++) out[i] = (i * 31 + seed * 7 + (i >> 9)) & 0xff; + return out; +} +/** Write `seconds` of audio in `chunkMs` chunks; awaits `manager.flush()` every `flushEvery` seconds so uploads progress. */ +async function writeSeconds(manager, writer, rate, seconds, { channels = 1, track = 0, chunkMs = 20, flushEvery = 10, seed = 0 } = {}) { + const frames = rate * chunkMs / 1000; let accepted = 0; let rejected = 0; let offset = 0; + const chunks = Math.round(seconds * 1000 / chunkMs); + for (let i = 0; i < chunks; i++) { + const data = pcm(frames, channels, seed + offset); offset += frames; + if (writer.tryWrite(data, track)) accepted++; else rejected++; + if (flushEvery && (i + 1) % Math.round(flushEvery * 1000 / chunkMs) === 0) await manager.flush(10000); + } + return { accepted, rejected }; +} +const mono = rate => ({ format: { sampleRateHz: rate }, layout: 'mixed', tracks: [{ channels: [0], direction: 'mixed', role: 'mixed' }], operation: 'observe_consumed' }); +module.exports = { core, nodeCore, v1, managerV1, span, pcm, writeSeconds, readAll, clock, mono }; diff --git a/typescript/packages/fi-core/test/attachments-wire.test.cjs b/typescript/packages/fi-core/test/attachments-wire.test.cjs new file mode 100644 index 000000000..e23476bed --- /dev/null +++ b/typescript/packages/fi-core/test/attachments-wire.test.cjs @@ -0,0 +1,32 @@ +/* T-TR-01: every F01 wire fixture for the shapes the SDK sends or parses, reproduced by the hand validators. */ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { core } = require('./attachments-v1.cjs'); +const directory = process.env.FI_ATTACHMENTS_SPEC_DIR; +const SDK = new Set(core.ATTACHMENT_WIRE_SHAPES); +test('T-TR-01 F01 wire fixtures agree with the core validators', { skip: !directory && 'FI_ATTACHMENTS_SPEC_DIR not set' }, () => { + const cases = JSON.parse(fs.readFileSync(path.join(directory, 'wire-fixtures.json'))).cases; + let checked = 0; const skipped = new Set(); + for (const fixture of cases) { + let schema = fixture.schema; let value = fixture.value; + if (schema === 'ImportRequest' && value && value.mode === 'completed_recording') schema = 'RecordingImportRequest'; + else if (schema === 'ImportRequest') { skipped.add('ImportRequest(single)'); continue; } + if (!SDK.has(schema)) { skipped.add(schema); continue; } + assert.equal(core.validAttachmentWire(schema, value), fixture.valid, fixture.id); checked++; + } + const imports = JSON.parse(fs.readFileSync(path.join(directory, 'recording-import-fixtures.json'))); + for (const fixture of imports.cases ?? []) { + if (!fixture.request || fixture.schema && fixture.schema !== 'RecordingImportRequest') continue; + assert.equal(core.validAttachmentWire('RecordingImportRequest', fixture.request), fixture.valid ?? true, fixture.id); checked++; + } + console.log(`# T-TR-01 checked=${checked} not-sdk-shapes=${[...skipped].sort().join(',')}`); + assert.ok(checked >= 80, `only ${checked} fixtures checked`); +}); +test('T-TR-08 lifecycle fixtures: complete is validating, never ready, in both orders', { skip: !directory && 'FI_ATTACHMENTS_SPEC_DIR not set' }, () => { + const lifecycle = JSON.parse(fs.readFileSync(path.join(directory, 'lifecycle-fixtures.json'))); + const text = JSON.stringify(lifecycle); + assert.match(text, /late-root-releases-ready-bytes/); assert.match(text, /root-first-still-needs-validation/); + assert.equal(lifecycle.initial?.generation ?? 0, 0, 'CQ-5: new occurrences start at deletion generation 0'); +}); diff --git a/typescript/packages/fi-core/test/attachments.test.cjs b/typescript/packages/fi-core/test/attachments.test.cjs new file mode 100644 index 000000000..8d0a1cf23 --- /dev/null +++ b/typescript/packages/fi-core/test/attachments.test.cjs @@ -0,0 +1,237 @@ +/* Component qualification only: synthetic bytes and a loopback attachment API. */ +const { test, before, after } = require('node:test'); +const assert = require('node:assert/strict'); +const http = require('node:http'); +const { once } = require('node:events'); +const { pathToFileURL } = require('node:url'); +const path = require('node:path'); +const { createRequire } = require('node:module'); +const packageJson = process.env.FI_CORE_PACKAGE_JSON || path.resolve(__dirname, '../package.json'); +const load = createRequire(packageJson); +const { AttachmentManager, AttachmentCorrelation, canonicalAttachmentTraceId, attachmentTraceUuid, + evaluateAttachmentPolicy, createAttachmentHttpTransport, AttachmentTransportError } = load('@traceai/fi-core/attachments'); +const { register, ProjectType } = load('@traceai/fi-core'); +const { identity, ids, policy, limits, options, source, expiry, fake } = require('./attachments-fixtures.cjs'); +async function manager(overrides = {}, transportOverrides = {}) { + const fixture = fake(transportOverrides); + const value = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', transport: fixture.transport, ...overrides }); + assert.equal(await value.ready(), true); + return { value, ...fixture }; +} +const live = setInterval(() => {}, 1000); // Explicit test owner, not an SDK keepalive. +after(() => clearInterval(live)); + +test('OFF by default: no bootstrap, source access or reference metadata', async () => { + const f = fake(); let read = false; + const value = new AttachmentManager({ transport: f.transport }); + const result = value.attach(identity, { kind: 'Bytes', get data() { read = true; throw Error('synthetic-secret'); } }, options); + assert.equal(result.admission, 'disabled'); assert.equal(read, false); assert.deepEqual(f.calls, []); + assert.deepEqual(await result.completion, { status: 'disabled' }); assert.equal(JSON.stringify(result).includes('synthetic-secret'), false); +}); + +test('unknown bootstrap retains no bytes and never reads a denied source', async () => { + const value = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', transport: fake({ bootstrap: () => new Promise(() => {}) }).transport }); + let read = false; + const h = value.attach(identity, { kind: 'Bytes', get data() { read = true; return new Uint8Array(1); } }, options); + assert.equal(h.admission, 'bootstrap_pending'); assert.equal(read, false); assert.equal(value.status().ownedBytes, 0); + await value.shutdown(1); +}); + +for (const [key, direction, mimeType, capture] of [ + ['hideInputs', 'input', 'audio/wav'], ['hideInputMessages', 'input', 'application/pdf'], ['hideInputText', 'input', 'image/png'], + ['hideOutputs', 'output', 'audio/wav'], ['hideOutputMessages', 'output', 'image/png'], ['hideOutputText', 'mixed', 'audio/wav'], + ['hideInputImages', 'output', 'image/png', { modelInput: true }], ['hideErrorDetails', 'output', 'audio/wav', { errorDetail: true }], + ['piiRedaction', 'mixed', 'audio/wav'], ['piiRedaction', 'input', 'image/png'], ['piiRedaction', 'output', 'video/mp4'], + ['piiRedaction', 'input', 'application/pdf'], ['piiRedaction', 'input', 'application/octet-stream'], +]) test(`no source access or upload with ${key}/${mimeType}`, async () => { + const f = await manager({ traceConfig: { [key]: true } }); let read = false; + const h = f.value.attach(identity, { kind: 'Bytes', get data() { read = true; throw Error('synthetic-secret'); } }, { mimeType, direction, capture }); + assert.equal(h.admission, key === 'piiRedaction' ? 'redaction_unavailable' : 'policy_denied'); + await f.value.flush(); assert.equal(read, false); assert.deepEqual(f.calls, ['bootstrap']); await f.value.shutdown(); +}); + +test('environment restrictions win over explicit per-call and constructor allows', async () => { + const f = await manager({ environment: () => ({ FI_HIDE_INPUTS: 'malformed' }), traceConfig: { hideInputs: false } }); + assert.equal(f.value.attach(identity, source(), options).admission, 'policy_denied'); + assert.equal(evaluateAttachmentPolicy({ enabled: true }, {}, policy, 'Bytes', options, { FI_ATTACHMENTS_ENABLED: 'false' }), 'disabled'); + await f.value.shutdown(); +}); + +test('mutable ownership is detached, admission asynchronous, completion not readiness', async () => { + const f = await manager(); const src = source(); + const h = f.value.attach(identity, src, options); src.data.fill(9); + assert.deepEqual(f.calls, ['bootstrap']); assert.equal(h.admission, 'queued'); + assert.deepEqual(await h.completion, { status: 'validating' }); assert.deepEqual(f.body(), [82, 73, 70, 70, 1, 2, 3, 4]); + assert.deepEqual(await f.value.flush(), { unfinished: 0, dropped: 0 }); assert.equal(f.value.status().ownedBytes, 0); await f.value.shutdown(); +}); + +test('Blob read is independent; unknown stream, file and URL never read', async () => { + const f = await manager(); const blob = new Blob([new Uint8Array([1, 2, 3])]); + assert.equal((await f.value.attach(identity, { kind: 'Blob', data: blob }, options).completion).status, 'validating'); + for (const kind of ['RemoteUrl', 'LocalFile', 'OwnedRecordingFile', 'ProviderFile']) { + const item = { kind, get url() { throw Error('must-not-read'); }, get path() { throw Error('must-not-read'); } }; + assert.notEqual(f.value.attach(identity, item, options).admission, 'queued'); + } + let pulled = false; + assert.equal(f.value.attach(identity, { kind: 'Bytes', data: { getReader() { pulled = true; } } }, options).admission, 'source_unavailable'); + assert.equal(pulled, false); await f.value.shutdown(); +}); + +test('known sampled identity required and exact trace bits preserved', async () => { + const f = await manager(); + for (const traceId of ['00000000000000000000000000000000', 'BAD', '0123456789ABCDEF0123456789ABCDEF']) assert.equal(canonicalAttachmentTraceId(traceId), undefined); + const hex = 'ffffffffffffffffffffffffffffffff'; assert.equal(attachmentTraceUuid(canonicalAttachmentTraceId(hex)), 'ffffffff-ffff-ffff-ffff-ffffffffffff'); + assert.equal(f.value.attach({ ...identity, traceFlags: 0 }, source(), options).admission, 'unsampled'); + assert.equal(f.value.attach(new AttachmentCorrelation(), source(), options).admission, 'correlation_unavailable'); + await f.value.shutdown(); +}); + +test('same-span references bounded to 64; ended span untouched by completion', async () => { + const f = await manager(); let recording = true; let writes = 0; const attrs = {}; + const span = { spanContext: () => identity, isRecording: () => recording, setAttribute(k, v) { writes++; attrs[k] = v; } }; + const correlation = f.value.correlation(span); const handles = []; + for (let i = 0; i < 70; i++) handles.push(f.value.attach(span, source(), options)); + assert.equal(JSON.parse(attrs['fi.attachment.refs']).length, 64); assert.ok(Buffer.byteLength(attrs['fi.attachment.refs']) <= 16384); + recording = false; const before = writes; + await Promise.all(handles.map(h => h.completion)); assert.equal(writes, before); + assert.equal(correlation.snapshot().sampled, true); await f.value.shutdown(); +}); + +test('memory/descriptors include active transfer and backpressure drops without copying', async () => { + let release; const wait = new Promise(resolve => { release = resolve; }); + const f = await manager({ maxOwnedBytes: 8, maxDescriptors: 1 }, { upload: async () => { await wait; } }); + const h = f.value.attach(identity, source(), options); let read = false; + assert.equal(f.value.attach(identity, { kind: 'Bytes', get data() { read = true; return new Uint8Array(8); } }, options).admission, 'queue_full'); + assert.equal(read, false); assert.equal(f.value.status().ownedBytes, 8); + release(); await h.completion; await f.value.flush(); assert.equal(f.value.status().ownedBytes, 0); + assert.equal(f.value.attach(identity, { kind: 'Bytes', data: new Uint8Array(9) }, options).admission, 'too_large'); + await f.value.shutdown(); +}); + +test('policy tightening cancels queued bytes and does not resurrect after re-enable', async () => { + const f = await manager(); const h = f.value.attach(identity, source(), options); + f.value.updatePolicy({ traceConfig: { hideInputs: true } }); + assert.equal((await h.completion).status, 'policy_denied'); assert.equal(f.value.status().ownedBytes, 0); + f.value.updatePolicy({ traceConfig: { hideInputs: false } }); await f.value.flush(); assert.deepEqual(f.calls, ['bootstrap']); await f.value.shutdown(); +}); + +test('bounded retries recheck live policy and preserve occurrence IDs', async () => { + let tries = 0; const requests = []; const f = fake(); + const fixture = await manager({}, { initialize: async (s, r, signal) => { requests.push(r.occurrence_id); if (++tries < 3) throw new AttachmentTransportError('failed', true); return f.transport.initialize(s, r, signal); } }); + const h = fixture.value.attach(identity, source(), options); await h.completion; + assert.equal(tries, 3); assert.equal(new Set(requests).size, 1); await fixture.value.shutdown(); +}); + +test('authorization failures never retry and exception bodies are absent', async () => { + let tries = 0; + const f = await manager({}, { initialize: async () => { tries++; throw new AttachmentTransportError('credential_scope_unsupported'); } }); + const h = f.value.attach(identity, source(), options); assert.deepEqual(await h.completion, { status: 'credential_scope_unsupported' }); assert.equal(tries, 1); await f.value.shutdown(); +}); + +test('flush deadline and idempotent shutdown cancel active work', async () => { + let aborted = false; + const f = await manager({}, { upload: (_s, _u, _d, signal) => new Promise((resolve, reject) => signal.addEventListener('abort', () => { aborted = true; reject(Error('synthetic-secret')); }, { once: true })) }); + const h = f.value.attach(identity, source(), options); + await new Promise(resolve => setImmediate(resolve)); + assert.equal((await f.value.flush(5)).unfinished, 1); + const closing = f.value.shutdown(5); assert.equal(f.value.shutdown(5), closing); await closing; + assert.equal(aborted, true); assert.equal((await h.completion).status, 'incomplete'); + assert.equal(f.value.attach(identity, source(), options).admission, 'incomplete'); +}); + +test('HTTP v1 loopback upload: stable identity, bounded response, no source URLs', async () => { + const calls = []; const uploads = new Map(); let policyOverride; + const server = http.createServer(async (req, res) => { + const chunks = []; for await (const chunk of req) chunks.push(chunk); + const data = Buffer.concat(chunks); calls.push({ method: req.method, path: req.url, data, headers: req.headers }); + res.setHeader('content-type', 'application/json'); + if (req.url.endsWith('attachment-sessions')) { + const body = JSON.parse(data); const fixture = await fake().transport.bootstrap(body); + res.end(JSON.stringify({ ...fixture, policy: policyOverride ?? policy })); + } else if (req.url.endsWith('/uploads')) { + const body = JSON.parse(data); const upload = await fake().transport.initialize(null, body); uploads.set(upload.upload_id, upload); + assert.equal(body.association.parent.trace_id, '01234567-89ab-cdef-0123-456789abcdef'); res.statusCode = 201; res.end(JSON.stringify(upload)); + } else if (req.url.endsWith('/content')) { + assert.deepEqual([...data], [...source().data]); res.end(JSON.stringify({ upload_id: ids[0], state: 'pending_upload', reason: 'queued' })); + } else { res.statusCode = 202; res.end(JSON.stringify({ upload_id: ids[0], state: 'validating', reason: 'binding_pending' })); } + }); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const client = req => new Promise((resolve, reject) => { + const request = http.request(`http://127.0.0.1:${server.address().port}${new URL(req.url).pathname}`, { method: req.method, headers: req.headers, signal: req.signal }, response => { + const chunks = []; response.on('data', c => chunks.push(c)); response.on('end', () => resolve({ status: response.statusCode, body: Buffer.concat(chunks) })); + }); request.on('error', reject); request.end(req.body); + }); + try { + const transport = createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: { authorization: 'Bearer synthetic', cookie: 'never-forward' }, client }); + const f = await manager({ transport }); const h = f.value.attach(identity, source(), options); + assert.equal((await h.completion).status, 'validating'); assert.equal(calls.length, 4); assert.equal(calls.some(c => c.headers.cookie), false); + await f.value.shutdown(); + for (const traceConfig of [{ hideInputs: true }, { hideOutputs: true }, { piiRedaction: true }]) { + const value = new AttachmentManager({ enabled: true, projectName: 'fixture', projectType: 'observe', transport, traceConfig }); + await value.ready(); const before = calls.length; assert.notEqual(value.attach(identity, source(), options).admission, 'queued'); + await value.flush(); assert.equal(calls.length, before); await value.shutdown(); + } + } finally { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); } +}); + +test('transport blocks redirects and invalid origins', async () => { + for (const endpoint of ['http://example.com', 'https://user:secret@example.com', 'https://example.com/path', 'https://example.com/?token=secret']) assert.throws(() => createAttachmentHttpTransport({ endpoint, headers: {} }), /endpoint_unconfigured/); + const transport = createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: {}, client: async () => ({ status: 302, body: Buffer.from('synthetic-secret') }) }); + // Redirects are never followed; a 3xx is an unmapped status: incomplete, not retried (design 0.5). + await assert.rejects(transport.bootstrap({}, new AbortController().signal), error => error.message === 'incomplete' && !error.retryable); +}); + +test('CJS and ESM leaf export the public manager API', async () => { + const esm = await import(pathToFileURL(path.resolve(path.dirname(packageJson), 'dist/esm/attachments/index.js'))); + assert.equal(esm.AttachmentManager, AttachmentManager); assert.equal(esm.AttachmentCorrelation, AttachmentCorrelation); assert.equal(typeof esm.createAttachmentHttpTransport, 'function'); +}); + +test('provider owns one manager and flush/shutdown drain its lifecycle', async () => { + const f = fake(); + const provider = register({ projectName: 'fixture', projectType: ProjectType.OBSERVE, setGlobalTracerProvider: false, spanProcessors: [], attachments: { enabled: true, transport: f.transport } }); + await provider.attachments.ready(); + const h = provider.attachments.attach(identity, source(), options); await provider.forceFlush(); assert.equal((await h.completion).status, 'validating'); + const first = provider.shutdown(); assert.equal(first, provider.shutdown()); await first; +}); + +test('late completion uses the stamped writer carrier and never mutates an ended span', async () => { + const f = await manager(); const attributes = {}; let recording = true; let writes = 0; + const span = { spanContext: () => identity, isRecording: () => recording, setAttribute(k, v) { attributes[k] = v; writes++; } }; + const correlation = f.value.correlation(span); + // Design 0.8: a correlation snapshot alone stamps nothing; only an admitted capture on a live span does. + assert.equal(writes, 0); + recording = false; const before = writes; + const h = f.value.attach(correlation, source(), { ...options, conversationId: 'conversation-1', turnId: 'turn-1', toolCallId: 'call-1' }); + assert.equal((await h.completion).status, 'validating'); assert.equal(writes, before); await f.value.shutdown(); +}); + +test('untrusted metadata keys and malformed IDs are not retained or projected', async () => { + const f = await manager(); + const bad = ['abc\n', 'has spaces', 'secret@example.com', '../path', 'x'.repeat(129)]; + for (const conversationId of bad) assert.equal(f.value.attach(identity, source(), { ...options, conversationId }).admission, 'policy_denied'); + assert.equal(canonicalAttachmentTraceId(identity.traceId + '\n'), undefined); + assert.equal(f.value.attach({ ...identity, spanId: identity.spanId + '\n' }, source(), options).admission, 'correlation_unavailable'); + assert.equal(f.value.attach(identity, source(), { ...options, capture: { recordInputs: 'unknown' } }).admission, 'policy_denied'); + await f.value.shutdown(); +}); + +test('upload response binds exact IDs, reservation and capability', async () => { + for (const field of ['attachment_id', 'occurrence_id', 'reserved_bytes', 'write_capability']) { + const t = fake(); const f = await manager({}, { initialize: async (s, r) => ({ ...await t.transport.initialize(s, r), [field]: field === 'reserved_bytes' ? 999 : 'wrong' }) }); + const h = f.value.attach(identity, source(), options); assert.equal((await h.completion).status, 'unsupported_version'); + assert.equal(f.calls.includes('upload'), false); await f.value.shutdown(); + } +}); + +test('successful response reader rejects oversized control data', async () => { + const transport = createAttachmentHttpTransport({ endpoint: 'https://attachments.example', headers: {}, client: async () => ({ status: 200, body: new Uint8Array(65537) }) }); + await assert.rejects(transport.bootstrap({}, new AbortController().signal), error => error.message === 'unsupported_version' && !error.retryable); +}); + +test('provider trace options cannot weaken attachment-level privacy restrictions', async () => { + const f = fake(); + const provider = register({ projectName: 'fixture', projectType: ProjectType.OBSERVE, setGlobalTracerProvider: false, spanProcessors: [], + traceConfig: { hideInputs: false }, attachments: { enabled: true, transport: f.transport, traceConfig: { hideInputs: true } } }); + await provider.attachments.ready(); assert.equal(provider.attachments.attach(identity, source(), options).admission, 'policy_denied'); + await provider.shutdown(); assert.deepEqual(f.calls, ['bootstrap']); +}); diff --git a/typescript/packages/fi-core/test/types/attachments-compat.ts b/typescript/packages/fi-core/test/types/attachments-compat.ts new file mode 100644 index 000000000..cabf1a2d3 --- /dev/null +++ b/typescript/packages/fi-core/test/types/attachments-compat.ts @@ -0,0 +1,32 @@ +/* + * Type-level guard for ATTACHMENTS-MIGRATION.md, compiled against the BUILT declarations (what consumers see). + * Each `@ts-expect-error` must be consumed; if a retired value becomes assignable again, tsc fails with TS2578. + */ +import type { + AttachmentCompletion, AttachmentManager, AttachmentManagerOptions, AttachmentStatus, AttachmentTransport, AttachmentTransportV1, +} from "../../dist/src/attachments/index"; + +declare const manager: AttachmentManager; +declare const status: AttachmentStatus; +declare const completion: AttachmentCompletion; +declare const legacy: AttachmentTransport; +declare const wire: AttachmentTransportV1; + +// Ready state: "queued" -> "ready" (design 0.2). +// @ts-expect-error TS2367: "queued" is no longer a runtime state; compare with "ready". +export const oldReady = manager.status().state !== "queued"; +// @ts-expect-error TS2367: same check through a stored status object. +export const oldReadyStored = status.state === "queued"; +export const newReady: boolean = manager.status().state === "ready"; + +// Completion: "failed" retired for "incomplete" / "unsupported_version" (design 0.3). +// @ts-expect-error TS2367: "failed" is never produced. +export const oldFailed = completion.status === "failed"; +export const newFailed: boolean = completion.status === "incomplete" || completion.status === "unsupported_version"; + +// Admission results keep "queued": that is the admitted reason, not a runtime state. +export const admitted: boolean = manager.attach(undefined, { kind: "Bytes", data: new Uint8Array(1) }, { mimeType: "image/png", direction: "input" }).admission === "queued"; + +// Both transport shapes still type-check; the legacy one is warned about at runtime. +export const legacyOptions: AttachmentManagerOptions = { transport: legacy }; +export const v1Options: AttachmentManagerOptions = { transport: wire }; diff --git a/typescript/packages/fi-core/test/types/tsconfig.json b/typescript/packages/fi-core/test/types/tsconfig.json new file mode 100644 index 000000000..24bcc060c --- /dev/null +++ b/typescript/packages/fi-core/test/types/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "strict": true, + "noEmit": true, + "target": "es2021", + "lib": ["es2021", "dom"], + "module": "commonjs", + "moduleResolution": "node", + "skipLibCheck": true, + "types": [] + }, + "files": ["attachments-compat.ts"] +}