From ddfb92c83aff93f851afb686f879f801f5523ea8 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:02:10 -0700 Subject: [PATCH 1/2] fix(anthropic): leave native TypeScript SDK tracing unpatched Keep legacy SDK tracing and the automatic version gate unchanged. Avoid duplicate spans and APIPromise/stream contract breakage when manual instrumentation is used with a native-OTel SDK. Warn that legacy traceConfig does not govern native spans. Co-Authored-By: Claude Code --- .../workflows/anthropic-typescript-tests.yml | 76 ++++++++ .../packages/traceai_anthropic/README.md | 10 +- .../contract/native-boundary.cjs | 165 ++++++++++++++++++ .../src/__tests__/nativeBoundary.test.ts | 38 ++++ .../traceai_anthropic/src/instrumentation.ts | 15 ++ 5 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/anthropic-typescript-tests.yml create mode 100644 typescript/packages/traceai_anthropic/contract/native-boundary.cjs create mode 100644 typescript/packages/traceai_anthropic/src/__tests__/nativeBoundary.test.ts diff --git a/.github/workflows/anthropic-typescript-tests.yml b/.github/workflows/anthropic-typescript-tests.yml new file mode 100644 index 000000000..cb066a03c --- /dev/null +++ b/.github/workflows/anthropic-typescript-tests.yml @@ -0,0 +1,76 @@ +name: Anthropic TypeScript tests + +on: + pull_request: + branches: [main, dev] + paths: + - 'typescript/packages/traceai_anthropic/**' + - 'typescript/packages/fi-core/**' + - 'typescript/packages/fi-semantic-conventions/**' + - 'typescript/pnpm-lock.yaml' + - '.github/workflows/anthropic-typescript-tests.yml' + push: + branches: [dev] + paths: + - 'typescript/packages/traceai_anthropic/**' + - 'typescript/packages/fi-core/**' + - 'typescript/packages/fi-semantic-conventions/**' + - 'typescript/pnpm-lock.yaml' + - '.github/workflows/anthropic-typescript-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + contract: + name: Anthropic TypeScript native boundary + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22.23.4' + - name: Install workspace and pinned real SDK fixtures + working-directory: typescript + run: | + mkdir -p "$RUNNER_TEMP/corepack-bin" + corepack enable --install-directory "$RUNNER_TEMP/corepack-bin" pnpm + echo "$RUNNER_TEMP/corepack-bin" >> "$GITHUB_PATH" + corepack pnpm --filter @traceai/anthropic... install --frozen-lockfile --ignore-scripts + mkdir -p "$RUNNER_TEMP/anthropic-contract" + npm install --prefix "$RUNNER_TEMP/anthropic-contract" --ignore-scripts --no-audit --no-fund \ + @anthropic-ai/sdk@1.3.0 \ + anthropic-floor@npm:@anthropic-ai/sdk@0.20.0 \ + anthropic-peer@npm:@anthropic-ai/sdk@0.27.3 \ + @opentelemetry/api@1.9.0 \ + @opentelemetry/sdk-trace-base@2.4.0 \ + @opentelemetry/context-async-hooks@2.4.0 \ + @opentelemetry/exporter-trace-otlp-http@0.208.0 + - name: Build all package formats + working-directory: typescript + run: | + pnpm run build:deps + pnpm --filter @traceai/anthropic run build + - name: Unit tests and SDK contracts without external networking + working-directory: typescript + run: | + # A separate network namespace exposes only loopback for the OTLP receiver. + # Run tests as the unprivileged runner, with no inherited credentials. + mkdir -p "$RUNNER_TEMP/anthropic-test-home" + node_bin="$(command -v node)" + sudo unshare --net sh -c 'ip link set lo up; exec runuser -u runner -- "$@"' sh \ + env -i PATH="$PATH" HOME="$RUNNER_TEMP/anthropic-test-home" \ + "$node_bin" node_modules/jest/bin/jest.js --config packages/traceai_anthropic/jest.config.js --runInBand + for sdk in @anthropic-ai/sdk anthropic-floor anthropic-peer; do + sudo unshare --net sh -c 'ip link set lo up; exec runuser -u runner -- "$@"' sh \ + env -i PATH="$PATH" HOME="$RUNNER_TEMP/anthropic-test-home" \ + "$node_bin" packages/traceai_anthropic/contract/native-boundary.cjs \ + "$RUNNER_TEMP/anthropic-contract" "$sdk" \ + packages/traceai_anthropic/dist/src/instrumentation.js + done + git diff --exit-code -- pnpm-lock.yaml diff --git a/typescript/packages/traceai_anthropic/README.md b/typescript/packages/traceai_anthropic/README.md index 43a768ac3..4a5b66cc3 100644 --- a/typescript/packages/traceai_anthropic/README.md +++ b/typescript/packages/traceai_anthropic/README.md @@ -1,6 +1,14 @@ # @traceai/anthropic -OpenTelemetry instrumentation for Anthropic's Claude API. This package provides automatic tracing and monitoring for your Anthropic applications. +OpenTelemetry instrumentation for Anthropic SDKs without built-in tracing. The supported peer dependency is `@anthropic-ai/sdk@^0.27.3`. + +## SDKs with native OpenTelemetry + +Do not add this legacy instrumentor to SDKs with built-in OpenTelemetry, such as `@anthropic-ai/sdk@1.3.0`. Those SDKs already own their request spans, promise helpers and stream lifecycle. `manuallyInstrument()` leaves them unmodified and logs a diagnostic warning. It does not enable native tracing if you have disabled it. + +Configure native SDK tracing through the SDK's `openTelemetry` option and your OpenTelemetry tracer provider. The legacy instrumentor's `traceConfig` and `FI_HIDE_*` controls do **not** apply to native SDK spans. Use the SDK's content controls and your export pipeline's privacy policy instead. See [Anthropic's SDK tracing configuration](https://www.npmjs.com/package/@anthropic-ai/sdk/v/1.3.0#opentelemetry). + +The setup below is for the legacy SDK peer dependency only. ## Installation diff --git a/typescript/packages/traceai_anthropic/contract/native-boundary.cjs b/typescript/packages/traceai_anthropic/contract/native-boundary.cjs new file mode 100644 index 000000000..2207b68f1 --- /dev/null +++ b/typescript/packages/traceai_anthropic/contract/native-boundary.cjs @@ -0,0 +1,165 @@ +'use strict'; +// Real SDKs, synthetic fetch, local OTLP receiver. No credentials or provider calls. +// Usage: node native-boundary.cjs +const assert = require('node:assert/strict'); +const http = require('node:http'); +const path = require('node:path'); +const { createRequire } = require('node:module'); +const deps = path.resolve(process.argv[2]); +const sdkPackage = process.argv[3]; +process.env.NODE_PATH = deps + '/node_modules'; +require('node:module').Module._initPaths(); +const req = createRequire(deps + '/package.json'); +const { trace, context, SpanStatusCode, diag } = req('@opentelemetry/api'); +const { BasicTracerProvider, SimpleSpanProcessor, InMemorySpanExporter } = req('@opentelemetry/sdk-trace-base'); +const { AsyncLocalStorageContextManager } = req('@opentelemetry/context-async-hooks'); +const { OTLPTraceExporter } = req('@opentelemetry/exporter-trace-otlp-http'); +const Anthropic = req(sdkPackage).default; +const sdkVersion = JSON.parse(require('node:fs').readFileSync(path.join(path.dirname(req.resolve(sdkPackage)), 'package.json'), 'utf8')).version; +const native = 'openTelemetry' in Anthropic.prototype; +const warnings = []; +diag.setLogger({ warn: message => warnings.push(message), error() {}, info() {}, debug() {}, verbose() {} }); +const params = { model: 'synthetic-model', max_tokens: 32, + messages: [{ role: 'user', content: 'PRIVATE_INPUT_CANARY' }] }; +const message = { id: 'msg_synthetic', type: 'message', role: 'assistant', model: params.model, + content: [{ type: 'text', text: 'PRIVATE_OUTPUT_CANARY' }], stop_reason: 'end_turn', stop_sequence: null, + usage: { input_tokens: 7, output_tokens: 3 } }; +const events = [ + { type: 'message_start', message: { ...message, content: [], stop_reason: null, + usage: { input_tokens: 7, output_tokens: 0 } } }, + { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } }, + { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'PRIVATE_OUTPUT_CANARY' } }, + { type: 'content_block_stop', index: 0 }, + { type: 'message_delta', delta: { stop_reason: 'end_turn', stop_sequence: null }, usage: { output_tokens: 3 } }, + { type: 'message_stop' }, +]; +let requests = 0; +const fetch = async (_url, init) => { + requests++; + const body = JSON.parse(init.body); + if (body.model === 'synthetic-error') { + return new Response(JSON.stringify({ type: 'error', error: { type: 'invalid_request_error', message: 'SYNTHETIC_ERROR_DETAIL' } }), + { status: 400, headers: { 'content-type': 'application/json', 'request-id': 'req_error' } }); + } + return new Response(body.stream + ? events.map(event => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`).join('') + : JSON.stringify(message), { status: 200, headers: { + 'content-type': body.stream ? 'text/event-stream' : 'application/json', 'request-id': 'req_synthetic', + } }); +}; +async function main() { + const wire = []; + const receiver = http.createServer((request, response) => { + assert.equal(request.url, '/v1/traces'); + const chunks = []; + request.on('data', chunk => chunks.push(chunk)); + request.on('end', () => { wire.push(JSON.parse(Buffer.concat(chunks))); response.end('{}'); }); + }); + await new Promise(resolve => receiver.listen(0, '127.0.0.1', resolve)); + const memory = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(memory), + new SimpleSpanProcessor(new OTLPTraceExporter({ url: `http://127.0.0.1:${receiver.address().port}/v1/traces` }))] }); + const manager = new AsyncLocalStorageContextManager().enable(); + context.setGlobalContextManager(manager); + trace.setGlobalTracerProvider(provider); + const { AnthropicInstrumentation, isPatched } = require(path.resolve(process.argv[4])); + const instrumentation = new AnthropicInstrumentation({ instrumentationConfig: { enabled: false } }); + instrumentation.setTracerProvider(provider); + instrumentation.enable(); + const original = Anthropic.Messages.prototype.create; + const cases = []; + try { + instrumentation.manuallyInstrument(Anthropic); + if (native) { + assert.equal(Anthropic.Messages.prototype.create, original, 'native SDK must not be monkeypatched'); + assert.equal(isPatched(), false); + assert.equal(Anthropic.fiPatched, undefined); + assert.ok(warnings.some(w => w.includes('traceConfig') && w.includes('native'))); + class DerivedAnthropic extends Anthropic {} + instrumentation.manuallyInstrument(DerivedAnthropic); + assert.equal(Anthropic.Messages.prototype.create, original, 'inherited native capability must also be respected'); + } else { + assert.notEqual(Anthropic.Messages.prototype.create, original); + assert.equal(isPatched(), true); + } + const client = new Anthropic({ apiKey: 'not-a-real-key', baseURL: 'https://offline.invalid', maxRetries: 0, + fetch, ...(native ? { openTelemetry: { tracerProvider: provider, traces: { contentMode: 'metadata_only' } } } : {}) }); + async function run(label, operation, count = 1, status = SpanStatusCode.OK) { + memory.reset(); + wire.length = 0; + await operation(); + await provider.forceFlush(); + const spans = memory.getFinishedSpans(); + assert.equal(spans.length, count, label); + const wireSpans = wire.flatMap(r => r.resourceSpans).flatMap(r => r.scopeSpans).flatMap(s => s.spans); + assert.equal(wireSpans.length, count, label + ' on OTLP wire'); + if (count) assert.equal(spans[0].status.code, status, label + ' status'); + if (native && status === SpanStatusCode.OK) { + const serialized = JSON.stringify(wire); + assert.ok(!serialized.includes('PRIVATE_INPUT_CANARY'), label + ': metadata-only input'); + assert.ok(!serialized.includes('PRIVATE_OUTPUT_CANARY'), label + ': metadata-only output'); + } + cases.push({ label, spans: count, wireSpans: wireSpans.length, ...(count ? { status: spans[0].status.code } : {}) }); + return spans; + } + const spans = await run('await', async () => assert.equal((await client.messages.create(params)).id, message.id)); + assert.equal(spans[0].attributes['gen_ai.usage.input_tokens'], 7); + assert.equal(spans[0].attributes['gen_ai.usage.output_tokens'], 3); + if (!native) { + // Keep the existing supported legacy entry point. No claim of native parity here. + assert.equal(spans[0].attributes['fi.span.kind'], 'LLM'); + } else { + await run('withResponse', async () => { + const promise = client.messages.create(params); + assert.equal(typeof promise.withResponse, 'function'); + const { data, response, request_id } = await promise.withResponse(); + assert.equal(data.id, message.id); + assert.equal(response.status, 200); + assert.equal(request_id, 'req_synthetic'); + }); + await run('asResponse retains unread body', async () => { + const raw = await client.messages.create(params).asResponse(); + assert.equal(raw.bodyUsed, false); + assert.equal((await raw.json()).id, message.id); + }); + await run('stream withResponse', async () => { + const { data: stream } = await client.messages.create({ ...params, stream: true }).withResponse(); + assert.equal(typeof stream.tee, 'function'); + assert.equal(memory.getFinishedSpans().length, 0, 'stream must not be drained in the background'); + const received = []; + for await (const event of stream) received.push(event); + assert.deepEqual(received, events); + }); + await run('stream helper', async () => { + const result = await client.messages.stream(params).finalMessage(); + assert.equal(result.content[0].text, 'PRIVATE_OUTPUT_CANARY'); + }); + await run('stream cancellation', async () => { + const stream = await client.messages.create({ ...params, stream: true }); + for await (const event of stream) { assert.equal(event.type, 'message_start'); break; } + assert.equal(stream.controller.signal.aborted, true); + }, 1, SpanStatusCode.UNSET); + await run('API error remains typed', async () => { + await assert.rejects(client.messages.create({ ...params, model: 'synthetic-error' }), Anthropic.BadRequestError); + }, 1, SpanStatusCode.ERROR); + const errorWireContainsDetail = JSON.stringify(wire).includes('SYNTHETIC_ERROR_DETAIL'); + cases[cases.length - 1].errorWireContainsDetail = errorWireContainsDetail; + await run('native disabled remains disabled', async () => { + const quiet = new Anthropic({ apiKey: 'not-a-real-key', baseURL: 'https://offline.invalid', maxRetries: 0, fetch, openTelemetry: false }); + const result = await quiet.messages.create(params).withResponse(); + assert.equal(result.data.id, message.id); + }, 0); + instrumentation.disable(); + assert.equal(Anthropic.Messages.prototype.create, original); + await run('disable does not alter native SDK', async () => { await client.messages.create(params); }); + } + console.log(JSON.stringify({ status: 'PASS', sdkVersion, native, syntheticRequests: requests, providerCalls: 0, + cases, limits: 'OTLP wire only; no Future AGI backend/UI qualification. Native FI privacy settings are not applied.' }, null, 2)); + } finally { + instrumentation.disable(); + await provider.shutdown(); + manager.disable(); + await new Promise(resolve => receiver.close(resolve)); + } +} +main().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/typescript/packages/traceai_anthropic/src/__tests__/nativeBoundary.test.ts b/typescript/packages/traceai_anthropic/src/__tests__/nativeBoundary.test.ts new file mode 100644 index 000000000..0aebc71c2 --- /dev/null +++ b/typescript/packages/traceai_anthropic/src/__tests__/nativeBoundary.test.ts @@ -0,0 +1,38 @@ +import Anthropic from "@anthropic-ai/sdk"; +import { diag } from "@opentelemetry/api"; +import { AnthropicInstrumentation, isPatched } from "../instrumentation"; + +// The check must never invoke an instance getter on a prototype (SDK getters +// may access private fields). Native telemetry being disabled is still native. +class NativeSDK { + static Messages = class { + create() { return "original"; } + }; + get openTelemetry(): never { throw new Error("do not call a client getter"); } +} +class DerivedSDK extends NativeSDK {} + +describe("native SDK boundary", () => { + it.each([ + ["constructor", NativeSDK], + ["subclass", DerivedSDK], + ["module namespace", { Anthropic: NativeSDK, Messages: NativeSDK.Messages }], + ])("leaves the %s unmodified and warns about privacy configuration", (_label, sdk) => { + const warn = jest.spyOn(diag, "warn").mockImplementation(() => {}); + const original = NativeSDK.Messages.prototype.create; + const instrumentation = new AnthropicInstrumentation(); + try { + instrumentation.manuallyInstrument(sdk as unknown as typeof Anthropic); + instrumentation.manuallyInstrument(sdk as unknown as typeof Anthropic); + expect(NativeSDK.Messages.prototype.create).toBe(original); + expect(isPatched()).toBe(false); + expect(sdk).not.toHaveProperty("fiPatched"); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("traceConfig")); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("native SDK spans")); + } finally { + instrumentation.disable(); + warn.mockRestore(); + } + expect(NativeSDK.Messages.prototype.create).toBe(original); + }); +}); diff --git a/typescript/packages/traceai_anthropic/src/instrumentation.ts b/typescript/packages/traceai_anthropic/src/instrumentation.ts index 4f337f556..8497d0eda 100644 --- a/typescript/packages/traceai_anthropic/src/instrumentation.ts +++ b/typescript/packages/traceai_anthropic/src/instrumentation.ts @@ -83,6 +83,21 @@ export class AnthropicInstrumentation extends InstrumentationBase{ } private patch(moduleExports: typeof Anthropic, moduleVersion?: string) { + // Native SDK tracing owns the APIPromise, stream lifecycle and privacy options. + // Do not wrap it, even when the client explicitly disables native telemetry. + // Check the public capability, not just the automatic loader's version gate: + // manuallyInstrument() bypasses that gate, and subclasses inherit this getter. + const sdkPrototype = moduleExports.prototype ?? moduleExports.Anthropic?.prototype; + if (sdkPrototype && "openTelemetry" in sdkPrototype) { + diag.warn( + "@traceai/anthropic: this SDK has native OpenTelemetry support; " + + "leaving it unpatched. Configure the SDK's openTelemetry option and " + + "tracer provider directly. traceConfig on this legacy instrumentor " + + "does not apply to native SDK spans.", + ); + return moduleExports; + } + if ((moduleExports as any).fiPatched || _isFIPatched) { return moduleExports; } From edcec9ed1572f9054b9265c1633d462c0ec295f8 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 19:05:28 -0700 Subject: [PATCH 2/2] fix(anthropic): instrument the legacy ESM quick start Explicitly instrument the imported legacy SDK so native ESM emits spans. Execute the README in fresh ESM and CommonJS processes against loopback provider and OTLP endpoints in CI, checking exactly one span and its model and token attributes. Co-Authored-By: Claude Code --- .../workflows/anthropic-typescript-tests.yml | 3 + .../packages/traceai_anthropic/README.md | 9 +- .../contract/readme-quickstart.cjs | 116 ++++++++++++++++++ 3 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 typescript/packages/traceai_anthropic/contract/readme-quickstart.cjs diff --git a/.github/workflows/anthropic-typescript-tests.yml b/.github/workflows/anthropic-typescript-tests.yml index cb066a03c..7f3437585 100644 --- a/.github/workflows/anthropic-typescript-tests.yml +++ b/.github/workflows/anthropic-typescript-tests.yml @@ -73,4 +73,7 @@ jobs: "$RUNNER_TEMP/anthropic-contract" "$sdk" \ packages/traceai_anthropic/dist/src/instrumentation.js done + sudo unshare --net sh -c 'ip link set lo up; exec runuser -u runner -- "$@"' sh \ + env -i PATH="$PATH" HOME="$RUNNER_TEMP/anthropic-test-home" \ + "$node_bin" packages/traceai_anthropic/contract/readme-quickstart.cjs git diff --exit-code -- pnpm-lock.yaml diff --git a/typescript/packages/traceai_anthropic/README.md b/typescript/packages/traceai_anthropic/README.md index 4a5b66cc3..7c8a29999 100644 --- a/typescript/packages/traceai_anthropic/README.md +++ b/typescript/packages/traceai_anthropic/README.md @@ -54,6 +54,10 @@ The `module` setting can be `"commonjs"`, `"esnext"`, or any other module system ## Quick Start +Native ESM imports are not intercepted by the CommonJS automatic instrumentation hook. After importing the legacy SDK, call `manuallyInstrument(Anthropic)` before creating a client. This also works in CommonJS when automatic instrumentation has already patched the SDK; it does not add a second wrapper. + +This example uses top-level `await` for ESM. For CommonJS, use `require()` for the imports and run the awaited setup and request inside an async function. + ```typescript // export Future AGI API KEYS // export FI_API_KEY=your_api_key @@ -81,8 +85,9 @@ registerInstrumentations({ tracerProvider: tracerProvider, }); -// 3. Dynamically import Anthropic SDK AFTER instrumentation is registered +// 3. Import and explicitly instrument the legacy SDK (required for native ESM) const Anthropic = (await import("@anthropic-ai/sdk")).default; +anthropicInstrumentation.manuallyInstrument(Anthropic); const client = new Anthropic(); // 4. Use Anthropic as normal @@ -120,7 +125,7 @@ FI_SECRET_KEY=your_secret_key - Support for both streaming and non-streaming requests - Captures token usage and response metadata - Tracks tool usage in conversations -- No manual instrumentation required +- One-time SDK setup; no per-request tracing code required - Integration with TraceAI's observability platform ## Peer Dependencies diff --git a/typescript/packages/traceai_anthropic/contract/readme-quickstart.cjs b/typescript/packages/traceai_anthropic/contract/readme-quickstart.cjs new file mode 100644 index 000000000..ccfcdaea0 --- /dev/null +++ b/typescript/packages/traceai_anthropic/contract/readme-quickstart.cjs @@ -0,0 +1,116 @@ +'use strict'; +// Execute the README, not a copied setup, against real SDKs and loopback HTTP. +// Run after building the workspace, with external networking denied by the caller. +const assert = require('node:assert/strict'); +const { spawn } = require('node:child_process'); +const fs = require('node:fs'); +const http = require('node:http'); +const path = require('node:path'); +const ts = require('typescript'); + +const packageDir = path.resolve(__dirname, '..'); +const readme = fs.readFileSync(path.join(packageDir, 'README.md'), 'utf8'); +const quickStart = readme.split('## Quick Start\n')[1]?.split('\n## ')[0]; +const sample = quickStart?.match(/```typescript\n([\s\S]*?)```/)?.[1]; +assert.ok(sample, 'README must contain the executable Quick Start'); + +async function runSample(filename, url) { + // No inherited credentials, provider settings, or runtime preloads. + const child = spawn(process.execPath, [filename], { + env: { + PATH: process.env.PATH, + HOME: process.env.HOME, + FI_BASE_URL: url, + FI_API_KEY: 'synthetic-fi-key', + FI_SECRET_KEY: 'synthetic-fi-secret', + ANTHROPIC_API_KEY: 'synthetic-anthropic-key', + ANTHROPIC_BASE_URL: url, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', chunk => { stdout += chunk; }); + child.stderr.on('data', chunk => { stderr += chunk; }); + const timer = setTimeout(() => child.kill('SIGKILL'), 30_000); + try { + const code = await new Promise((resolve, reject) => { + child.on('error', reject); + child.on('close', resolve); + }); + assert.equal(code, 0, `${path.basename(filename)} failed: ${stdout}\n${stderr}`); + } finally { + clearTimeout(timer); + } +} + +async function main() { + // Keeping samples beneath the package preserves normal import/require resolution. + const sampleDir = fs.mkdtempSync(path.join(packageDir, '.readme-contract-')); + const requests = []; + const payloads = []; + const unexpected = []; + const receiver = http.createServer((request, response) => { + const chunks = []; + request.on('data', chunk => chunks.push(chunk)); + request.on('end', () => { + response.setHeader('content-type', 'application/json'); + const body = JSON.parse(Buffer.concat(chunks).toString()); + if (request.method === 'POST' && request.url === '/v1/messages') { + requests.push(body); + response.end(JSON.stringify({ + id: 'msg_readme', type: 'message', role: 'assistant', model: body.model, + content: [{ type: 'text', text: 'Synthetic response' }], + stop_reason: 'end_turn', stop_sequence: null, + usage: { input_tokens: 7, output_tokens: 3 }, + })); + } else if (request.method === 'POST' && request.url === '/tracer/v1/traces') { + payloads.push(body); + response.end('{}'); + } else { + unexpected.push(`${request.method} ${request.url}`); + response.writeHead(404).end('{}'); + } + }); + }); + const results = []; + try { + await new Promise(resolve => receiver.listen(0, '127.0.0.1', resolve)); + const url = `http://127.0.0.1:${receiver.address().port}`; + for (const mode of ['esm', 'commonjs']) { + requests.length = payloads.length = unexpected.length = 0; + const esm = mode === 'esm'; + const compiled = ts.transpileModule(sample, { compilerOptions: { + target: ts.ScriptTarget.ES2022, + module: esm ? ts.ModuleKind.ES2022 : ts.ModuleKind.CommonJS, + esModuleInterop: true, + } }).outputText; + const filename = path.join(sampleDir, esm ? 'quickstart.mjs' : 'quickstart.cjs'); + // CommonJS does not support top-level await; only wrap its executable body. + fs.writeFileSync(filename, esm ? compiled : + `(async () => {\n${compiled}\n})().catch(error => { console.error(error); process.exitCode = 1; });\n`); + await runSample(filename, url); + assert.deepEqual(unexpected, [], `${mode}: unexpected HTTP route`); + assert.equal(requests.length, 1, `${mode}: exactly one SDK request`); + const spans = payloads.flatMap(p => p.resourceSpans) + .flatMap(r => r.scopeSpans).flatMap(s => s.spans); + results.push({ mode, requests: requests.length, spans: spans.length }); + // Validate actual exported attributes, not just a successful request. + for (const span of spans) { + const attributes = Object.fromEntries(span.attributes.map(a => [a.key, a.value])); + assert.equal(attributes['fi.span.kind']?.stringValue, 'LLM'); + assert.equal(attributes['gen_ai.request.model']?.stringValue, requests[0].model); + assert.equal(Number(attributes['gen_ai.usage.input_tokens']?.intValue), 7); + assert.equal(Number(attributes['gen_ai.usage.output_tokens']?.intValue), 3); + } + } + console.log(JSON.stringify({ cases: results }, null, 2)); + for (const result of results) { + assert.equal(result.spans, 1, `${result.mode}: README must export exactly one span`); + } + } finally { + await new Promise(resolve => receiver.close(resolve)); + fs.rmSync(sampleDir, { recursive: true, force: true }); + } +} +main().catch(error => { console.error(error); process.exitCode = 1; });