diff --git a/.github/workflows/vertexai-typescript-tests.yml b/.github/workflows/vertexai-typescript-tests.yml new file mode 100644 index 000000000..a644c0940 --- /dev/null +++ b/.github/workflows/vertexai-typescript-tests.yml @@ -0,0 +1,62 @@ +name: Vertex AI TypeScript compatibility + +on: + pull_request: + branches: [dev, main] + paths: + - 'typescript/packages/traceai_vertexai/**' + - 'typescript/packages/fi-core/**' + - 'typescript/packages/fi-semantic-conventions/**' + - 'typescript/pnpm-lock.yaml' + - 'typescript/tsconfig*.json' + - '.github/workflows/vertexai-typescript-tests.yml' + +permissions: + contents: read + +jobs: + compatibility: + name: Vertex SDK ${{ matrix.sdk }} / Node ${{ matrix.node }} + runs-on: macos-15 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + sdk: ['1.0.0', '1.12.0'] + node: ['18.19.0', '22.15.0', '26.8.1'] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22.15.0' + - name: Install frozen build dependencies + run: | + npm install --global pnpm@10.30.1 + pnpm --dir typescript --filter '@traceai/fi-instrumentation-vertexai...' install --ignore-scripts --frozen-lockfile + - name: Build, unit test and pack without outbound networking + run: | + policy="$GITHUB_WORKSPACE/typescript/packages/traceai_vertexai/test/network.sb" + sandbox-exec -f "$policy" pnpm --dir typescript --filter '@traceai/fi-instrumentation-vertexai...' run build + sandbox-exec -f "$policy" pnpm --dir typescript --filter @traceai/fi-instrumentation-vertexai test --runInBand + sandbox-exec -f "$policy" pnpm --dir typescript/packages/traceai_vertexai pack --pack-destination "$RUNNER_TEMP/tarballs" + sandbox-exec -f "$policy" python3 -I .github/release/release_plan.py --check + - name: Install packed adapter with registry dependencies and real SDK + env: + VERTEX_SDK: ${{ matrix.sdk }} + run: | + tarball=$(find "$RUNNER_TEMP/tarballs" -name '*.tgz' -type f) + npm install --prefix "$RUNNER_TEMP/consumer" --ignore-scripts --no-audit --no-fund \ + "$tarball" "@google-cloud/vertexai@$VERTEX_SDK" + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: ${{ matrix.node }} + - name: Verify real SDK, README and OTLP payloads without outbound networking + run: | + export VERTEX_CONSUMER="$RUNNER_TEMP/consumer" + export VERTEX_INSTRUMENTATION="$VERTEX_CONSUMER/node_modules/@traceai/fi-instrumentation-vertexai" + sandbox-exec -f typescript/packages/traceai_vertexai/test/network.sb \ + node --test typescript/packages/traceai_vertexai/test/compat.test.cjs \ + typescript/packages/traceai_vertexai/test/export.test.cjs \ + typescript/packages/traceai_vertexai/test/readme.test.cjs diff --git a/typescript/package.json b/typescript/package.json index 0c2e9d897..91d4ee45b 100644 --- a/typescript/package.json +++ b/typescript/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "pnpm run build:ordered", "build:deps": "pnpm --filter @traceai/fi-core --filter @traceai/fi-semantic-conventions run build", - "build:consumers": "pnpm --filter @traceai/openai --filter @traceai/anthropic --filter @traceai/langchain --filter @traceai/bedrock --filter @traceai/vercel --filter @traceai/chromadb --filter @traceai/pinecone --filter @traceai/qdrant --filter @traceai/weaviate --filter @traceai/milvus --filter @traceai/lancedb --filter @traceai/mongodb --filter @traceai/pgvector --filter @traceai/redis --filter @traceai/together --filter @traceai/huggingface --filter @traceai/vllm --filter @traceai/deepseek --filter @traceai/fireworks --filter @traceai/cerebras --filter @traceai/xai run build", + "build:consumers": "pnpm --filter @traceai/openai --filter @traceai/anthropic --filter @traceai/langchain --filter @traceai/bedrock --filter @traceai/vercel --filter @traceai/chromadb --filter @traceai/pinecone --filter @traceai/qdrant --filter @traceai/weaviate --filter @traceai/milvus --filter @traceai/lancedb --filter @traceai/mongodb --filter @traceai/pgvector --filter @traceai/redis --filter @traceai/together --filter @traceai/huggingface --filter @traceai/vllm --filter @traceai/deepseek --filter @traceai/fireworks --filter @traceai/cerebras --filter @traceai/xai --filter @traceai/fi-instrumentation-vertexai run build", "build:ordered": "pnpm run build:deps && pnpm run build:consumers", "test": "pnpm run -r test", "test:coverage": "pnpm run -r test:coverage", diff --git a/typescript/packages/traceai_vertexai/README.md b/typescript/packages/traceai_vertexai/README.md index a50c0ac33..5d7becf0d 100644 --- a/typescript/packages/traceai_vertexai/README.md +++ b/typescript/packages/traceai_vertexai/README.md @@ -1,64 +1,82 @@ # @traceai/fi-instrumentation-vertexai -OpenTelemetry instrumentation for Google Cloud Vertex AI SDK. Part of traceAI, Future AGI's OpenTelemetry instrumentation libraries. +Trace generation, chat, streaming responses and token-count requests from the legacy `@google-cloud/vertexai` SDK in Future AGI. -[Documentation](https://docs.futureagi.com/docs/integrations/traceai/vertexai) · [Source](https://github.com/future-agi/traceAI/tree/dev/typescript/packages/traceai_vertexai) · [Issues](https://github.com/future-agi/traceAI/issues) +Google has deprecated this SDK in favor of [`@google/genai`](https://github.com/googleapis/js-genai). Use this adapter for existing Vertex AI SDK applications; it does not instrument the modern Google Gen AI SDK. See Google's [migration guide](https://cloud.google.com/vertex-ai/generative-ai/docs/deprecations/genai-vertexai-sdk) when moving an application. ## Install +Use Node.js 18.19+ on the 18.x line, or Node.js 20.6+ (a maintained Node.js release is recommended). + ```sh -npm install @traceai/fi-instrumentation-vertexai @traceai/fi-core @opentelemetry/instrumentation @opentelemetry/api@^1.9.0 @opentelemetry/context-async-hooks@^2.0.1 -npm install @google-cloud/vertexai@"^1.0.0" +npm install @traceai/fi-instrumentation-vertexai @traceai/fi-core @google-cloud/vertexai@"^1.0.0" @opentelemetry/api@"^1.9.0" @opentelemetry/context-async-hooks@"^2.0.1" ``` -The SDK ranges above match this instrumentor's module-version filters. Install -only the SDKs your application uses. For Future AGI export, set `FI_API_KEY` and -`FI_SECRET_KEY` in your environment; keep provider credentials outside source code. - -## Package entry points - -This source revision fixes the package entry points. Both CommonJS `require()` -and native Node ESM `import` use the same CommonJS implementation; this avoids -broken, extensionless ESM build imports. The corrected packages still need a -patch release. A previously published `0.1.0` tarball is not changed by this README. +Configure Google Application Default Credentials and set `GOOGLE_CLOUD_PROJECT` for the Google project where Vertex AI is enabled. Set `FI_API_KEY` and `FI_SECRET_KEY` for Future AGI. Keep credentials outside your source code. -## Quick start +## Trace a request -Load this setup **before** loading the SDK you want to instrument. This CommonJS -example registers tracing; it does not make a provider request. +Save this as `vertex.cjs`. Manual instrumentation works when the SDK is already loaded, so the example does not depend on import order. ```javascript const { context } = require("@opentelemetry/api"); const { AsyncLocalStorageContextManager } = require("@opentelemetry/context-async-hooks"); const { register, ProjectType } = require("@traceai/fi-core"); -// Standalone process only: if your OTel SDK already installs a context manager, -// reuse it instead of registering another one. -context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable()); -const { registerInstrumentations } = require("@opentelemetry/instrumentation"); const { VertexAIInstrumentation } = require("@traceai/fi-instrumentation-vertexai"); -const tracerProvider = register({ - projectName: "vertexai-app", - projectType: ProjectType.OBSERVE, -}); -registerInstrumentations({ - tracerProvider, instrumentations: [new VertexAIInstrumentation()], +const vertexai = require("@google-cloud/vertexai"); + +// In an application with an existing OTel SDK, reuse its context manager. +context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable()); +const provider = register({ projectName: "vertexai-app", projectType: ProjectType.OBSERVE }); +const instrumentation = new VertexAIInstrumentation(); +instrumentation.setTracerProvider(provider); +instrumentation.manuallyInstrument(vertexai); + +async function main() { + try { + const vertex = new vertexai.VertexAI({ + project: process.env.GOOGLE_CLOUD_PROJECT, + location: process.env.GOOGLE_CLOUD_LOCATION || "us-central1", + }); + const model = vertex.getGenerativeModel({ model: "gemini-2.5-flash" }); + const result = await model.generateContent("Explain why the sky looks blue in one sentence."); + console.log(result.response.candidates?.[0]?.content?.parts); + } finally { + instrumentation.disable(); + await provider.shutdown(); + } +} + +main().catch(console.error); +``` + +Run `node vertex.cjs`, then open the **vertexai-app** project in Future AGI to inspect the request and response. `register()` exports standard OTLP to `https://api.futureagi.com/tracer/v1/traces` using your Future AGI credentials. + +CommonJS applications can alternatively register the instrumentation before requiring the SDK to use automatic module hooks. For native ESM, import the SDK namespace and pass it to `manuallyInstrument()`; do not rely on static import ordering for automatic hooks. + +## Capture and privacy + +The adapter covers the stable and preview `GenerativeModel` and `ChatSession` APIs in SDK versions `>=1.0.0 <2.0.0`. It records model, generation settings, chat history, text and tool content, finish reasons, and usage returned by the SDK. It does not add embedding or cached-content management APIs that these generation hooks do not implement. + +Streaming spans finish when the SDK's existing `response` promise settles. You can consume `result.stream`, await `result.response`, or do both; the adapter does not consume your iterator or replace either object. Breaking out of iteration does not cancel the SDK's independent aggregation. Configure the SDK's request timeout when you need to limit request duration. + +Content capture is bounded. Raw request/response bodies, media bytes, file URIs, and provider error messages or stacks are not exported. Text, function arguments and function results may still contain sensitive information. To disable content capture, construct the instrumentation with: + +```javascript +const instrumentation = new VertexAIInstrumentation({ + traceConfig: { hideInputs: true, hideOutputs: true }, }); ``` -`register()` installs a tracer provider, not an async context manager. The -explicit context manager above keeps nested asynchronous calls in one trace. -Run your application's SDK calls after this setup. Call -`await tracerProvider.shutdown()` when the application finishes to flush pending -spans. For ESM, follow the documentation for your OpenTelemetry loader and SDK -instead of assuming import order alone enables automatic patching. +The shared FI controls also support input/output message hiding, text hiding, PII redaction, and the corresponding `FI_HIDE_*` environment variables. Content is kept in the standard `gen_ai.input.messages` and `gen_ai.output.messages` attributes so granular text controls also apply to function data. + +Calling `disable()` restores this instance's automatic and manual hooks. Requests already in progress still complete their spans. Do not register an additional instrumentor for the same SDK methods. -## Compatibility and privacy +## Troubleshooting -This instruments the SDK hooks implemented by this package, not every API offered -by the provider. The install ranges above are intentionally limited to the module versions this -instrumentor patches; its peer declarations may be broader. -Traces can contain prompts, responses, and tool data; configure capture and -redaction before sending production traffic. +- **No spans:** pass the provider returned by `register()` to `setTracerProvider()` and pass the actual SDK module to `manuallyInstrument()` before making requests. +- **Authentication or model errors:** check your Google credentials, project, regional model availability and SDK response. Provider errors retain their original behavior in your application; spans record failure without copying provider error text. +- **Missing stream spans at shutdown:** await the SDK's `result.response` before shutting down the tracer provider. ## License diff --git a/typescript/packages/traceai_vertexai/package.json b/typescript/packages/traceai_vertexai/package.json index 3f08496b9..18a16d970 100644 --- a/typescript/packages/traceai_vertexai/package.json +++ b/typescript/packages/traceai_vertexai/package.json @@ -20,6 +20,7 @@ "clean": "rimraf dist", "lint": "eslint src --ext .ts", "test": "jest", + "test:compat": "node --test test/compat.test.cjs test/export.test.cjs test/readme.test.cjs", "test:watch": "jest --watch", "test:coverage": "jest --coverage", "prepublishOnly": "npm run clean && npm run build" @@ -63,7 +64,7 @@ "typescript": "^5.3.0" }, "peerDependencies": { - "@google-cloud/vertexai": ">=1.0.0" + "@google-cloud/vertexai": ">=1.0.0 <2.0.0" }, "peerDependenciesMeta": { "@google-cloud/vertexai": { @@ -71,7 +72,7 @@ } }, "engines": { - "node": ">=18.0.0" + "node": "^18.19.0 || >=20.6.0" }, "homepage": "https://docs.futureagi.com/docs/integrations/traceai/vertexai", "bugs": { diff --git a/typescript/packages/traceai_vertexai/src/__tests__/instrumentation.test.ts b/typescript/packages/traceai_vertexai/src/__tests__/instrumentation.test.ts index 15460d801..bbf78d58e 100644 --- a/typescript/packages/traceai_vertexai/src/__tests__/instrumentation.test.ts +++ b/typescript/packages/traceai_vertexai/src/__tests__/instrumentation.test.ts @@ -1,304 +1,80 @@ -/** - * Tests for Vertex AI instrumentation. - */ - -import { VertexAIInstrumentation, isPatched, _resetPatchedStateForTesting } from "../instrumentation"; +import { VertexAIInstrumentation, isPatched } from "../instrumentation"; import { VERSION } from "../version"; -import { InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; -import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node"; - -describe("VertexAIInstrumentation", () => { - const memoryExporter = new InMemorySpanExporter(); - const tracerProvider = new NodeTracerProvider({ - spanProcessors: [new SimpleSpanProcessor(memoryExporter)] - }); - tracerProvider.register(); - - const instrumentation = new VertexAIInstrumentation(); - instrumentation.disable(); - instrumentation.setTracerProvider(tracerProvider); - - beforeEach(() => { - memoryExporter.reset(); - _resetPatchedStateForTesting(); - }); - - afterAll(() => { - instrumentation.disable(); - }); - - describe("VERSION", () => { - it("should export VERSION constant", () => { - expect(VERSION).toBe(require("../../package.json").version); - }); - }); - - describe("VertexAIInstrumentation", () => { - it("should create instrumentation instance", () => { - const inst = new VertexAIInstrumentation(); - expect(inst).toBeInstanceOf(VertexAIInstrumentation); - }); - - it("should accept configuration options", () => { - const inst = new VertexAIInstrumentation({ - traceConfig: { - hideInputs: true, - hideOutputs: true, - }, - }); - expect(inst).toBeInstanceOf(VertexAIInstrumentation); - }); - - it("should have instrumentationName", () => { - const inst = new VertexAIInstrumentation(); - expect(inst.instrumentationName).toBe("@traceai/fi-instrumentation-vertexai"); - }); - - it("should have instrumentationVersion", () => { - const inst = new VertexAIInstrumentation(); - expect(inst.instrumentationVersion).toBe(VERSION); - }); - }); - - describe("isPatched", () => { - it("should return false when not patched", () => { - instrumentation.disable(); - expect(isPatched()).toBe(false); - }); - }); - - describe("manuallyInstrument", () => { - it("should handle null module", () => { - const inst = new VertexAIInstrumentation(); - inst.setTracerProvider(tracerProvider); - inst.enable(); - - // Should not throw - inst.manuallyInstrument(null); - expect(true).toBe(true); - }); - - it("should mark module as patched after manual instrumentation", () => { - const inst = new VertexAIInstrumentation(); - inst.setTracerProvider(tracerProvider); - inst.enable(); - - const mockModule = { GenerativeModel: {} }; - inst.manuallyInstrument(mockModule); - expect(mockModule).toHaveProperty("_fiPatched", true); - }); - }); - - describe("enable/disable", () => { - it("should enable instrumentation", () => { - const inst = new VertexAIInstrumentation(); - inst.setTracerProvider(tracerProvider); - inst.enable(); - // Should not throw - expect(true).toBe(true); - }); - - it("should disable instrumentation", () => { - const inst = new VertexAIInstrumentation(); - inst.setTracerProvider(tracerProvider); - inst.enable(); - inst.disable(); - // Should not throw - expect(true).toBe(true); - }); - }); -}); - -describe("GenerativeModel Patching", () => { - const memoryExporter = new InMemorySpanExporter(); - const tracerProvider = new NodeTracerProvider({ - spanProcessors: [new SimpleSpanProcessor(memoryExporter)] - }); - tracerProvider.register(); - - beforeEach(() => { - memoryExporter.reset(); - _resetPatchedStateForTesting(); - }); - - it("should wrap GenerativeModel.generateContent when available", () => { - const inst = new VertexAIInstrumentation(); - inst.setTracerProvider(tracerProvider); - inst.enable(); - - const originalGenerate = jest.fn().mockResolvedValue({ - response: { - candidates: [{ content: { role: "model", parts: [{ text: "Hello!" }] } }], - usageMetadata: { promptTokenCount: 10, candidatesTokenCount: 5, totalTokenCount: 15 }, - }, - }); - - const mockModule = { - GenerativeModel: { - prototype: { - generateContent: originalGenerate, - }, - }, - }; - - inst.manuallyInstrument(mockModule); - expect((mockModule as any)._fiPatched).toBe(true); - }); - - it("should wrap GenerativeModel.generateContentStream when available", () => { - const inst = new VertexAIInstrumentation(); - inst.setTracerProvider(tracerProvider); - inst.enable(); - - const originalStream = jest.fn().mockResolvedValue({ - stream: (async function* () { - yield { candidates: [{ content: { parts: [{ text: "Hello" }] } }] }; - })(), - response: Promise.resolve({}), - }); - - const mockModule = { - GenerativeModel: { - prototype: { - generateContentStream: originalStream, - }, - }, - }; - - inst.manuallyInstrument(mockModule); - expect((mockModule as any)._fiPatched).toBe(true); - }); - - it("should wrap GenerativeModel.countTokens when available", () => { - const inst = new VertexAIInstrumentation(); - inst.setTracerProvider(tracerProvider); - inst.enable(); - - const originalCount = jest.fn().mockResolvedValue({ totalTokens: 100 }); - - const mockModule = { - GenerativeModel: { - prototype: { - countTokens: originalCount, - }, - }, - }; - - inst.manuallyInstrument(mockModule); - expect((mockModule as any)._fiPatched).toBe(true); - }); - - it("should not double-patch module", () => { - const inst = new VertexAIInstrumentation(); - inst.setTracerProvider(tracerProvider); - inst.enable(); - - const mockModule = { - _fiPatched: true, - GenerativeModel: { - prototype: { - generateContent: jest.fn(), - }, - }, - }; - - const original = mockModule.GenerativeModel.prototype.generateContent; - inst.manuallyInstrument(mockModule); - - // Should not have changed - expect(mockModule.GenerativeModel.prototype.generateContent).toBe(original); - }); -}); - -describe("ChatSession Patching", () => { - const memoryExporter = new InMemorySpanExporter(); - const tracerProvider = new NodeTracerProvider({ - spanProcessors: [new SimpleSpanProcessor(memoryExporter)] - }); - tracerProvider.register(); - - beforeEach(() => { - memoryExporter.reset(); - _resetPatchedStateForTesting(); - }); - - it("should wrap ChatSession.sendMessage when available", () => { - const inst = new VertexAIInstrumentation(); - inst.setTracerProvider(tracerProvider); - inst.enable(); - - const originalSend = jest.fn().mockResolvedValue({ - response: { - candidates: [{ content: { role: "model", parts: [{ text: "Response" }] } }], - }, - }); - - const mockModule = { - ChatSession: { - prototype: { - sendMessage: originalSend, - }, - }, - }; - - inst.manuallyInstrument(mockModule); - expect((mockModule as any)._fiPatched).toBe(true); - }); - - it("should wrap ChatSession.sendMessageStream when available", () => { - const inst = new VertexAIInstrumentation(); - inst.setTracerProvider(tracerProvider); - inst.enable(); - - const originalStream = jest.fn().mockResolvedValue({ - stream: (async function* () { - yield { candidates: [{ content: { parts: [{ text: "Streaming" }] } }] }; - })(), - }); - - const mockModule = { - ChatSession: { - prototype: { - sendMessageStream: originalStream, - }, - }, - }; - - inst.manuallyInstrument(mockModule); - expect((mockModule as any)._fiPatched).toBe(true); - }); -}); - -describe("TextEmbeddingModel Patching", () => { - const memoryExporter = new InMemorySpanExporter(); - const tracerProvider = new NodeTracerProvider({ - spanProcessors: [new SimpleSpanProcessor(memoryExporter)] - }); - tracerProvider.register(); - - beforeEach(() => { - memoryExporter.reset(); - _resetPatchedStateForTesting(); - }); - - it("should wrap TextEmbeddingModel.embed when available", () => { - const inst = new VertexAIInstrumentation(); - inst.setTracerProvider(tracerProvider); - inst.enable(); - - const originalEmbed = jest.fn().mockResolvedValue({ - embeddings: [{ values: [0.1, 0.2, 0.3] }], - }); - - const mockModule = { - TextEmbeddingModel: { - prototype: { - embed: originalEmbed, - }, - }, - }; - inst.manuallyInstrument(mockModule); - expect((mockModule as any)._fiPatched).toBe(true); +describe("VertexAIInstrumentation ownership", () => { + const instances: VertexAIInstrumentation[] = []; + const instrument = () => { + const instance = new VertexAIInstrumentation(); + instances.push(instance); + return instance; + }; + const module = () => ({ GenerativeModel: { prototype: { generateContent: jest.fn().mockResolvedValue({ response: {} }) } } }); + afterEach(() => { for (const instance of instances.splice(0)) instance.disable(); }); + + it("exports the package version", () => { + expect(VERSION).toBe(require("../../package.json").version); + expect(instrument().instrumentationVersion).toBe(VERSION); + }); + it("does not mark null or unsupported modules as patched", () => { + const instance = instrument(); + instance.manuallyInstrument(null); + instance.manuallyInstrument({}); + expect(isPatched()).toBe(false); + }); + it("patches multiple separately loaded copies instead of using a global gate", () => { + const instance = instrument(); + const first = module(), second = module(); + const a = first.GenerativeModel.prototype.generateContent, b = second.GenerativeModel.prototype.generateContent; + instance.manuallyInstrument(first); + instance.manuallyInstrument(second); + expect(first.GenerativeModel.prototype.generateContent).not.toBe(a); + expect(second.GenerativeModel.prototype.generateContent).not.toBe(b); + instance.disable(); + expect(first.GenerativeModel.prototype.generateContent).toBe(a); + expect(second.GenerativeModel.prototype.generateContent).toBe(b); + expect(isPatched()).toBe(false); + }); + it("does not steal another instance's wrappers or remove them on disable", () => { + const first = instrument(), second = instrument(), sdk = module(); + const original = sdk.GenerativeModel.prototype.generateContent; + first.manuallyInstrument(sdk); + const wrapped = sdk.GenerativeModel.prototype.generateContent; + second.manuallyInstrument(sdk); + second.disable(); + expect(sdk.GenerativeModel.prototype.generateContent).toBe(wrapped); + first.disable(); + expect(sdk.GenerativeModel.prototype.generateContent).toBe(original); + }); + it("can patch a frozen module namespace without adding private exports", () => { + const sdk = Object.freeze(module()); + const original = sdk.GenerativeModel.prototype.generateContent; + const instance = instrument(); + instance.manuallyInstrument(sdk); + expect(sdk.GenerativeModel.prototype.generateContent).not.toBe(original); + instance.disable(); + expect(sdk.GenerativeModel.prototype.generateContent).toBe(original); + }); + it("does not wrap the same classes twice through an ESM namespace alias", () => { + const instance = instrument(), sdk = module(); + instance.manuallyInstrument(sdk); + const wrapped = sdk.GenerativeModel.prototype.generateContent; + instance.manuallyInstrument(Object.freeze({ ...sdk })); + expect(sdk.GenerativeModel.prototype.generateContent).toBe(wrapped); + }); + it("does not patch while disabled", () => { + const instance = instrument(), sdk = module(); + const original = sdk.GenerativeModel.prototype.generateContent; + instance.disable(); + instance.manuallyInstrument(sdk); + expect(sdk.GenerativeModel.prototype.generateContent).toBe(original); + }); + it("preserves SDK success and error identity if telemetry throws", async () => { + const instance = instrument(), sdk = module(); + const result = { response: {} }, error = new Error("synthetic"); + sdk.GenerativeModel.prototype.generateContent.mockResolvedValueOnce(result).mockRejectedValueOnce(error); + instance.manuallyInstrument(sdk); + (instance as any).fiTracer.startSpan = () => { throw new Error("telemetry failure"); }; + await expect(sdk.GenerativeModel.prototype.generateContent()).resolves.toBe(result); + await expect(sdk.GenerativeModel.prototype.generateContent()).rejects.toBe(error); }); }); diff --git a/typescript/packages/traceai_vertexai/src/instrumentation.ts b/typescript/packages/traceai_vertexai/src/instrumentation.ts index 6d70fbeb8..2d029a04c 100644 --- a/typescript/packages/traceai_vertexai/src/instrumentation.ts +++ b/typescript/packages/traceai_vertexai/src/instrumentation.ts @@ -1,28 +1,12 @@ -/** - * OpenTelemetry instrumentation for Google Cloud Vertex AI. - * - * Vertex AI provides access to Google's generative AI models including Gemini. - * This instrumentation captures model generation, chat sessions, embeddings, - * and streaming responses. - */ - import { InstrumentationBase, InstrumentationNodeModuleDefinition, } from "@opentelemetry/instrumentation"; import { - Span, - SpanKind, - SpanStatusCode, - context, - trace, + Attributes, Span, SpanKind, SpanStatusCode, TracerProvider, context, diag, trace, } from "@opentelemetry/api"; -import { FITracer, TraceConfigOptions, safelyJSONStringify } from "@traceai/fi-core"; -import { - SemanticConventions, - FISpanKind, - MimeType, -} from "@traceai/fi-semantic-conventions"; +import { FITracer, TraceConfigOptions, getAttributesFromContext } from "@traceai/fi-core"; +import { SemanticConventions as S, FISpanKind } from "@traceai/fi-semantic-conventions"; import { VERSION } from "./version"; export interface VertexAIInstrumentationConfig { @@ -30,609 +14,317 @@ export interface VertexAIInstrumentationConfig { traceConfig?: TraceConfigOptions; } -/** - * Vertex AI Instrumentation class. - * - * Provides automatic instrumentation for the @google-cloud/vertexai SDK. - */ +type SDKObject = Record; +type Method = (...args: any[]) => any; +type Patch = { target: SDKObject; method: string; original: Method; wrapped: Method }; +const owners = new WeakMap(); +const wrappedMethods = new WeakSet(); +let patchedModules = 0; +const methods: Record = { + GenerativeModel: ["generateContent", "generateContentStream", "countTokens"], + GenerativeModelPreview: ["generateContent", "generateContentStream", "countTokens"], + ChatSession: ["sendMessage", "sendMessageStream"], + ChatSessionPreview: ["sendMessage", "sendMessageStream"], +}; + +/** Compatibility instrumentation for the legacy @google-cloud/vertexai 1.x SDK. */ export class VertexAIInstrumentation extends InstrumentationBase { private fiTracer!: FITracer; - private _traceConfig?: TraceConfigOptions; + private traceConfig?: TraceConfigOptions; + private patches = new Map(); constructor(config: VertexAIInstrumentationConfig = {}) { - super( - "@traceai/fi-instrumentation-vertexai", - VERSION, - config.instrumentationConfig || {} - ); - this._traceConfig = config.traceConfig; + super("@traceai/fi-instrumentation-vertexai", VERSION, config.instrumentationConfig || {}); + this.traceConfig = config.traceConfig; + // InstrumentationBase calls enable() before subclass fields are initialized. + this.refreshTracer(); + } + + private refreshTracer(): void { + this.fiTracer = new FITracer({ tracer: this.tracer, traceConfig: this.traceConfig }); + } + + override setTracerProvider(provider: TracerProvider): void { + super.setTracerProvider(provider); + this.refreshTracer(); } override enable(): void { super.enable(); - this.fiTracer = new FITracer({ - tracer: this.tracer, - traceConfig: this._traceConfig, - }); + this.refreshTracer(); + } + + override disable(): void { + super.disable(); + // OTel only knows about automatic hooks; explicitly restore manual patches too. + for (const module of this.patches?.keys() || []) this.unpatch(module); } protected init() { - const module = new InstrumentationNodeModuleDefinition( - "@google-cloud/vertexai", - ["^1.0.0"], - this.patch.bind(this), - this.unpatch.bind(this) + return new InstrumentationNodeModuleDefinition( + "@google-cloud/vertexai", [">=1.0.0 <2.0.0"], + this.patch.bind(this), this.unpatch.bind(this), ); - return module; } - /** - * Manually instrument the @google-cloud/vertexai module. - */ manuallyInstrument(vertexModule: any): void { - if (vertexModule == null) return; - this.patch(vertexModule); + if (vertexModule != null && this.isEnabled()) this.patch(vertexModule); } - private patch(vertexModule: any & { _fiPatched?: boolean }): any { - if (vertexModule?._fiPatched || _isFIPatched) { - return vertexModule; - } - - const instrumentation = this; - - // Wrap GenerativeModel.generateContent - if (vertexModule.GenerativeModel?.prototype?.generateContent) { - this._wrap( - vertexModule.GenerativeModel.prototype, - "generateContent", - (original: Function) => { - return function patchedGenerateContent(this: any, ...args: any[]) { - return instrumentation.traceGenerateContent(original, this, args); - }; - } - ); - } - - // Wrap GenerativeModel.generateContentStream - if (vertexModule.GenerativeModel?.prototype?.generateContentStream) { - this._wrap( - vertexModule.GenerativeModel.prototype, - "generateContentStream", - (original: Function) => { - return function patchedGenerateContentStream(this: any, ...args: any[]) { - return instrumentation.traceGenerateContentStream(original, this, args); - }; - } - ); - } - - // Wrap ChatSession.sendMessage - if (vertexModule.ChatSession?.prototype?.sendMessage) { - this._wrap( - vertexModule.ChatSession.prototype, - "sendMessage", - (original: Function) => { - return function patchedSendMessage(this: any, ...args: any[]) { - return instrumentation.traceSendMessage(original, this, args); - }; - } - ); - } - - // Wrap ChatSession.sendMessageStream - if (vertexModule.ChatSession?.prototype?.sendMessageStream) { - this._wrap( - vertexModule.ChatSession.prototype, - "sendMessageStream", - (original: Function) => { - return function patchedSendMessageStream(this: any, ...args: any[]) { - return instrumentation.traceSendMessageStream(original, this, args); - }; - } - ); - } - - // Wrap GenerativeModel.countTokens - if (vertexModule.GenerativeModel?.prototype?.countTokens) { - this._wrap( - vertexModule.GenerativeModel.prototype, - "countTokens", - (original: Function) => { - return function patchedCountTokens(this: any, ...args: any[]) { - return instrumentation.traceCountTokens(original, this, args); - }; - } - ); + private patch(module: SDKObject): SDKObject { + if (owners.has(module)) return module; + const patches: Patch[] = []; + for (const [name, names] of Object.entries(methods)) { + const target = module[name]?.prototype; + if (!target) continue; + for (const method of names) { + const original = target[method]; + if (typeof original !== "function" || wrappedMethods.has(original)) continue; + const instrumentation = this; + const wrapped = function(this: SDKObject, ...args: any[]) { + return instrumentation.invoke(original, this, args, method); + }; + target[method] = wrapped; + wrappedMethods.add(wrapped); + patches.push({ target, method, original, wrapped }); + } } - - // Wrap TextEmbeddingModel.embed (if available) - if (vertexModule.TextEmbeddingModel?.prototype?.embed) { - this._wrap( - vertexModule.TextEmbeddingModel.prototype, - "embed", - (original: Function) => { - return function patchedEmbed(this: any, ...args: any[]) { - return instrumentation.traceEmbed(original, this, args); - }; - } - ); + if (patches.length) { + owners.set(module, this); + this.patches.set(module, patches); + patchedModules++; } - - vertexModule._fiPatched = true; - _isFIPatched = true; - return vertexModule; + return module; } - private unpatch(vertexModule: any & { _fiPatched?: boolean }): void { - if (vertexModule?.GenerativeModel?.prototype?.generateContent) { - this._unwrap(vertexModule.GenerativeModel.prototype, "generateContent"); - } - if (vertexModule?.GenerativeModel?.prototype?.generateContentStream) { - this._unwrap(vertexModule.GenerativeModel.prototype, "generateContentStream"); - } - if (vertexModule?.ChatSession?.prototype?.sendMessage) { - this._unwrap(vertexModule.ChatSession.prototype, "sendMessage"); - } - if (vertexModule?.ChatSession?.prototype?.sendMessageStream) { - this._unwrap(vertexModule.ChatSession.prototype, "sendMessageStream"); - } - if (vertexModule?.GenerativeModel?.prototype?.countTokens) { - this._unwrap(vertexModule.GenerativeModel.prototype, "countTokens"); - } - if (vertexModule?.TextEmbeddingModel?.prototype?.embed) { - this._unwrap(vertexModule.TextEmbeddingModel.prototype, "embed"); + private unpatch(module: object): void { + if (owners.get(module) !== this) return; + for (const patch of this.patches.get(module) || []) { + // Do not remove another instrumentor's later wrapper. + if (patch.target[patch.method] === patch.wrapped) patch.target[patch.method] = patch.original; } - if (vertexModule) { - vertexModule._fiPatched = false; - } - _isFIPatched = false; + this.patches.delete(module); + owners.delete(module); + patchedModules--; } - private async traceGenerateContent( - original: Function, - instance: any, - args: any[] - ): Promise { - const request = args[0] || {}; - const modelName = instance.model || instance._model || "unknown"; - - const span = this.fiTracer.startSpan(`Vertex AI Generate Content`, { - kind: SpanKind.CLIENT, - attributes: { - [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, - [SemanticConventions.LLM_PROVIDER]: "google", - [SemanticConventions.LLM_MODEL_NAME]: modelName, - [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", - [SemanticConventions.INPUT_VALUE]: safeJsonStringify(request), - [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, - [SemanticConventions.RAW_INPUT]: safeJsonStringify(request), - ...this.getInputMessagesAttributes(request), - }, - }); - - const execContext = trace.setSpan(context.active(), span); - + private async invoke(original: Method, instance: SDKObject, args: any[], method: string): Promise { + if (!this.isEnabled()) return original.apply(instance, args); + const streaming = method.endsWith("Stream"); + const counting = method === "countTokens"; + let span: Span | undefined; try { - const result = await context.with(execContext, () => { - return original.apply(instance, args); + const model = instance.model || (instance.resourcePath ?? instance.publisherModelEndpoint)?.split("/models/")[1] || "unknown"; + span = this.fiTracer.startSpan(`Vertex AI ${method}`, { + kind: SpanKind.CLIENT, + attributes: { + ...getAttributesFromContext(context.active()), + [S.FI_SPAN_KIND]: FISpanKind.LLM, + [S.LLM_PROVIDER]: "google", + [S.LLM_MODEL_NAME]: model, + [S.GEN_AI_OPERATION_NAME]: counting ? "count_tokens" : "chat", + "vertexai.streaming": streaming, + ...inputAttributes(instance, args[0], method), + }, }); - - this.setResponseAttributes(span, result); - span.setStatus({ code: SpanStatusCode.OK }); - span.end(); - - return result; - } catch (error) { - span.recordException(error as Error); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: (error as Error).message, - }); - span.end(); - throw error; + } catch { + diag.warn("Vertex AI input telemetry failed"); } - } - - private async traceGenerateContentStream( - original: Function, - instance: any, - args: any[] - ): Promise { - const request = args[0] || {}; - const modelName = instance.model || instance._model || "unknown"; - - const span = this.fiTracer.startSpan(`Vertex AI Generate Content Stream`, { - kind: SpanKind.CLIENT, - attributes: { - [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, - [SemanticConventions.LLM_PROVIDER]: "google", - [SemanticConventions.LLM_MODEL_NAME]: modelName, - [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", - [SemanticConventions.INPUT_VALUE]: safeJsonStringify(request), - [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, - [SemanticConventions.RAW_INPUT]: safeJsonStringify(request), - "vertexai.streaming": true, - ...this.getInputMessagesAttributes(request), - }, - }); - - const execContext = trace.setSpan(context.active(), span); - + const activeSpan = span; + const executionContext = activeSpan ? trace.setSpan(context.active(), activeSpan) : context.active(); + let result: any; try { - const result = await context.with(execContext, () => { - return original.apply(instance, args); - }); - - // Wrap the stream to capture output - return this.wrapStream(result, span); + result = await context.with(executionContext, () => original.apply(instance, args)); } catch (error) { - span.recordException(error as Error); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: (error as Error).message, - }); - span.end(); + if (activeSpan) finish(activeSpan, undefined, true, counting); throw error; } + if (activeSpan) { + if (streaming && result?.response?.then) { + // The SDK eagerly reads an independent tee into response, even if the caller + // never iterates (or breaks early). Observe that existing promise, not the + // caller's iterator: no extra buffering, reads, promises or history mutation. + void result.response.then( + (response: SDKObject) => finish(activeSpan, response, false, counting), + () => finish(activeSpan, undefined, true, counting), + ); + } else { + finish(activeSpan, result?.response || result, false, counting); + } + } + return result; } +} - private async traceSendMessage( - original: Function, - instance: any, - args: any[] - ): Promise { - const message = args[0]; - const modelName = instance._model?.model || instance._model?._model || "unknown"; - - const inputContent = typeof message === "string" ? message : safeJsonStringify(message); - const span = this.fiTracer.startSpan(`Vertex AI Chat Send Message`, { - kind: SpanKind.CLIENT, - attributes: { - [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, - [SemanticConventions.LLM_PROVIDER]: "google", - [SemanticConventions.LLM_MODEL_NAME]: modelName, - [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", - [SemanticConventions.INPUT_VALUE]: safeJsonStringify(message), - [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, - [SemanticConventions.LLM_INPUT_MESSAGES]: safelyJSONStringify([{ role: "user", content: inputContent }]) ?? "[]", - }, - }); - - const execContext = trace.setSpan(context.active(), span); - - try { - const result = await context.with(execContext, () => { - return original.apply(instance, args); - }); - - this.setResponseAttributes(span, result); +function finish(span: Span, response: SDKObject | undefined, failed: boolean, counting: boolean): void { + try { + if (failed) { + // Provider errors can contain prompts, signed URLs and credentials. Never + // copy their message, stack or response body, including in default capture. + span.setAttribute("error.type", "vertexai.request_failed"); + span.setStatus({ code: SpanStatusCode.ERROR }); + } else { + if (response) span.setAttributes(outputAttributes(response, counting)); span.setStatus({ code: SpanStatusCode.OK }); - span.end(); - - return result; - } catch (error) { - span.recordException(error as Error); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: (error as Error).message, - }); - span.end(); - throw error; } + } catch { + diag.warn("Vertex AI output telemetry failed"); + } finally { + try { span.end(); } catch { diag.warn("Vertex AI span completion failed"); } } +} - private async traceSendMessageStream( - original: Function, - instance: any, - args: any[] - ): Promise { - const message = args[0]; - const modelName = instance._model?.model || instance._model?._model || "unknown"; - - const streamInputContent = typeof message === "string" ? message : safeJsonStringify(message); - const span = this.fiTracer.startSpan(`Vertex AI Chat Send Message Stream`, { - kind: SpanKind.CLIENT, - attributes: { - [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, - [SemanticConventions.LLM_PROVIDER]: "google", - [SemanticConventions.LLM_MODEL_NAME]: modelName, - [SemanticConventions.GEN_AI_OPERATION_NAME]: "chat", - [SemanticConventions.INPUT_VALUE]: safeJsonStringify(message), - [SemanticConventions.INPUT_MIME_TYPE]: MimeType.JSON, - "vertexai.streaming": true, - [SemanticConventions.LLM_INPUT_MESSAGES]: safelyJSONStringify([{ role: "user", content: streamInputContent }]) ?? "[]", - }, - }); - - const execContext = trace.setSpan(context.active(), span); - - try { - const result = await context.with(execContext, () => { - return original.apply(instance, args); - }); - - return this.wrapStream(result, span); - } catch (error) { - span.recordException(error as Error); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: (error as Error).message, - }); - span.end(); - throw error; - } +function inputAttributes(instance: SDKObject, request: any, method: string): Attributes { + const chat = method.startsWith("sendMessage"); + const contents = chat + ? [...(instance.historyInternal || []).slice(-32), { role: "user", parts: typeof request === "string" ? [{ text: request }] : request }] + : typeof request === "string" ? [{ role: "user", parts: [{ text: request }] }] : request?.contents || []; + const attributes: Attributes = {}; + if (contents.length > 32 || (chat && instance.historyInternal?.length > 31)) attributes["vertexai.capture.truncated"] = true; + const messages = contents.slice(-32).map((content: SDKObject) => message(content, attributes)); + // countTokens sends the explicit request verbatim; model generation defaults + // are not part of the counted payload, even when the model has them configured. + const counting = method === "countTokens"; + const instruction = counting ? request?.systemInstruction : (request?.systemInstruction ?? instance.systemInstruction); + if (instruction) messages.unshift(message(typeof instruction === "string" ? { role: "system", parts: [{ text: instruction }] } : { ...instruction, role: "system" }, attributes)); + const tools = counting ? request?.tools : (request?.tools ?? instance.tools); + // Keep all content-bearing values inside message.content so FI's granular + // text/messages privacy controls also hide tool arguments and declarations. + if (Array.isArray(tools)) { + if (tools.length > 32) attributes["vertexai.capture.truncated"] = true; + messages.unshift({ role: "system", content: tools.slice(0, 32).map(safeTool) }); } - - private async traceCountTokens( - original: Function, - instance: any, - args: any[] - ): Promise { - const request = args[0] || {}; - const modelName = instance.model || instance._model || "unknown"; - - const span = this.fiTracer.startSpan(`Vertex AI Count Tokens`, { - kind: SpanKind.CLIENT, - attributes: { - [SemanticConventions.FI_SPAN_KIND]: FISpanKind.LLM, - [SemanticConventions.LLM_PROVIDER]: "google", - [SemanticConventions.LLM_MODEL_NAME]: modelName, - "vertexai.operation": "count_tokens", - [SemanticConventions.INPUT_VALUE]: safeJsonStringify(request), - }, - }); - - const execContext = trace.setSpan(context.active(), span); - - try { - const result = await context.with(execContext, () => { - return original.apply(instance, args); - }); - - if (result?.totalTokens) { - span.setAttribute("vertexai.total_tokens", result.totalTokens); - } - - span.setAttributes({ - [SemanticConventions.OUTPUT_VALUE]: safeJsonStringify(result), - }); - - span.setStatus({ code: SpanStatusCode.OK }); - span.end(); - - return result; - } catch (error) { - span.recordException(error as Error); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: (error as Error).message, - }); - span.end(); - throw error; - } + if (messages.length > 32) { + const prefix = (instruction ? 1 : 0) + (Array.isArray(tools) ? 1 : 0); + messages.splice(prefix, messages.length - 32); + attributes["vertexai.capture.truncated"] = true; } - - private async traceEmbed( - original: Function, - instance: any, - args: any[] - ): Promise { - const request = args[0] || {}; - const modelName = instance.model || instance._model || "unknown"; - const texts = request.texts || request.instances || []; - - const span = this.fiTracer.startSpan(`Vertex AI Embed`, { - kind: SpanKind.CLIENT, - attributes: { - [SemanticConventions.FI_SPAN_KIND]: FISpanKind.EMBEDDING, - [SemanticConventions.LLM_PROVIDER]: "google", - [SemanticConventions.EMBEDDING_MODEL_NAME]: modelName, - [SemanticConventions.GEN_AI_OPERATION_NAME]: "embeddings", - [SemanticConventions.INPUT_VALUE]: safeJsonStringify(request), - "vertexai.input_count": Array.isArray(texts) ? texts.length : 1, - }, - }); - - const execContext = trace.setSpan(context.active(), span); - - try { - const result = await context.with(execContext, () => { - return original.apply(instance, args); - }); - - if (result?.embeddings) { - span.setAttribute("vertexai.embedding_count", result.embeddings.length); - if (result.embeddings[0]?.values) { - span.setAttribute("vertexai.embedding_dimensions", result.embeddings[0].values.length); - } - } - - span.setStatus({ code: SpanStatusCode.OK }); - span.end(); - - return result; - } catch (error) { - span.recordException(error as Error); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: (error as Error).message, - }); - span.end(); - throw error; + capture(attributes, S.LLM_INPUT_MESSAGES, messages, true); + if (!counting) { + const config = request?.generationConfig ?? instance.generationConfig; + for (const [name, key] of Object.entries({ temperature: "gen_ai.request.temperature", topP: "gen_ai.request.top_p", topK: "gen_ai.request.top_k", maxOutputTokens: "gen_ai.request.max_tokens" })) { + if (typeof config?.[name] === "number" && Number.isFinite(config[name])) attributes[key] = config[name]; } } + return attributes; +} - private getInputMessagesAttributes(request: any): Record { - const contents = request.contents || []; - if (!Array.isArray(contents) || contents.length === 0) { - return {}; - } - const messages = contents.map((content: any) => { - const msg: Record = { role: content.role || "user" }; - if (content.parts) { - const textParts = content.parts - .filter((p: any) => p.text) - .map((p: any) => p.text) - .join(""); - if (textParts) { - msg.content = textParts; - } - } - return msg; - }); - return { - [SemanticConventions.LLM_INPUT_MESSAGES]: safelyJSONStringify(messages) ?? "[]", - }; +/** Capture supported tool kinds, never retrieval resource names or unknown payloads. */ +function safeTool(tool: SDKObject): SDKObject { + if (!tool || typeof tool !== "object") return { type: "unknown" }; + if (Array.isArray(tool.functionDeclarations)) return { functionDeclarations: tool.functionDeclarations }; + if (tool.googleSearchRetrieval && typeof tool.googleSearchRetrieval === "object") { + const config = tool.googleSearchRetrieval; + return { googleSearchRetrieval: typeof config.disableAttribution === "boolean" + ? { disableAttribution: config.disableAttribution } : {} }; } + if (tool.retrieval && typeof tool.retrieval === "object") { + const config = tool.retrieval; + const type = config.vertexAiSearch ? "vertexAiSearch" : config.vertexRagStore ? "vertexRagStore" : "retrieval"; + return { retrieval: { type, ...(typeof config.disableAttribution === "boolean" + ? { disableAttribution: config.disableAttribution } : {}) } }; + } + return { type: "unknown" }; +} - private setResponseAttributes(span: Span, result: any): void { - if (!result) return; - - const response = result.response || result; - - if (response.candidates?.[0]?.content) { - const content = response.candidates[0].content; - const msg: Record = { role: content.role || "model" }; - - if (content.parts) { - const textParts = content.parts - .filter((p: any) => p.text) - .map((p: any) => p.text) - .join(""); - if (textParts) { - msg.content = textParts; - span.setAttribute(SemanticConventions.OUTPUT_VALUE, textParts); - } - } +function message(content: SDKObject, attributes: Attributes): SDKObject { + if (Array.isArray(content?.parts) && content.parts.length > 32) attributes["vertexai.capture.truncated"] = true; + const parts = Array.isArray(content?.parts) ? content.parts.slice(0, 32) : []; + return { + role: ["user", "model", "system", "function"].includes(content?.role) ? content.role : "user", + content: parts.map((part: SDKObject | string) => { + if (typeof part === "string") return { type: "text", text: part }; + if (typeof part?.text === "string") return { type: "text", text: part.text }; + if (part?.functionCall) return { type: "tool_call", name: part.functionCall.name, arguments: part.functionCall.args }; + if (part?.functionResponse) return { type: "tool_result", name: part.functionResponse.name, response: part.functionResponse.response }; + // Never copy inlineData.data, fileUri, arbitrary response fields or media. + return { type: "media" }; + }), + }; +} - span.setAttribute( - SemanticConventions.LLM_OUTPUT_MESSAGES, - safelyJSONStringify([msg]) ?? "[]" - ); +function outputAttributes(response: SDKObject, counting: boolean): Attributes { + const attributes: Attributes = {}; + if (counting) { + if (typeof response.totalTokens === "number") attributes["vertexai.total_tokens"] = response.totalTokens; + return attributes; + } + if (Array.isArray(response.candidates)) { + if (response.candidates.length > 32) attributes["vertexai.capture.truncated"] = true; + capture(attributes, S.LLM_OUTPUT_MESSAGES, response.candidates.slice(0, 32).map((candidate: SDKObject) => message(candidate.content, attributes))); + attributes[S.GEN_AI_RESPONSE_FINISH_REASONS] = response.candidates.slice(0, 32).map((candidate: SDKObject) => candidate.finishReason).filter((reason: unknown) => typeof reason === "string").map((reason: string) => reason.slice(0, 128)); + } + const usage = response.usageMetadata; + for (const [name, key] of Object.entries({ promptTokenCount: S.LLM_TOKEN_COUNT_PROMPT, candidatesTokenCount: S.LLM_TOKEN_COUNT_COMPLETION, totalTokenCount: S.LLM_TOKEN_COUNT_TOTAL, cachedContentTokenCount: S.LLM_TOKEN_COUNT_PROMPT_DETAILS_CACHE_READ, thoughtsTokenCount: S.LLM_TOKEN_COUNT_COMPLETION_DETAILS_REASONING })) { + if (typeof usage?.[name] === "number" && Number.isFinite(usage[name]) && usage[name] >= 0) attributes[key] = usage[name]; + } + return attributes; +} - // Add finish reasons - const finishReasons = response.candidates - ?.map((c: any) => c.finishReason) - .filter(Boolean); - if (finishReasons?.length) { - span.setAttribute( - SemanticConventions.GEN_AI_RESPONSE_FINISH_REASONS, - safelyJSONStringify(finishReasons) ?? "[]" - ); - } +/** Bound content before serialization; do not retain chunks or serialize whole SDK objects. */ +function capture(attributes: Attributes, key: string, messages: SDKObject[], newestFirst = false): void { + let budget = 24_000; + let truncated = false; + const bound = (item: any, depth = 0): any => { + if (budget <= 0 || depth > 10) { truncated = true; return "[truncated]"; } + budget -= 16; + if (typeof item === "string") { + const limit = Math.min(4096, Math.max(0, budget)); + truncated ||= item.length > limit; + const text = item.slice(0, limit); + budget -= text.length; + return text; } - - if (response.usageMetadata) { - const usage = response.usageMetadata; - if (usage.promptTokenCount) { - span.setAttribute(SemanticConventions.LLM_TOKEN_COUNT_PROMPT, usage.promptTokenCount); - } - if (usage.candidatesTokenCount) { - span.setAttribute(SemanticConventions.LLM_TOKEN_COUNT_COMPLETION, usage.candidatesTokenCount); - } - if (usage.totalTokenCount) { - span.setAttribute(SemanticConventions.LLM_TOKEN_COUNT_TOTAL, usage.totalTokenCount); + if (item === null || typeof item === "boolean" || typeof item === "number") return item; + if (Array.isArray(item)) { + truncated ||= item.length > 32; + const result = []; + for (const entry of item.slice(0, 32)) { + if (budget <= 0) { truncated = true; break; } + result.push(bound(entry, depth + 1)); } + return result; } - - span.setAttribute(SemanticConventions.RAW_OUTPUT, safeJsonStringify(response)); - } - - private async wrapStream(streamResult: any, span: Span): Promise { - const chunks: any[] = []; - let fullContent = ""; - - // Vertex AI streams return a response object with a stream property - const originalStream = streamResult.stream; - - if (!originalStream || typeof originalStream[Symbol.asyncIterator] !== "function") { - // If not a proper stream, just return as-is - span.setStatus({ code: SpanStatusCode.OK }); - span.end(); - return streamResult; + if (item && typeof item === "object") { + const result: Record = Object.create(null); + const keys = Object.keys(item); + truncated ||= keys.length > 32; + for (const name of keys.slice(0, 32)) { + if (budget <= 0) { truncated = true; break; } + const boundedName = name.slice(0, 128); + truncated ||= name.length > boundedName.length; + budget -= boundedName.length; + result[boundedName] = bound(item[name], depth + 1); + } + return result; } - - const wrappedStream = async function* () { - try { - for await (const chunk of originalStream) { - chunks.push(chunk); - if (chunk.candidates?.[0]?.content?.parts) { - const textParts = chunk.candidates[0].content.parts - .filter((p: any) => p.text) - .map((p: any) => p.text) - .join(""); - fullContent += textParts; - } - yield chunk; - } - - span.setAttributes({ - [SemanticConventions.OUTPUT_VALUE]: fullContent, - [SemanticConventions.LLM_OUTPUT_MESSAGES]: safelyJSONStringify([{ role: "model", content: fullContent }]) ?? "[]", - [SemanticConventions.RAW_OUTPUT]: safeJsonStringify(chunks), - }); - - // Get usage from the aggregated response if available - if (streamResult.response) { - const response = await streamResult.response; - if (response?.usageMetadata) { - const usage = response.usageMetadata; - if (usage.promptTokenCount) { - span.setAttribute(SemanticConventions.LLM_TOKEN_COUNT_PROMPT, usage.promptTokenCount); - } - if (usage.candidatesTokenCount) { - span.setAttribute(SemanticConventions.LLM_TOKEN_COUNT_COMPLETION, usage.candidatesTokenCount); - } - if (usage.totalTokenCount) { - span.setAttribute(SemanticConventions.LLM_TOKEN_COUNT_TOTAL, usage.totalTokenCount); - } - } - } - - span.setStatus({ code: SpanStatusCode.OK }); - } catch (error) { - span.recordException(error as Error); - span.setStatus({ - code: SpanStatusCode.ERROR, - message: (error as Error).message, - }); - throw error; - } finally { - span.end(); + return undefined; + }; + const captured: SDKObject[] = []; + // Spend the input budget on the current turn, then recent history. Restore + // chronological order afterwards; output candidates retain provider order. + for (const message of newestFirst ? [...messages].reverse() : messages) { + if (budget < 128) { truncated = true; break; } + budget -= 64; // Reserve the role/content envelope; never truncate it. + const content: SDKObject[] = []; + for (const part of message.content) { + if (budget < 64) { truncated = true; break; } + budget -= 64; // Reserve the part envelope, including its fixed type. + const capturedPart: SDKObject = {}; + // These keys come from message(), not arbitrary SDK objects. Only payloads + // use the recursive limiter, so exhaustion cannot turn messages/parts into strings. + for (const [name, value] of Object.entries(part)) { + capturedPart[name] = name === "type" ? value : bound(value); } - }; - - return { - ...streamResult, - stream: wrappedStream(), - }; + content.push(capturedPart); + } + captured.push({ role: message.role, content }); } + attributes[key] = JSON.stringify(newestFirst ? captured.reverse() : captured); + if (truncated) attributes["vertexai.capture.truncated"] = true; } -// Global patched flag -let _isFIPatched = false; +export function isPatched(): boolean { return patchedModules > 0; } -/** - * Check if the module has been patched. - */ -export function isPatched(): boolean { - return _isFIPatched; -} - -/** - * Reset the patched state (for testing only). - */ -export function _resetPatchedStateForTesting(): void { - _isFIPatched = false; -} - -// Helper functions - -function safeJsonStringify(obj: unknown): string { - try { - return JSON.stringify(obj); - } catch { - return String(obj); - } -} +/** @deprecated Test compatibility only. Disable the owning instance to restore patches. */ +export function _resetPatchedStateForTesting(): void { /* No global ownership reset. */ } diff --git a/typescript/packages/traceai_vertexai/test/compat.test.cjs b/typescript/packages/traceai_vertexai/test/compat.test.cjs new file mode 100644 index 000000000..2367464e9 --- /dev/null +++ b/typescript/packages/traceai_vertexai/test/compat.test.cjs @@ -0,0 +1,506 @@ +const { test, beforeEach, afterEach, after } = require('node:test'); +const assert = require('node:assert/strict'); +const { createRequire } = require('node:module'); +const path = require('node:path'); +const local = createRequire(process.env.VERTEX_INSTRUMENTATION ? path.resolve(process.env.VERTEX_INSTRUMENTATION, 'package.json') : path.resolve(__dirname, '../package.json')); +const consumer = process.env.VERTEX_CONSUMER ? createRequire(path.resolve(process.env.VERTEX_CONSUMER, 'package.json')) : local; +const { VertexAIInstrumentation } = local('@traceai/fi-instrumentation-vertexai'); +const { context, trace } = local('@opentelemetry/api'); +const core = createRequire(local.resolve('@traceai/fi-core')); +const { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } = core('@opentelemetry/sdk-trace-base'); +const { AsyncLocalStorageContextManager } = core('@opentelemetry/context-async-hooks'); +const { suppressTracing } = core('@opentelemetry/core'); +const { SemanticConventions: S } = local('@traceai/fi-semantic-conventions'); +const sdk = consumer('@google-cloud/vertexai'); +context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable()); +const exporter = new InMemorySpanExporter(); +const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); +let instrumentation, requests, reply, failure, streamBody; +const originalFetch = global.fetch; +const response = (text = 'hello') => ({ candidates: [{ index: 0, content: { role: 'model', parts: [{ text }] }, finishReason: 'STOP' }], usageMetadata: { promptTokenCount: 3, candidatesTokenCount: 0, totalTokenCount: 3 } }); +function model(preview = false, config = {}) { + const vertex = new sdk.VertexAI({ project: 'synthetic-project', location: 'us-central1', googleAuthOptions: { authClient: { getAccessToken: async () => ({ token: 'synthetic-only' }) } } }); + return (preview ? vertex.preview : vertex).getGenerativeModel({ model: 'gemini-test', generationConfig: { temperature: 0.2 }, ...config }); +} +function instrument(config = {}) { + instrumentation = new VertexAIInstrumentation(config); + instrumentation.setTracerProvider(provider); + instrumentation.manuallyInstrument(sdk); + return instrumentation; +} +const settle = () => new Promise(resolve => setImmediate(resolve)); +beforeEach(() => { + exporter.reset(); requests = []; reply = response(); failure = false; streamBody = undefined; + global.fetch = async (url, options) => { + requests.push({ url, body: JSON.parse(options.body) }); + if (failure) return new Response(JSON.stringify({ error: { message: 'secret-provider-error' } }), { status: 400 }); + if (String(url).includes('countTokens')) return new Response(JSON.stringify({ totalTokens: 0 })); + if (String(url).includes('streamGenerateContent')) { + return new Response(streamBody ?? `data: ${JSON.stringify(reply)}\n\n`, { headers: { 'content-type': 'text/event-stream' } }); + } + return new Response(JSON.stringify(reply)); + }; +}); +afterEach(() => { instrumentation?.disable(); instrumentation = undefined; global.fetch = originalFetch; }); +after(async () => { await provider.shutdown(); context.disable(); }); + +for (const preview of [false, true]) { + test(`${preview ? 'preview' : 'stable'} generation emits exactly one parented span and preserves zero usage`, async () => { + instrument(); + const parent = provider.getTracer('test').startSpan('parent'); + const result = await context.with(trace.setSpan(context.active(), parent), () => model(preview).generateContent('question')); + assert.equal(result.response.candidates[0].content.parts[0].text, 'hello'); + const spans = exporter.getFinishedSpans(); + assert.equal(spans.length, 1); + assert.equal(spans[0].parentSpanContext.spanId, parent.spanContext().spanId); + assert.equal(spans[0].attributes[S.LLM_TOKEN_COUNT_COMPLETION], 0); + assert.equal(spans[0].attributes[S.LLM_MODEL_NAME], 'gemini-test'); + assert.match(spans[0].attributes[S.LLM_INPUT_MESSAGES], /question/); + parent.end(); + }); + test(`${preview ? 'preview' : 'stable'} chat uses actual resource model and captures multi-turn history`, async () => { + instrument(); + const chat = model(preview).startChat(); + await chat.sendMessage('first question'); + await chat.sendMessage('second question'); + const spans = exporter.getFinishedSpans(); + assert.equal(spans.length, 2); + assert.equal(spans[1].attributes[S.LLM_MODEL_NAME], 'gemini-test'); + assert.match(spans[1].attributes[S.LLM_INPUT_MESSAGES], /first question/); + assert.match(spans[1].attributes[S.LLM_INPUT_MESSAGES], /second question/); + assert.equal((await chat.getHistory()).length, 4); + assert.equal(requests.length, 2); + }); + test(`${preview ? 'preview' : 'stable'} aggregate-only stream ends once without consuming caller iterator`, async () => { + instrument(); + const result = await model(preview).generateContentStream('question'); + await result.response; + await settle(); + assert.equal(exporter.getFinishedSpans().length, 1); + assert.match(exporter.getFinishedSpans()[0].attributes[S.LLM_OUTPUT_MESSAGES], /hello/); + const chunks = []; + for await (const chunk of result.stream) chunks.push(chunk); + assert.equal(chunks.length, 1); + assert.equal(exporter.getFinishedSpans().length, 1); + }); +} + +for (const preview of [false, true]) { + for (const streaming of [false, true]) { + for (const [label, usage] of [ + ['positive', { cachedContentTokenCount: 80, thoughtsTokenCount: 5 }], + ['zero', { cachedContentTokenCount: 0, thoughtsTokenCount: 0 }], + ['absent', {}], + ]) { + test(`${preview ? 'preview' : 'stable'} ${streaming ? 'streaming' : 'unary'} preserves ${label} cache and reasoning usage under FI semantic keys`, async () => { + instrument(); + // Provider JSON can include thoughtsTokenCount even when the SDK types omit it. + Object.assign(reply.usageMetadata, usage); + const m = model(preview); + const result = await (streaming ? m.generateContentStream('question') : m.generateContent('question')); + const aggregate = await result.response; + await settle(); + assert.deepEqual(aggregate.usageMetadata, reply.usageMetadata); + const spans = exporter.getFinishedSpans(); + assert.equal(spans.length, 1); + const attributes = spans[0].attributes; + assert.deepEqual([ + attributes[S.LLM_TOKEN_COUNT_PROMPT_DETAILS_CACHE_READ], + attributes[S.LLM_TOKEN_COUNT_COMPLETION_DETAILS_REASONING], + ], [usage.cachedContentTokenCount, usage.thoughtsTokenCount]); + assert.equal(attributes[S.LLM_TOKEN_COUNT_PROMPT], 3); + assert.equal(attributes[S.LLM_TOKEN_COUNT_COMPLETION], 0); + assert.equal(attributes[S.LLM_TOKEN_COUNT_TOTAL], 3); + }); + } + } +} + +for (const preview of [false, true]) { + test(`${preview ? 'preview' : 'stable'} records grounding and retrieval tool types without datastore IDs`, async () => { + instrument(); + const tools = [ + { googleSearchRetrieval: { disableAttribution: true } }, + { retrieval: { vertexAiSearch: { datastore: 'projects/PRIVATE_DATASTORE' }, disableAttribution: true } }, + { functionDeclarations: [{ name: 'lookup', description: 'tool description' }] }, + ]; + await model(preview, { tools }).generateContent('question'); + assert.deepEqual(requests[0].body.tools, tools); + const captured = JSON.parse(exporter.getFinishedSpans()[0].attributes[S.LLM_INPUT_MESSAGES]); + assert.deepEqual(captured[0].content, [ + { googleSearchRetrieval: { disableAttribution: true } }, + { retrieval: { type: 'vertexAiSearch', disableAttribution: true } }, + { functionDeclarations: tools[2].functionDeclarations }, + ]); + assert.doesNotMatch(JSON.stringify(exporter.getFinishedSpans()[0].attributes), /PRIVATE_DATASTORE/); + }); + test(`${preview ? 'preview' : 'stable'} hides grounding tool details with input privacy controls`, async () => { + instrument({ traceConfig: { hideInputText: true } }); + await model(preview, { tools: [{ googleSearchRetrieval: { disableAttribution: true } }] }).generateContent('question'); + assert.doesNotMatch(JSON.stringify(exporter.getFinishedSpans()[0].attributes), /googleSearchRetrieval|disableAttribution/); + }); +} + +test('constructor privacy config applies without a second enable call', async () => { + instrument({ traceConfig: { hideInputs: true, hideOutputs: true } }); + await model().generateContent('private-question'); + const spans = exporter.getFinishedSpans(); + assert.equal(spans.length, 1); + assert.doesNotMatch(JSON.stringify(spans.map(({ attributes, status, events }) => ({ attributes, status, events }))), /private-question|hello/); +}); + +test('manual instrumentation disables and re-enables without duplicate spans', async () => { + instrument(); + const m = model(); + await m.generateContent('one'); + instrumentation.disable(); + await m.generateContent('two'); + assert.equal(exporter.getFinishedSpans().length, 1); + instrumentation.enable(); + instrumentation.manuallyInstrument(sdk); + await m.generateContent('three'); + assert.equal(exporter.getFinishedSpans().length, 2); +}); + +test('suppressed calls are not traced', async () => { + instrument(); + await context.with(suppressTracing(context.active()), () => model().generateContent('question')); + assert.equal(exporter.getFinishedSpans().length, 0); +}); + +test('all candidates and tool calls are captured; provider error messages stay out of spans', async () => { + instrument(); + reply.candidates.push({ index: 1, content: { role: 'model', parts: [{ functionCall: { name: 'weather', args: { city: 'Paris' } } }] }, finishReason: 'STOP' }); + await model().generateContent('question'); + assert.match(exporter.getFinishedSpans()[0].attributes[S.LLM_OUTPUT_MESSAGES], /weather/); + failure = true; + await assert.rejects(model().generateContent('question')); + assert.equal(exporter.getFinishedSpans()[1].status.code, 2); + assert.doesNotMatch(JSON.stringify(exporter.getFinishedSpans().map(({ attributes, status, events }) => ({ attributes, status, events }))), /secret-provider-error/); +}); + +test('raw binary media and URIs are never exported', async () => { + instrument(); + await model().generateContent({ contents: [{ role: 'user', parts: [{ text: 'question' }, { inlineData: { mimeType: 'image/png', data: 'BINARY_SENTINEL' } }, { fileData: { mimeType: 'image/png', fileUri: 'gs://URI_SENTINEL' } }] }] }); + assert.equal(requests[0].body.contents[0].parts[1].inlineData.data, 'BINARY_SENTINEL'); + assert.doesNotMatch(JSON.stringify(exporter.getFinishedSpans().map(({ attributes, status, events }) => ({ attributes, status, events }))), /BINARY_SENTINEL|URI_SENTINEL/); +}); + +test('zero countTokens result is preserved', async () => { + instrument(); + assert.equal((await model().countTokens({ contents: [{ role: 'user', parts: [{ text: 'question' }] }] })).totalTokens, 0); + assert.equal(exporter.getFinishedSpans()[0].attributes['vertexai.total_tokens'], 0); +}); + +for (const preview of [false, true]) { + test(`${preview ? 'preview' : 'stable'} countTokens captures the submitted request, not generation defaults`, async () => { + instrument(); + const configured = model(preview, { + systemInstruction: 'MODEL_INSTRUCTION_NOT_COUNTED', + tools: [{ functionDeclarations: [{ name: 'MODEL_TOOL_NOT_COUNTED' }] }], + generationConfig: { temperature: 0.7, maxOutputTokens: 123 }, + }); + const request = { contents: [{ role: 'user', parts: [{ text: 'COUNTED_REQUEST' }] }] }; + assert.equal((await configured.countTokens(request)).totalTokens, 0); + assert.deepEqual(requests[0].body, request); + const spans = exporter.getFinishedSpans(); + assert.equal(spans.length, 1); + const attrs = spans[0].attributes; + assert.equal(attrs[S.GEN_AI_OPERATION_NAME], 'count_tokens'); + assert.deepEqual(JSON.parse(attrs[S.LLM_INPUT_MESSAGES]), [{ role: 'user', content: [{ type: 'text', text: 'COUNTED_REQUEST' }] }]); + assert.equal(attrs['gen_ai.request.temperature'], undefined); + assert.equal(attrs['gen_ai.request.max_tokens'], undefined); + assert.equal(attrs['vertexai.total_tokens'], 0); + }); + + test(`${preview ? 'preview' : 'stable'} countTokens retains explicitly submitted instructions and tools`, async () => { + instrument(); + const configured = model(preview, { + systemInstruction: 'MODEL_INSTRUCTION_NOT_COUNTED', + tools: [{ functionDeclarations: [{ name: 'MODEL_TOOL_NOT_COUNTED' }] }], + generationConfig: { temperature: 0.7 }, + }); + const request = { + contents: [{ role: 'user', parts: [{ text: 'COUNTED_REQUEST' }] }], + systemInstruction: { parts: [{ text: 'EXPLICIT_INSTRUCTION' }] }, + tools: [{ functionDeclarations: [{ name: 'EXPLICIT_TOOL' }] }], + }; + assert.equal((await configured.countTokens(request)).totalTokens, 0); + assert.deepEqual(requests[0].body, request); + const attrs = exporter.getFinishedSpans()[0].attributes; + assert.match(attrs[S.LLM_INPUT_MESSAGES], /EXPLICIT_INSTRUCTION|EXPLICIT_TOOL/); + assert.doesNotMatch(attrs[S.LLM_INPUT_MESSAGES], /MODEL_INSTRUCTION_NOT_COUNTED|MODEL_TOOL_NOT_COUNTED/); + assert.equal(attrs['gen_ai.request.temperature'], undefined); + }); +} + +for (const flag of ['hideInputs', 'hideInputMessages', 'hideInputText', 'hideOutputs', 'hideOutputMessages', 'hideOutputText']) { + test(`${flag} hides content including tool arguments without hiding model/usage`, async () => { + instrument({ traceConfig: { [flag]: true } }); + reply = response('OUTPUT_SENTINEL'); + reply.candidates[0].content.parts.push({ functionCall: { name: 'outputTool', args: { secret: 'OUTPUT_SENTINEL' } } }); + await model().generateContent({ contents: [{ role: 'user', parts: [{ text: 'INPUT_SENTINEL' }, { functionResponse: { name: 'inputTool', response: { secret: 'INPUT_SENTINEL' } } }] }], tools: [{ functionDeclarations: [{ name: 'inputTool', description: 'INPUT_SENTINEL' }] }] }); + const span = exporter.getFinishedSpans()[0]; + assert.equal(span.attributes[S.LLM_MODEL_NAME], 'gemini-test'); + assert.equal(span.attributes[S.LLM_TOKEN_COUNT_PROMPT], 3); + assert.doesNotMatch(JSON.stringify(span.attributes), flag.startsWith('hideInput') ? /INPUT_SENTINEL/ : /OUTPUT_SENTINEL/); + }); +} + +test('environment privacy controls apply to automatically initialized FITracer', async () => { + process.env.FI_HIDE_INPUTS = 'true'; process.env.FI_HIDE_OUTPUTS = 'true'; + try { + instrument(); + await model().generateContent('INPUT_SENTINEL'); + const span = exporter.getFinishedSpans()[0]; + assert.equal(span.attributes[S.LLM_MODEL_NAME], 'gemini-test'); + assert.doesNotMatch(JSON.stringify(span.attributes), /INPUT_SENTINEL|hello/); + } finally { delete process.env.FI_HIDE_INPUTS; delete process.env.FI_HIDE_OUTPUTS; } +}); + +test('stream aggregate preserves repeated parts and final usage without double counting', async () => { + instrument(); + streamBody = [response('same'), response('same')].map(chunk => `data: ${JSON.stringify(chunk)}\n\n`).join(''); + const result = await model().generateContentStream('question'); + const chunks = []; + for await (const chunk of result.stream) chunks.push(chunk); + const aggregate = await result.response; + await settle(); + assert.equal(chunks.length, 2); + assert.equal(exporter.getFinishedSpans().length, 1); + const span = exporter.getFinishedSpans()[0]; + assert.equal(JSON.parse(span.attributes[S.LLM_OUTPUT_MESSAGES])[0].content.map(part => part.text).join(''), 'samesame'); + assert.equal(span.attributes[S.LLM_TOKEN_COUNT_PROMPT], aggregate.usageMetadata.promptTokenCount); +}); + +test('early iterator exit retains SDK-owned aggregate and chat history', async () => { + instrument(); + const chat = model().startChat(); + const result = await chat.sendMessageStream('question'); + for await (const chunk of result.stream) { assert.ok(chunk.candidates); break; } + await result.response; + await settle(); + assert.equal(exporter.getFinishedSpans().length, 1); + assert.equal((await chat.getHistory()).length, 2); + assert.equal(requests.length, 1); +}); + +test('malformed SSE keeps aggregate rejection and emits one error span', async () => { + instrument(); + streamBody = 'data: not-json\n\n'; + const result = await model().generateContentStream('question'); + await assert.rejects(result.response); + await assert.rejects(async () => { for await (const chunk of result.stream) void chunk; }); + await settle(); + assert.equal(exporter.getFinishedSpans().length, 1); + assert.equal(exporter.getFinishedSpans()[0].status.code, 2); +}); + +test('empty SSE retains the SDK aggregate rejection and completes exactly once', async () => { + instrument(); streamBody = ''; + const result = await model().generateContentStream('question'); + await assert.rejects(result.response, /final response in stream chunk is undefined/); + await settle(); + assert.equal(exporter.getFinishedSpans().length, 1); + assert.equal(exporter.getFinishedSpans()[0].status.code, 2); +}); + +test('an active stream finishes after disabling while new calls remain untraced', async () => { + instrument(); + let release; + streamBody = new ReadableStream({ start(controller) { release = () => { controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(reply)}\n\n`)); controller.close(); }; } }); + const result = await model().generateContentStream('question'); + instrumentation.disable(); release(); + await result.response; await settle(); + await model().generateContent('untraced'); + assert.equal(exporter.getFinishedSpans().length, 1); +}); + +test('changing the provider updates FITracer without re-enabling', async () => { + instrument(); + const secondExporter = new InMemorySpanExporter(); + const secondProvider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(secondExporter)] }); + try { + await model().generateContent('one'); + instrumentation.setTracerProvider(secondProvider); + await model().generateContent('two'); + assert.equal(exporter.getFinishedSpans().length, 1); + assert.equal(secondExporter.getFinishedSpans().length, 1); + } finally { await secondProvider.shutdown(); } +}); + +test('function-result chat messages retain the SDK role', async () => { + instrument(); + const chat = model().startChat(); + await chat.sendMessage([{ functionResponse: { name: 'weather', response: { city: 'Paris' } } }]); + const messages = JSON.parse(exporter.getFinishedSpans()[0].attributes[S.LLM_INPUT_MESSAGES]); + assert.equal(messages.at(-1).role, requests[0].body.contents.at(-1).role); +}); + +test('bounded history keeps the latest request when tools add a prefix message', async () => { + instrument(); + const contents = Array.from({ length: 39 }, (_, i) => ({ role: i % 2 ? 'model' : 'user', parts: [{ text: 'history' }] })); + contents.push({ role: 'user', parts: [{ text: 'LATEST_REQUEST' }] }); + await model().generateContent({ contents, tools: [{ functionDeclarations: [{ name: 'weather' }] }] }); + assert.match(exporter.getFinishedSpans()[0].attributes[S.LLM_INPUT_MESSAGES], /LATEST_REQUEST/); +}); + +test('list truncation is marked independently from the text budget', async () => { + instrument(); + reply.candidates[0].content.parts = Array.from({ length: 40 }, () => ({ text: 'x' })); + await model().generateContent('question'); + assert.equal(exporter.getFinishedSpans()[0].attributes['vertexai.capture.truncated'], true); +}); + +test('bounded capture marks truncation while leaving the SDK response intact', async () => { + instrument(); reply = response('x'.repeat(200000)); + const result = await model().generateContent('q'.repeat(200000)); + assert.equal(result.response.candidates[0].content.parts[0].text.length, 200000); + const span = exporter.getFinishedSpans()[0]; + assert.equal(span.attributes['vertexai.capture.truncated'], true); + assert.ok(span.attributes[S.LLM_OUTPUT_MESSAGES].length < 10000); + assert.ok(span.attributes[S.LLM_INPUT_MESSAGES].length < 10000); +}); + +for (const preview of [false, true]) { + for (const method of ['sendMessage', 'sendMessageStream']) { + test(`${preview ? 'preview' : 'stable'} ${method} captures strings in request arrays without mutating them`, async () => { + instrument(); + const request = ['ARRAY_STRING_SENTINEL', { text: 'part-object' }]; + const result = await model(preview).startChat()[method](request); + await result.response; await settle(); + assert.deepEqual(request, ['ARRAY_STRING_SENTINEL', { text: 'part-object' }]); + assert.equal(requests[0].body.contents.at(-1).parts[0].text, request[0]); + const messages = JSON.parse(exporter.getFinishedSpans()[0].attributes[S.LLM_INPUT_MESSAGES]); + assert.deepEqual(messages.at(-1).content, request.map(part => ({ type: 'text', text: typeof part === 'string' ? part : part.text }))); + }); + + test(`${preview ? 'preview' : 'stable'} ${method} budgets the current turn before recent history`, async () => { + instrument(); + const history = Array.from({ length: 10 }, (_, i) => ({ role: i % 2 ? 'model' : 'user', parts: [{ text: `history-${i} ${'h'.repeat(4000)}` }] })); + const result = await model(preview).startChat({ history })[method]('CURRENT_QUESTION_SENTINEL'); + await result.response; await settle(); + assert.equal(requests[0].body.contents.at(-1).parts[0].text, 'CURRENT_QUESTION_SENTINEL'); + const span = exporter.getFinishedSpans()[0]; + const messages = JSON.parse(span.attributes[S.LLM_INPUT_MESSAGES]); + assert.deepEqual(messages.at(-1), { role: 'user', content: [{ type: 'text', text: 'CURRENT_QUESTION_SENTINEL' }] }); + assert.match(JSON.stringify(messages.at(-2)), /history-9/); + assert.equal(span.attributes['vertexai.capture.truncated'], true); + assert.ok(span.attributes[S.LLM_INPUT_MESSAGES].length < 26000); + assert.ok(messages.every(message => typeof message.role === 'string' && Array.isArray(message.content))); + const indices = messages.slice(0, -1).map(message => Number(message.content[0].text.match(/history-(\d+)/)[1])); + assert.deepEqual(indices, [...indices].sort((a, b) => a - b)); + }); + } + + for (const method of ['generateContent', 'generateContentStream']) { + test(`${preview ? 'preview' : 'stable'} ${method} carries scoped FI context and OTel parent`, async () => { + instrument(); + const fi = local('@traceai/fi-core'); + const parent = provider.getTracer('test').startSpan('parent'); + const ctx = fi.setUser(fi.setSession(trace.setSpan(context.active(), parent), { sessionId: 'synthetic-session' }), { userId: 'synthetic-user' }); + try { + const result = await context.with(ctx, () => model(preview)[method]('question')); + await result.response; await settle(); + const span = exporter.getFinishedSpans()[0]; + assert.equal(span.attributes['session.id'], 'synthetic-session'); + assert.equal(span.attributes['user.id'], 'synthetic-user'); + assert.equal(span.parentSpanContext.spanId, parent.spanContext().spanId); + await model(preview).generateContent('unscoped'); + assert.equal(exporter.getFinishedSpans()[1].attributes['session.id'], undefined); + assert.equal(exporter.getFinishedSpans()[1].attributes['user.id'], undefined); + } finally { parent.end(); } + }); + } +} + +for (const method of ['generateContent', 'generateContentStream']) { + test(`${method} preserves output message and part structure when the budget runs out`, async () => { + instrument(); + reply.candidates = Array.from({ length: 8 }, (_, index) => ({ index, content: { role: 'model', parts: [{ text: 'r'.repeat(4000) }] }, finishReason: 'STOP' })); + const result = await model()[method]('question'); + const aggregate = await result.response; await settle(); + assert.equal(aggregate.candidates.length, 8); + assert.equal(aggregate.candidates[7].content.parts[0].text.length, 4000); + const span = exporter.getFinishedSpans()[0]; + const messages = JSON.parse(span.attributes[S.LLM_OUTPUT_MESSAGES]); + assert.ok(messages.length > 0); + for (const message of messages) { + assert.equal(message.role, 'model'); + assert.ok(Array.isArray(message.content)); + for (const part of message.content) { + assert.equal(part.type, 'text'); + assert.equal(typeof part.text, 'string'); + } + } + assert.equal(span.attributes['vertexai.capture.truncated'], true); + assert.ok(span.attributes[S.LLM_OUTPUT_MESSAGES].length < 26000); + }); +} + +test('large instructions and tool declarations cannot spend the current request budget', async () => { + instrument(); + await model().generateContent({ + systemInstruction: { parts: Array.from({ length: 10 }, () => ({ text: 's'.repeat(4000) })) }, + tools: [{ functionDeclarations: Array.from({ length: 10 }, (_, i) => ({ name: `tool_${i}`, description: 'd'.repeat(4000) })) }], + contents: [{ role: 'user', parts: [{ text: 'CURRENT_QUESTION_SENTINEL' }] }], + }); + const span = exporter.getFinishedSpans()[0]; + const messages = JSON.parse(span.attributes[S.LLM_INPUT_MESSAGES]); + assert.deepEqual(messages.at(-1), { role: 'user', content: [{ type: 'text', text: 'CURRENT_QUESTION_SENTINEL' }] }); + assert.ok(messages.every(message => typeof message.role === 'string' && Array.isArray(message.content))); + assert.equal(span.attributes['vertexai.capture.truncated'], true); +}); + +test('budget exhaustion within a message preserves part envelopes and does not modify responses', async () => { + instrument(); + reply.candidates[0].content.parts = Array.from({ length: 20 }, (_, i) => i % 2 + ? { functionCall: { name: `tool_${i}`, args: { payload: 'a'.repeat(4000) } } } + : { text: 't'.repeat(4000) }); + const result = await model().generateContent('question'); + const span = exporter.getFinishedSpans()[0]; + const messages = JSON.parse(span.attributes[S.LLM_OUTPUT_MESSAGES]); + assert.equal(messages.length, 1); + assert.equal(messages[0].role, 'model'); + for (const part of messages[0].content) { + assert.ok(part.type === 'text' || part.type === 'tool_call'); + if (part.type === 'text') assert.equal(typeof part.text, 'string'); + else { assert.equal(typeof part.name, 'string'); assert.ok(Object.hasOwn(part, 'arguments')); } + } + assert.equal(result.response.candidates[0].content.parts.length, 20); + assert.equal(span.attributes['vertexai.capture.truncated'], true); + assert.ok(span.attributes[S.LLM_OUTPUT_MESSAGES].length < 26000); +}); + +test('FI context attributes are privacy-masked and cannot override observed model/provider', async () => { + instrument({ traceConfig: { hideInputs: true } }); + const fi = local('@traceai/fi-core'); + const ctx = fi.setAttributes(fi.setSession(context.active(), { sessionId: 'synthetic-session' }), { + [S.INPUT_VALUE]: 'CONTEXT_SECRET', [S.LLM_PROVIDER]: 'incorrect', [S.LLM_MODEL_NAME]: 'incorrect', + }); + await context.with(ctx, () => model().generateContent('question')); + const span = exporter.getFinishedSpans()[0]; + assert.equal(span.attributes['session.id'], 'synthetic-session'); + assert.equal(span.attributes[S.LLM_PROVIDER], 'google'); + assert.equal(span.attributes[S.LLM_MODEL_NAME], 'gemini-test'); + assert.doesNotMatch(JSON.stringify(span.attributes), /CONTEXT_SECRET|question/); +}); + +test('concurrent streaming calls keep their own FI session and user context', async () => { + instrument(); + const fi = local('@traceai/fi-core'); + await Promise.all(['first', 'second'].map(async id => { + const ctx = fi.setUser(fi.setSession(context.active(), { sessionId: `session-${id}` }), { userId: `user-${id}` }); + const result = await context.with(ctx, () => model().generateContentStream(id)); + await result.response; + })); + await settle(); + const spans = exporter.getFinishedSpans(); + assert.equal(spans.length, 2); + for (const id of ['first', 'second']) { + const span = spans.find(span => span.attributes['session.id'] === `session-${id}`); + assert.ok(span); + assert.equal(span.attributes['user.id'], `user-${id}`); + assert.equal(JSON.parse(span.attributes[S.LLM_INPUT_MESSAGES])[0].content[0].text, id); + } +}); diff --git a/typescript/packages/traceai_vertexai/test/export.test.cjs b/typescript/packages/traceai_vertexai/test/export.test.cjs new file mode 100644 index 000000000..e0e08e291 --- /dev/null +++ b/typescript/packages/traceai_vertexai/test/export.test.cjs @@ -0,0 +1,71 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const http = require('node:http'); +const { once } = require('node:events'); +const { createRequire } = require('node:module'); +const path = require('node:path'); +const local = createRequire(process.env.VERTEX_INSTRUMENTATION ? path.resolve(process.env.VERTEX_INSTRUMENTATION, 'package.json') : path.resolve(__dirname, '../package.json')); +const consumer = process.env.VERTEX_CONSUMER ? createRequire(path.resolve(process.env.VERTEX_CONSUMER, 'package.json')) : local; + +test('automatic hook exports actual OTLP JSON with FI auth and one span per call', async () => { + const requests = []; + const server = http.createServer((req, res) => { + let body = ''; + req.on('data', chunk => { body += chunk; }); + req.on('end', () => { requests.push({ url: req.url, headers: req.headers, body: JSON.parse(body) }); res.setHeader('content-type', 'application/json'); res.end('{}'); }); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const originalFetch = global.fetch; + let instrumentation, provider; + const { context } = local('@opentelemetry/api'); + try { + const { register, ProjectType } = local('@traceai/fi-core'); + const core = createRequire(local.resolve('@traceai/fi-core')); + const { AsyncLocalStorageContextManager } = core('@opentelemetry/context-async-hooks'); + context.setGlobalContextManager(new AsyncLocalStorageContextManager().enable()); + provider = register({ projectName: 'vertexai-compat', projectType: ProjectType.OBSERVE, setGlobalTracerProvider: true, batch: false, + endpoint: `http://127.0.0.1:${server.address().port}`, headers: { 'x-api-key': 'synthetic-fi-key', 'x-secret-key': 'synthetic-fi-secret' } }); + const { VertexAIInstrumentation, isPatched } = local('@traceai/fi-instrumentation-vertexai'); + instrumentation = new VertexAIInstrumentation(); + instrumentation.setTracerProvider(provider); + const sdk = consumer('@google-cloud/vertexai'); + assert.equal(isPatched(), true, 'CommonJS require must activate the automatic hook'); + // An ESM-style namespace alias must not duplicate automatic CJS hooks. + instrumentation.manuallyInstrument(Object.freeze({ ...sdk })); + let calls = 0; + global.fetch = async (url, options) => { + calls++; + assert.match(String(url), /aiplatform.googleapis.com/); + assert.doesNotMatch(JSON.stringify(options.headers), /synthetic-fi-key|synthetic-fi-secret/); + return new Response(JSON.stringify({ candidates: [{ content: { role: 'model', parts: [{ text: 'four' }] }, finishReason: 'STOP' }], usageMetadata: { promptTokenCount: 4, candidatesTokenCount: 1, totalTokenCount: 5 } })); + }; + const vertex = new sdk.VertexAI({ project: 'synthetic-project', location: 'us-central1', googleAuthOptions: { authClient: { getAccessToken: async () => ({ token: 'synthetic-google-token' }) } } }); + await vertex.getGenerativeModel({ model: 'gemini-test' }).generateContent('What is two plus two?'); + await provider.forceFlush(); + assert.equal(calls, 1); + assert.equal(requests.length, 1); + assert.equal(requests[0].url, '/tracer/v1/traces'); + assert.equal(requests[0].headers['x-api-key'], 'synthetic-fi-key'); + assert.equal(requests[0].headers['x-secret-key'], 'synthetic-fi-secret'); + const spans = requests.flatMap(request => request.body.resourceSpans).flatMap(resource => resource.scopeSpans).flatMap(scope => scope.spans); + assert.equal(spans.length, 1); + const attrs = Object.fromEntries(spans[0].attributes.map(({ key, value }) => [key, value.stringValue ?? Number(value.intValue)])); + assert.equal(attrs['gen_ai.request.model'], 'gemini-test'); + assert.equal(attrs['gen_ai.usage.input_tokens'], 4); + assert.match(attrs['gen_ai.input.messages'], /two plus two/); + assert.match(attrs['gen_ai.output.messages'], /four/); + assert.doesNotMatch(JSON.stringify(requests[0].body), /synthetic-google-token|synthetic-fi-key|synthetic-fi-secret/); + instrumentation.disable(); + await vertex.getGenerativeModel({ model: 'gemini-test' }).generateContent('untraced'); + await provider.forceFlush(); + assert.equal(requests.length, 1); + } finally { + instrumentation?.disable(); + await provider?.shutdown(); + context.disable(); + global.fetch = originalFetch; + server.close(); + await once(server, 'close'); + } +}); diff --git a/typescript/packages/traceai_vertexai/test/network.sb b/typescript/packages/traceai_vertexai/test/network.sb new file mode 100644 index 000000000..e0a4a7d24 --- /dev/null +++ b/typescript/packages/traceai_vertexai/test/network.sb @@ -0,0 +1,5 @@ +(version 1) +(allow default) +(deny network-outbound) +(allow network-outbound (remote ip "localhost:*")) +(allow network-outbound (remote unix-socket)) diff --git a/typescript/packages/traceai_vertexai/test/readme.test.cjs b/typescript/packages/traceai_vertexai/test/readme.test.cjs new file mode 100644 index 000000000..7e3261bca --- /dev/null +++ b/typescript/packages/traceai_vertexai/test/readme.test.cjs @@ -0,0 +1,56 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { createRequire } = require('node:module'); +const { readFileSync } = require('node:fs'); +const vm = require('node:vm'); +const http = require('node:http'); +const { once } = require('node:events'); +const path = require('node:path'); +const root = process.env.VERTEX_INSTRUMENTATION || path.resolve(__dirname, '..'); +const local = createRequire(path.join(root, 'package.json')); +const consumer = process.env.VERTEX_CONSUMER ? createRequire(path.resolve(process.env.VERTEX_CONSUMER, 'package.json')) : local; + +test('the exact README application runs with synthetic authentication and loopback OTLP', async () => { + const code = readFileSync(path.join(root, 'README.md'), 'utf8').match(/```javascript\n([\s\S]+?)```/)[1]; + const sdk = consumer('@google-cloud/vertexai'); + const core = local('@traceai/fi-core'); + const coreRequire = createRequire(local.resolve('@traceai/fi-core')); + const finished = [], errors = [], printed = []; + const server = http.createServer((req, res) => { + let body = ''; + req.on('data', chunk => { body += chunk; }); + req.on('end', () => { + finished.push(...JSON.parse(body).resourceSpans.flatMap(resource => resource.scopeSpans).flatMap(scope => scope.spans)); + res.setHeader('content-type', 'application/json'); res.end('{}'); + }); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const originalFetch = global.fetch; + let provider; + try { + global.fetch = async () => new Response(JSON.stringify({ candidates: [{ content: { role: 'model', parts: [{ text: 'Light scatters in the atmosphere.' }] } }] })); + const fakeRequire = name => { + if (name === '@google-cloud/vertexai') return { ...sdk, VertexAI: class extends sdk.VertexAI { + constructor(options) { super({ ...options, googleAuthOptions: { authClient: { getAccessToken: async () => ({ token: 'synthetic-only' }) } } }); } + } }; + if (name === '@traceai/fi-core') return { ...core, register(options) { + provider = core.register({ ...options, endpoint: `http://127.0.0.1:${server.address().port}`, setGlobalTracerProvider: false, headers: { 'x-api-key': 'synthetic', 'x-secret-key': 'synthetic' } }); + return provider; + } }; + if (name === '@opentelemetry/context-async-hooks') return coreRequire(name); + return local(name); + }; + await vm.runInNewContext(code, { require: fakeRequire, process: { env: { GOOGLE_CLOUD_PROJECT: 'synthetic-project' } }, console: { log: value => printed.push(value), error: error => errors.push(error) } }); + assert.deepEqual(errors, []); + assert.equal(printed[0][0].text, 'Light scatters in the atmosphere.'); + assert.equal(finished.length, 1); + assert.equal(finished[0].attributes.find(attribute => attribute.key === 'gen_ai.request.model').value.stringValue, 'gemini-2.5-flash'); + } finally { + await provider?.shutdown(); + local('@opentelemetry/api').context.disable(); + global.fetch = originalFetch; + server.close(); + await once(server, 'close'); + } +});