diff --git a/.github/workflows/langchain-typescript-tests.yml b/.github/workflows/langchain-typescript-tests.yml new file mode 100644 index 000000000..a3f9e6906 --- /dev/null +++ b/.github/workflows/langchain-typescript-tests.yml @@ -0,0 +1,111 @@ +name: LangChain TypeScript contracts + +on: + pull_request: + branches: [main, dev] + paths: + - 'typescript/packages/traceai_langchain/**' + - 'typescript/packages/fi-core/**' + - 'typescript/packages/fi-semantic-conventions/**' + - 'typescript/pnpm-lock.yaml' + - 'typescript/tsconfig*.json' + - 'typescript/jest.setup.js' + - '.github/workflows/langchain-typescript-tests.yml' + push: + branches: [main, dev] + paths: + - 'typescript/packages/traceai_langchain/**' + - 'typescript/packages/fi-core/**' + - 'typescript/packages/fi-semantic-conventions/**' + - 'typescript/pnpm-lock.yaml' + - 'typescript/tsconfig*.json' + - 'typescript/jest.setup.js' + - '.github/workflows/langchain-typescript-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: langchain-typescript-${{ github.ref }} + cancel-in-progress: true + +jobs: + contracts: + name: LangChain core ${{ matrix.core }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + core: ['0.2.0', '0.3.0', '0.3.80', '1.0.0', '1.3.2'] + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22.23.4' + - name: Install locked workspace and build package tarballs + working-directory: typescript + run: | + corepack enable + corepack pnpm --filter '@traceai/langchain...' install --frozen-lockfile --ignore-scripts + corepack pnpm run build:deps + corepack pnpm --filter @traceai/langchain run build + mkdir -p "$RUNNER_TEMP/langchain-tarballs" + for package in fi-semantic-conventions fi-core traceai_langchain; do + corepack pnpm --dir "packages/$package" pack --pack-destination "$RUNNER_TEMP/langchain-tarballs" + done + git diff --exit-code -- pnpm-lock.yaml + - name: Install isolated consumer without bypassing peers + env: + CORE_VERSION: ${{ matrix.core }} + run: | + mkdir -p "$RUNNER_TEMP/langchain-consumer" + npm install --prefix "$RUNNER_TEMP/langchain-consumer" --ignore-scripts --no-audit --no-fund \ + "$RUNNER_TEMP/langchain-tarballs/"*.tgz "@langchain/core@$CORE_VERSION" \ + @opentelemetry/sdk-trace-base@2.0.1 @opentelemetry/exporter-trace-otlp-http@0.202.0 + cp typescript/packages/traceai_langchain/contract/lifecycle.mjs "$RUNNER_TEMP/langchain-consumer/" + cp typescript/packages/traceai_langchain/contract/types.mts "$RUNNER_TEMP/langchain-consumer/" + cp typescript/packages/traceai_langchain/contract/types.mts "$RUNNER_TEMP/langchain-consumer/types.cts" + - name: Install the README Quick Start in an empty consumer + if: matrix.core == '1.3.2' + run: | + node typescript/packages/traceai_langchain/contract/readme.mjs install \ + typescript/packages/traceai_langchain/README.md \ + "$RUNNER_TEMP/langchain-readme" "$RUNNER_TEMP/langchain-tarballs" + - name: Run the exact README Quick Start, localhost only + if: matrix.core == '1.3.2' + run: | + sudo unshare --net bash -c 'ip link set lo up; exec "$@"' -- \ + "$(command -v node)" typescript/packages/traceai_langchain/contract/readme.mjs run \ + typescript/packages/traceai_langchain/README.md \ + "$RUNNER_TEMP/langchain-readme" "$RUNNER_TEMP/langchain-tarballs" + - name: Type-check packed public API with the consumer's core version + run: | + typescript/node_modules/.bin/tsc --noEmit --strict --skipLibCheck --module nodenext --target es2022 \ + "$RUNNER_TEMP/langchain-consumer/types.mts" "$RUNNER_TEMP/langchain-consumer/types.cts" + - name: Unit regressions with outbound network disabled + working-directory: typescript + run: | + sudo unshare --net bash -c 'ip link set lo up; exec "$@"' -- \ + "$(command -v node)" packages/traceai_langchain/node_modules/jest/bin/jest.js \ + --config packages/traceai_langchain/jest.config.js --runInBand + - name: Real callback lifecycle and OTLP wire contracts, localhost only + env: + CORE_VERSION: ${{ matrix.core }} + run: | + modes='manual-cjs manual-esm' + # Early cores have no root package export for OTel's require hook; + # they use the documented explicit module setup instead. + case "$CORE_VERSION" in + 0.2.0|0.3.0) ;; + *) modes="$modes automatic" ;; + esac + for mode in $modes; do + for privacy in public private; do + sudo unshare --net bash -c 'ip link set lo up; exec "$@"' -- \ + "$(command -v node)" "$RUNNER_TEMP/langchain-consumer/lifecycle.mjs" "$mode" "$privacy" + done + done diff --git a/typescript/packages/traceai_langchain/README.md b/typescript/packages/traceai_langchain/README.md index 8704bf902..d0031d441 100644 --- a/typescript/packages/traceai_langchain/README.md +++ b/typescript/packages/traceai_langchain/README.md @@ -4,21 +4,31 @@ OpenTelemetry instrumentation for LangChain.js. This package provides automatic ## Installation +Use Node.js 22 or later. The Quick Start uses the OpenAI adapter with LangChain +core 1.x. Install every package imported by the example and a TypeScript runner: + ```bash -npm install @traceai/langchain -# or -yarn add @traceai/langchain -# or -pnpm add @traceai/langchain +npm install @traceai/langchain @traceai/fi-core @langchain/core@^1 @langchain/openai@^1 \ + @opentelemetry/api@^1.9.0 @opentelemetry/instrumentation@^0.202.0 +npm install --save-dev tsx@^4 ``` ## Quick Start -```typescript -// export Future AGI API KEYS -// export FI_API_KEY=your_api_key -// export FI_SECRET_KEY=your_secret_key +Set your Future AGI API keys and an OpenAI API key in the same terminal where you +will run the example. Replace the placeholder values with your own credentials; +the OpenAI key is required for the model call, separately from tracing. + +```bash +export FI_API_KEY="your_future_agi_api_key" +export FI_SECRET_KEY="your_future_agi_secret_key" +export OPENAI_API_KEY="your_openai_api_key" +``` +Save the following as `quickstart.mts`. The `.mts` extension enables ES modules +and top-level `await` without changing your project's module settings. + +```typescript import { register, ProjectType } from "@traceai/fi-core"; import { registerInstrumentations } from "@opentelemetry/instrumentation"; import { LangChainInstrumentation } from "@traceai/langchain"; @@ -26,6 +36,7 @@ import { ChatOpenAI } from "@langchain/openai"; import { HumanMessage } from "@langchain/core/messages"; import { diag, DiagConsoleLogger, DiagLogLevel } from "@opentelemetry/api"; import * as CallbackManagerModule from "@langchain/core/callbacks/manager"; +import { awaitAllCallbacks } from "@langchain/core/callbacks/promises"; // Enable OpenTelemetry internal diagnostics (optional, for debugging) diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG); @@ -55,39 +66,47 @@ const chatModel = new ChatOpenAI({ }, }); -// Your LangChain code here... - -// 5. Don't forget to shutdown the tracer provider when done try { + const response = await chatModel.invoke([new HumanMessage("What is OpenTelemetry?")]); + console.log(response.content); +} finally { + // LangChain runs tracing callbacks in the background. Finish them before + // shutting down the provider so the root span is not lost. + await awaitAllCallbacks(); await tracerProvider.shutdown(); - console.log("Tracer provider shut down successfully."); -} catch (error) { - console.error("Error shutting down tracer provider:", error); } ``` -## Environment Variables - -The following environment variables are required for telemetry: +Run the example from the directory where you installed the packages: ```bash -FI_API_KEY=your_api_key -FI_SECRET_KEY=your_secret_key +npx tsx quickstart.mts ``` +The terminal prints the model's response. Open the `your-project-name` project in +Future AGI to inspect the `ChatOpenAI` span. +For a long-running service, leave the provider running between requests. When the +service stops, finish in-flight chains, await `awaitAllCallbacks()`, then shut down +the provider. Before an explicit `forceFlush()`, drain callbacks in the same order. ## Features - Automatic tracing of LangChain operations - Support for both ESM and CommonJS modules -- Compatible with LangChain.js v0.2.0 and v0.3.0 +- Compatible with `@langchain/core` 0.2.x, 0.3.x, and 1.x - Integration with TraceAI's observability platform ## Peer Dependencies This package requires the following peer dependencies: -- `@langchain/core`: ^0.2.0 || ^0.3.0 +- `@langchain/core`: ^0.2.0 || ^0.3.0 || ^1.0.0 + +Keep your LangChain adapters on a release compatible with your core version. For +example, use `@langchain/openai` 1.x with core 1.x. On core 0.3.80 and 1.x, +CommonJS applications can use the automatic require hook when instrumentation is +initialized before importing LangChain. For ESM, an already-loaded module, core +0.2.x, or earlier 0.3.x, use `manuallyInstrument()` as above. ## Development diff --git a/typescript/packages/traceai_langchain/contract/lifecycle.mjs b/typescript/packages/traceai_langchain/contract/lifecycle.mjs new file mode 100644 index 000000000..eba6cdde7 --- /dev/null +++ b/typescript/packages/traceai_langchain/contract/lifecycle.mjs @@ -0,0 +1,118 @@ +// Copy this fixture beside an isolated package.json after installing the built +// tarballs and the desired @langchain/core version. No model or vendor calls. +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import { createServer } from "node:http"; +import { once } from "node:events"; + +const require = createRequire(import.meta.url); +const mode = process.argv[2] ?? "automatic"; +const hidden = process.argv[3] === "private"; +assert.ok(["automatic", "manual-cjs", "manual-esm"].includes(mode)); +// Exercise the public ESM export as well as CommonJS, without crossing the two +// separate LangChain module graphs when manually instrumenting them. +const load = mode === "manual-esm" ? (name) => import(name) : async (name) => require(name); +const { LangChainInstrumentation, isPatched } = await load("@traceai/langchain"); +const { BasicTracerProvider, SimpleSpanProcessor } = await load("@opentelemetry/sdk-trace-base"); +const { OTLPTraceExporter } = await load("@opentelemetry/exporter-trace-otlp-http"); +const { SemanticConventions } = await load("@traceai/fi-semantic-conventions"); + +const requests = []; +const collector = createServer(async (request, response) => { + const parts = []; + for await (const part of request) parts.push(part); + requests.push({ url: request.url, body: JSON.parse(Buffer.concat(parts).toString()) }); + response.writeHead(200, { "content-type": "application/json" }); + response.end("{}"); +}); +collector.listen(0, "127.0.0.1"); +await once(collector, "listening"); +const unfinished = new Set(); +const provider = new BasicTracerProvider({ + spanProcessors: [ + { + onStart(span) { unfinished.add(span.spanContext().spanId); }, + onEnd(span) { unfinished.delete(span.spanContext().spanId); }, + async forceFlush() {}, + async shutdown() {}, + }, + new SimpleSpanProcessor(new OTLPTraceExporter({ + url: `http://127.0.0.1:${collector.address().port}/v1/traces`, + })), + ], +}); +const instrumentation = new LangChainInstrumentation({ + tracerProvider: provider, + traceConfig: hidden ? { hideInputs: true, hideOutputs: true, hideErrorDetails: true } : {}, +}); + +try { + // Loading these AFTER initialization is essential to the automatic hook test. + const manager = await load("@langchain/core/callbacks/manager"); + if (mode !== "automatic") { + instrumentation.manuallyInstrument(manager); + instrumentation.manuallyInstrument(manager); // Repeated setup must not duplicate spans. + } + assert.equal(isPatched(), true, `${mode} must patch core ${require("@langchain/core/package.json").version}`); + const { RunnableLambda } = await load("@langchain/core/runnables"); + const { awaitAllCallbacks } = await load("@langchain/core/callbacks/promises"); + const secret = "synthetic-private-input"; + const makeChain = (name) => RunnableLambda.from((input) => `${input}-first`) + .withConfig({ runName: `${name}_first` }) + .pipe(RunnableLambda.from((input) => `${input}-second`).withConfig({ runName: `${name}_second` })) + .withConfig({ runName: `${name}_root` }); + + assert.equal(await makeChain("invoke").invoke(secret), `${secret}-first-second`); + { + const chunks = []; + for await (const chunk of await makeChain("stream").stream(secret)) chunks.push(chunk); + assert.deepEqual(chunks, [`${secret}-first-second`]); + const originalError = new Error(`failure for ${secret}`); + const failing = RunnableLambda.from((input) => input) + .withConfig({ runName: "error_first" }) + .pipe(RunnableLambda.from(() => { throw originalError; }).withConfig({ runName: "error_second" })) + .withConfig({ runName: "error_root" }); + await assert.rejects(failing.invoke(secret), (error) => error === originalError); + // This is the documented shutdown order, including the failure path. + await awaitAllCallbacks(); + await provider.shutdown(); + } + + assert.equal(unfinished.size, 0, "every started span must end, including on transitional cores"); + assert.ok(requests.length > 0, "the real OTLP exporter must reach the local collector"); + assert.ok(requests.every((request) => request.url === "/v1/traces")); + const spans = requests.flatMap(({ body }) => body.resourceSpans.flatMap( + (resource) => resource.scopeSpans.flatMap((scope) => scope.spans), + )); + const names = spans.map((span) => span.name); + { + assert.equal(spans.length, 9, "three nested spans per invocation, stream, and error; no duplicates"); + assert.equal(new Set(spans.map((span) => span.spanId)).size, 9); + for (const operation of ["invoke", "stream", "error"]) { + const root = spans.find((span) => span.name === `${operation}_root`); + assert.ok(root, `${operation} root must survive shutdown`); + assert.ok(!root.parentSpanId, "a root must have no parent"); + for (const step of ["first", "second"]) { + const child = spans.find((span) => span.name === `${operation}_${step}`); + assert.ok(child); + assert.equal(child.parentSpanId, root.spanId); + assert.equal(child.traceId, root.traceId); + } + } + for (const span of spans) { + const failed = span.name === "error_root" || span.name === "error_second"; + assert.equal(span.status.code, failed ? 2 : 1); + assert.ok(BigInt(span.endTimeUnixNano) >= BigInt(span.startTimeUnixNano)); + const attributes = Object.fromEntries(span.attributes.map(({ key, value }) => [key, value])); + assert.equal(attributes[SemanticConventions.FI_SPAN_KIND]?.stringValue, "CHAIN"); + if (!hidden) assert.ok(attributes[SemanticConventions.INPUT_VALUE]); + } + if (hidden) assert.ok(!JSON.stringify(requests).includes(secret), "private content must not reach OTLP"); + else assert.ok(JSON.stringify(requests).includes(secret)); + } + console.log(JSON.stringify({ core: require("@langchain/core/package.json").version, mode, hidden, spans: names, result: "PASS" })); +} finally { + instrumentation.disable(); + await provider.shutdown(); + await new Promise((resolve, reject) => collector.close((error) => error ? reject(error) : resolve())); +} diff --git a/typescript/packages/traceai_langchain/contract/readme.mjs b/typescript/packages/traceai_langchain/contract/readme.mjs new file mode 100644 index 000000000..ac0f7a39a --- /dev/null +++ b/typescript/packages/traceai_langchain/contract/readme.mjs @@ -0,0 +1,118 @@ +// Install only the README's dependencies, then run its exact example offline. +// Usage: node readme.mjs install|run README.md CONSUMER_DIR TARBALL_DIR +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { once } from "node:events"; +import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { join, resolve } from "node:path"; + +const [phase, readmePath, consumerPath, tarballPath] = process.argv.slice(2); +assert.ok(["install", "run"].includes(phase)); +const consumer = resolve(consumerPath); +const readme = await readFile(readmePath, "utf8"); +const setup = readme.split("## Features")[0].replace(/\\\r?\n\s*/g, " "); +const commands = [...setup.matchAll(/^npm install (.+)$/gm)].map((m) => m[1].split(/\s+/)); +const source = setup.match(/```typescript\n([\s\S]*?)\n```/)[1]; +const packageName = (specifier) => specifier.startsWith("@") + ? specifier.split("/").slice(0, 2).join("/").replace(/@[^/]*$/, "") + : specifier.split("/")[0].split("@")[0]; +const dependencies = commands.flat().filter((arg) => !arg.startsWith("-")); +for (const [, specifier] of source.matchAll(/from "([^"]+)"/g)) { + assert.ok(dependencies.some((dep) => packageName(dep) === packageName(specifier)), + `README must explicitly install ${packageName(specifier)}`); +} +assert.ok(dependencies.some((dep) => packageName(dep) === "tsx"), "README must install its TypeScript runner"); +for (const name of ["FI_API_KEY", "FI_SECRET_KEY", "OPENAI_API_KEY"]) { + assert.match(setup, new RegExp(`^export ${name}=`, "m"), `README must export ${name}`); +} +const runner = setup.match(/^npx (tsx [\w.-]+\.mts)$/m)?.[1].split(" "); +assert.ok(runner, "README must include an executable ESM TypeScript command"); +const env = { + PATH: process.env.PATH, + HOME: join(consumer, ".home"), + TMPDIR: process.env.TMPDIR ?? "/tmp", + NPM_CONFIG_USERCONFIG: "/dev/null", + NPM_CONFIG_CACHE: join(consumer, ".npm-cache"), +}; + +if (phase === "install") { + await mkdir(consumer, { recursive: true }); + assert.equal((await readdir(consumer)).length, 0, "Install must start in an empty consumer"); + await mkdir(env.HOME); + const tarballs = await readdir(tarballPath); + for (const command of commands) { + // Replace only documented Future AGI dependencies with the candidate release. + // No preinstalled adapters, runner, peer bypasses or extra dependencies. + const args = command.map((dep) => { + if (!dep.startsWith("@traceai/")) return dep; + const prefix = packageName(dep).slice(1).replace("/", "-") + "-"; + const matches = tarballs.filter((name) => name.startsWith(prefix) && name.endsWith(".tgz")); + assert.equal(matches.length, 1, `Expected exactly one tarball for ${dep}`); + return resolve(tarballPath, matches[0]); + }); + const result = spawnSync("npm", ["install", "--ignore-scripts", "--no-audit", "--no-fund", ...args], + { cwd: consumer, env, stdio: "inherit" }); + assert.equal(result.status, 0, `README install failed: ${result.error ?? result.signal ?? "npm"}`); + } + console.log("PASS: installed only the documented dependencies in an empty consumer"); +} else { + const packedReadme = await readFile(join(consumer, "node_modules/@traceai/langchain/README.md"), "utf8"); + assert.equal(packedReadme, readme, "Run the README that actually ships in the tarball"); + const spans = []; + const requests = []; + const server = createServer(async (request, response) => { + const chunks = []; + for await (const chunk of request) chunks.push(chunk); + const body = JSON.parse(Buffer.concat(chunks).toString()); + requests.push(request.url); + response.setHeader("content-type", "application/json"); + if (request.url === "/v1/chat/completions") { + assert.equal(request.headers.authorization, "Bearer synthetic-openai-key"); + assert.equal(body.messages.at(-1).content, "What is OpenTelemetry?"); + response.end(JSON.stringify({ + id: "synthetic-completion", object: "chat.completion", created: 1, model: body.model, + choices: [{ index: 0, message: { role: "assistant", content: "Synthetic tracing example." }, finish_reason: "stop" }], + usage: { prompt_tokens: 5, completion_tokens: 4, total_tokens: 9 }, + })); + } else if (request.url === "/tracer/v1/traces") { + for (const resource of body.resourceSpans) { + const attrs = Object.fromEntries(resource.resource.attributes.map((a) => [a.key, a.value.stringValue])); + assert.equal(attrs["service.name"], "your-project-name"); + for (const scope of resource.scopeSpans) spans.push(...scope.spans); + } + response.end("{}"); + } else { + response.statusCode = 404; + response.end("{}"); + } + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + try { + await writeFile(join(consumer, runner[1]), source + "\n"); + const baseUrl = `http://127.0.0.1:${server.address().port}`; + const child = spawn("npx", runner, { cwd: consumer, env: { + ...env, FI_API_KEY: "synthetic-fi-key", FI_SECRET_KEY: "synthetic-fi-secret", + OPENAI_API_KEY: "synthetic-openai-key", FI_BASE_URL: baseUrl, OPENAI_BASE_URL: `${baseUrl}/v1`, + }, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = "", stderr = ""; + child.stdout.on("data", (data) => { stdout += data; }); + child.stderr.on("data", (data) => { stderr += data; }); + const watchdog = setTimeout(() => child.kill("SIGKILL"), 30000); + try { + const [code, signal] = await once(child, "exit"); + assert.equal(code, 0, `README command failed (${signal}): ${stderr}`); + } finally { clearTimeout(watchdog); } + assert.ok(stdout.includes("Synthetic tracing example.")); + assert.deepEqual(requests.sort(), ["/tracer/v1/traces", "/v1/chat/completions"]); + assert.equal(spans.length, 1, "No missing or duplicate model span"); + assert.equal(spans[0].name, "ChatOpenAI"); + assert.notEqual(spans[0].status?.code, 2); + assert.ok(BigInt(spans[0].endTimeUnixNano) > BigInt(spans[0].startTimeUnixNano)); + console.log("PASS: exact README runner, real ChatOpenAI, response and completed OTLP span"); + } finally { + server.closeAllConnections(); + await new Promise((done) => server.close(done)); + } +} diff --git a/typescript/packages/traceai_langchain/contract/types.mts b/typescript/packages/traceai_langchain/contract/types.mts new file mode 100644 index 000000000..498b77c28 --- /dev/null +++ b/typescript/packages/traceai_langchain/contract/types.mts @@ -0,0 +1,18 @@ +import { LangChainInstrumentation } from "@traceai/langchain"; +import * as CallbackManagerModule from "@langchain/core/callbacks/manager"; +import { awaitAllCallbacks } from "@langchain/core/callbacks/promises"; +import { RunnableLambda } from "@langchain/core/runnables"; +import { BasicTracerProvider } from "@opentelemetry/sdk-trace-base"; + +// Type-check the built public declarations against each consumer's core version, +// rather than only the version used to build this workspace. +async function example(): Promise { + const provider = new BasicTracerProvider(); + const instrumentation = new LangChainInstrumentation({ tracerProvider: provider }); + instrumentation.manuallyInstrument(CallbackManagerModule); + const result: number = await RunnableLambda.from((input: number) => input + 1).invoke(3); + await awaitAllCallbacks(); + await provider.shutdown(); + return result; +} +void example; diff --git a/typescript/packages/traceai_langchain/package.json b/typescript/packages/traceai_langchain/package.json index 4fe7c3162..542dbaf85 100644 --- a/typescript/packages/traceai_langchain/package.json +++ b/typescript/packages/traceai_langchain/package.json @@ -8,9 +8,14 @@ "types": "./dist/src/index.d.ts", "exports": { ".": { - "types": "./dist/src/index.d.ts", - "import": "./dist/esm/index.js", - "require": "./dist/src/index.js" + "import": { + "types": "./dist/esm/index.d.ts", + "default": "./dist/esm/index.js" + }, + "require": { + "types": "./dist/src/index.d.ts", + "default": "./dist/src/index.js" + } } }, "scripts": { @@ -43,7 +48,7 @@ "@traceai/fi-semantic-conventions": "workspace:^" }, "peerDependencies": { - "@langchain/core": "^0.2.0 || ^0.3.0" + "@langchain/core": "^0.2.0 || ^0.3.0 || ^1.0.0" }, "devDependencies": { "@langchain/openai": "0.3.16", diff --git a/typescript/packages/traceai_langchain/src/__tests__/instrumentation.test.ts b/typescript/packages/traceai_langchain/src/__tests__/instrumentation.test.ts index 518cf61f5..354f6c74d 100644 --- a/typescript/packages/traceai_langchain/src/__tests__/instrumentation.test.ts +++ b/typescript/packages/traceai_langchain/src/__tests__/instrumentation.test.ts @@ -186,10 +186,16 @@ describe('LangChain Instrumentation', () => { mockCallbackManagerModule.CallbackManager.configure = jest.fn(); }); - it('should patch configure when _configureSync is not available', () => { + it('should inject a tracer through configure when _configureSync is not available', () => { + const original = mockCallbackManagerModule.CallbackManager.configure; + const returned = Promise.resolve({ handlers: [] }); + original.mockReturnValue(returned); instrumentation.manuallyInstrument(mockCallbackManagerModule as any); - expect(mockCallbackManagerModule.CallbackManager.configure).toBeDefined(); + const result = mockCallbackManagerModule.CallbackManager.configure([]); + expect(result).toBe(returned); + expect(original).toHaveBeenCalledTimes(1); + expect(original.mock.calls[0][0]).toHaveLength(1); }); }); diff --git a/typescript/packages/traceai_langchain/src/__tests__/lifecycle.test.ts b/typescript/packages/traceai_langchain/src/__tests__/lifecycle.test.ts new file mode 100644 index 000000000..247e51d29 --- /dev/null +++ b/typescript/packages/traceai_langchain/src/__tests__/lifecycle.test.ts @@ -0,0 +1,61 @@ +import { awaitAllCallbacks } from "@langchain/core/callbacks/promises"; +import { RunnableLambda } from "@langchain/core/runnables"; +import type { Run } from "@langchain/core/tracers/base"; +import { BasicTracerProvider, ReadableSpan, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; +import { FITracer } from "@traceai/fi-core"; +import { LangChainTracer } from "../tracer"; + +// Hold a background root callback open without timing assumptions. SDK shutdown +// alone cannot drain work that LangChain has not finished yet. +async function runWithPendingRoot(drainBeforeShutdown: boolean) { + let releaseRoot!: () => void; + const rootGate = new Promise((resolve) => { releaseRoot = resolve; }); + class PendingRootTracer extends LangChainTracer { + protected async persistRun(run: Run): Promise { + await rootGate; + await super.persistRun(run); + } + } + const spans: ReadableSpan[] = []; + const provider = new BasicTracerProvider({ + spanProcessors: [new SimpleSpanProcessor({ + export(batch, done) { spans.push(...batch); done({ code: 0 }); }, + async shutdown() {}, + })], + }); + const tracer = new PendingRootTracer(new FITracer({ tracer: provider.getTracer("lifecycle-test") })); + const chain = RunnableLambda.from((input: number) => input + 1) + .withConfig({ runName: "first" }) + .pipe(RunnableLambda.from((input: number) => input * 2).withConfig({ runName: "second" })) + .withConfig({ runName: "root" }); + try { + expect(await chain.invoke(3, { callbacks: [tracer] })).toBe(8); + expect(spans.map((span) => span.name)).toEqual(["first", "second"]); + if (!drainBeforeShutdown) await provider.shutdown(); + releaseRoot(); + await awaitAllCallbacks(); + if (drainBeforeShutdown) await provider.shutdown(); + return spans; + } finally { + releaseRoot(); + await awaitAllCallbacks(); + await provider.shutdown(); + } +} + +describe("LangChain background callback shutdown", () => { + it("loses a pending root when the provider is shut down first (negative control)", async () => { + const spans = await runWithPendingRoot(false); + expect(spans.map((span) => span.name)).toEqual(["first", "second"]); + }); + + it("exports the complete tree when callbacks are drained before shutdown", async () => { + const spans = await runWithPendingRoot(true); + expect(spans.map((span) => span.name)).toEqual(["first", "second", "root"]); + const root = spans[2]; + for (const child of spans.slice(0, 2)) { + expect(child.parentSpanContext?.spanId).toBe(root.spanContext().spanId); + expect(child.spanContext().traceId).toBe(root.spanContext().traceId); + } + }); +}); diff --git a/typescript/packages/traceai_langchain/src/instrumentation.ts b/typescript/packages/traceai_langchain/src/instrumentation.ts index 41ae437de..e995f58bb 100644 --- a/typescript/packages/traceai_langchain/src/instrumentation.ts +++ b/typescript/packages/traceai_langchain/src/instrumentation.ts @@ -91,14 +91,12 @@ export class LangChainInstrumentation extends InstrumentationBase { } protected init(): InstrumentationModuleDefinition { - const module = - new InstrumentationNodeModuleDefinition( - "@langchain/core/dist/callbacks/manager.cjs", - ["^0.1.0", "^0.2.0", "^0.3.0"], - this.patch.bind(this), - this.unpatch.bind(this), - ); - return module; + return new InstrumentationNodeModuleDefinition( + "@langchain/core/dist/callbacks/manager.cjs", + ["^0.1.0", "^0.2.0", "^0.3.0", "^1.0.0"], + this.patch.bind(this), + this.unpatch.bind(this), + ); } get tracer(): Tracer { @@ -141,12 +139,9 @@ export class LangChainInstrumentation extends InstrumentationBase { const instrumentation = this; /** - * _configureSync is only available in v0.2.0 and above - * It was added as a replacement to the configure method which is marked as soon to be deprecated - * In v0.2.0 and above, the configure method is a wrapper around _configureSync - * However, configure is not always called, where as _configureSync is always called - * so we want to patch only configure sync if it's available - * and only configure if _configureSync is not available so we don't get duplicate traces + * Later 0.2.x releases added _configureSync; the 0.2.0 floor only has + * configure. Patch exactly one entry point, since newer configure methods + * delegate to _configureSync and wrapping both would duplicate tracing. */ if ("_configureSync" in module.CallbackManager) { this._wrap(module.CallbackManager, "_configureSync", (original) => { @@ -163,6 +158,16 @@ export class LangChainInstrumentation extends InstrumentationBase { ); args[0] = newInheritableHandlers; + return original.apply(this, args); + }; + }); + } else { + this._wrap(module.CallbackManager, "configure", (original) => { + return function ( + this: typeof CallbackManagerModuleV02, + ...args: Parameters + ) { + args[0] = addTracerToHandlers(instrumentation.fiTracer, args[0]); return original.apply(this, args); }; }); diff --git a/typescript/packages/traceai_langchain/src/tracer.ts b/typescript/packages/traceai_langchain/src/tracer.ts index c5999c0f9..f0e9921a3 100644 --- a/typescript/packages/traceai_langchain/src/tracer.ts +++ b/typescript/packages/traceai_langchain/src/tracer.ts @@ -157,7 +157,11 @@ export class LangChainTracer extends BaseTracer { // @ts-ignore await super._startTrace(run); } - await this.startTracing(run); + // Transitional cores expose both _startTrace and onRunCreate. The super + // call can already have created the span through onRunCreate. + if (this.runs[run.id] == null) { + await this.startTracing(run); + } } /** diff --git a/typescript/pnpm-lock.yaml b/typescript/pnpm-lock.yaml index c6feab057..fc925a23f 100644 --- a/typescript/pnpm-lock.yaml +++ b/typescript/pnpm-lock.yaml @@ -1130,7 +1130,7 @@ importers: packages/traceai_langchain: dependencies: '@langchain/core': - specifier: ^0.2.0 || ^0.3.0 + specifier: ^0.2.0 || ^0.3.0 || ^1.0.0 version: 0.3.56(openai@4.98.0(ws@8.22.0)(zod@3.24.4)) '@opentelemetry/api': specifier: ^1.9.0