Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 111 additions & 0 deletions .github/workflows/langchain-typescript-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
name: LangChain TypeScript contracts

on:
pull_request:
branches: [main, dev]
paths:
- 'typescript/packages/traceai_langchain/**'
- 'typescript/packages/fi-core/**'
- 'typescript/packages/fi-semantic-conventions/**'
- 'typescript/pnpm-lock.yaml'
- 'typescript/tsconfig*.json'
- 'typescript/jest.setup.js'
- '.github/workflows/langchain-typescript-tests.yml'
push:
branches: [main, dev]
paths:
- 'typescript/packages/traceai_langchain/**'
- 'typescript/packages/fi-core/**'
- 'typescript/packages/fi-semantic-conventions/**'
- 'typescript/pnpm-lock.yaml'
- 'typescript/tsconfig*.json'
- 'typescript/jest.setup.js'
- '.github/workflows/langchain-typescript-tests.yml'
workflow_dispatch:

permissions:
contents: read

concurrency:
group: langchain-typescript-${{ github.ref }}
cancel-in-progress: true

jobs:
contracts:
name: LangChain core ${{ matrix.core }}
runs-on: ubuntu-24.04
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
core: ['0.2.0', '0.3.0', '0.3.80', '1.0.0', '1.3.2']
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
persist-credentials: false
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: '22.23.4'
- name: Install locked workspace and build package tarballs
working-directory: typescript
run: |
corepack enable
corepack pnpm --filter '@traceai/langchain...' install --frozen-lockfile --ignore-scripts
corepack pnpm run build:deps
corepack pnpm --filter @traceai/langchain run build
mkdir -p "$RUNNER_TEMP/langchain-tarballs"
for package in fi-semantic-conventions fi-core traceai_langchain; do
corepack pnpm --dir "packages/$package" pack --pack-destination "$RUNNER_TEMP/langchain-tarballs"
done
git diff --exit-code -- pnpm-lock.yaml
- name: Install isolated consumer without bypassing peers
env:
CORE_VERSION: ${{ matrix.core }}
run: |
mkdir -p "$RUNNER_TEMP/langchain-consumer"
npm install --prefix "$RUNNER_TEMP/langchain-consumer" --ignore-scripts --no-audit --no-fund \
"$RUNNER_TEMP/langchain-tarballs/"*.tgz "@langchain/core@$CORE_VERSION" \
@opentelemetry/sdk-trace-base@2.0.1 @opentelemetry/exporter-trace-otlp-http@0.202.0
cp typescript/packages/traceai_langchain/contract/lifecycle.mjs "$RUNNER_TEMP/langchain-consumer/"
cp typescript/packages/traceai_langchain/contract/types.mts "$RUNNER_TEMP/langchain-consumer/"
cp typescript/packages/traceai_langchain/contract/types.mts "$RUNNER_TEMP/langchain-consumer/types.cts"
- name: Install the README Quick Start in an empty consumer
if: matrix.core == '1.3.2'
run: |
node typescript/packages/traceai_langchain/contract/readme.mjs install \
typescript/packages/traceai_langchain/README.md \
"$RUNNER_TEMP/langchain-readme" "$RUNNER_TEMP/langchain-tarballs"
- name: Run the exact README Quick Start, localhost only
if: matrix.core == '1.3.2'
run: |
sudo unshare --net bash -c 'ip link set lo up; exec "$@"' -- \
"$(command -v node)" typescript/packages/traceai_langchain/contract/readme.mjs run \
typescript/packages/traceai_langchain/README.md \
"$RUNNER_TEMP/langchain-readme" "$RUNNER_TEMP/langchain-tarballs"
- name: Type-check packed public API with the consumer's core version
run: |
typescript/node_modules/.bin/tsc --noEmit --strict --skipLibCheck --module nodenext --target es2022 \
"$RUNNER_TEMP/langchain-consumer/types.mts" "$RUNNER_TEMP/langchain-consumer/types.cts"
- name: Unit regressions with outbound network disabled
working-directory: typescript
run: |
sudo unshare --net bash -c 'ip link set lo up; exec "$@"' -- \
"$(command -v node)" packages/traceai_langchain/node_modules/jest/bin/jest.js \
--config packages/traceai_langchain/jest.config.js --runInBand
- name: Real callback lifecycle and OTLP wire contracts, localhost only
env:
CORE_VERSION: ${{ matrix.core }}
run: |
modes='manual-cjs manual-esm'
# Early cores have no root package export for OTel's require hook;
# they use the documented explicit module setup instead.
case "$CORE_VERSION" in
0.2.0|0.3.0) ;;
*) modes="$modes automatic" ;;
esac
for mode in $modes; do
for privacy in public private; do
sudo unshare --net bash -c 'ip link set lo up; exec "$@"' -- \
"$(command -v node)" "$RUNNER_TEMP/langchain-consumer/lifecycle.mjs" "$mode" "$privacy"
done
done
63 changes: 41 additions & 22 deletions typescript/packages/traceai_langchain/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,28 +4,39 @@ OpenTelemetry instrumentation for LangChain.js. This package provides automatic

## Installation

Use Node.js 22 or later. The Quick Start uses the OpenAI adapter with LangChain
core 1.x. Install every package imported by the example and a TypeScript runner:

```bash
npm install @traceai/langchain
# or
yarn add @traceai/langchain
# or
pnpm add @traceai/langchain
npm install @traceai/langchain @traceai/fi-core @langchain/core@^1 @langchain/openai@^1 \
@opentelemetry/api@^1.9.0 @opentelemetry/instrumentation@^0.202.0
npm install --save-dev tsx@^4
```

## Quick Start

```typescript
// export Future AGI API KEYS
// export FI_API_KEY=your_api_key
// export FI_SECRET_KEY=your_secret_key
Set your Future AGI API keys and an OpenAI API key in the same terminal where you
will run the example. Replace the placeholder values with your own credentials;
the OpenAI key is required for the model call, separately from tracing.

```bash
export FI_API_KEY="your_future_agi_api_key"
export FI_SECRET_KEY="your_future_agi_secret_key"
export OPENAI_API_KEY="your_openai_api_key"
```

Save the following as `quickstart.mts`. The `.mts` extension enables ES modules
and top-level `await` without changing your project's module settings.

```typescript
import { register, ProjectType } from "@traceai/fi-core";
import { registerInstrumentations } from "@opentelemetry/instrumentation";
import { LangChainInstrumentation } from "@traceai/langchain";
import { ChatOpenAI } from "@langchain/openai";
import { HumanMessage } from "@langchain/core/messages";
import { diag, DiagConsoleLogger, DiagLogLevel } from "@opentelemetry/api";
import * as CallbackManagerModule from "@langchain/core/callbacks/manager";
import { awaitAllCallbacks } from "@langchain/core/callbacks/promises";

// Enable OpenTelemetry internal diagnostics (optional, for debugging)
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG);
Expand Down Expand Up @@ -55,39 +66,47 @@ const chatModel = new ChatOpenAI({
},
});

// Your LangChain code here...

// 5. Don't forget to shutdown the tracer provider when done
try {
const response = await chatModel.invoke([new HumanMessage("What is OpenTelemetry?")]);
console.log(response.content);
} finally {
// LangChain runs tracing callbacks in the background. Finish them before
// shutting down the provider so the root span is not lost.
await awaitAllCallbacks();
await tracerProvider.shutdown();
console.log("Tracer provider shut down successfully.");
} catch (error) {
console.error("Error shutting down tracer provider:", error);
}
```

## Environment Variables

The following environment variables are required for telemetry:
Run the example from the directory where you installed the packages:

```bash
FI_API_KEY=your_api_key
FI_SECRET_KEY=your_secret_key
npx tsx quickstart.mts
```

The terminal prints the model's response. Open the `your-project-name` project in
Future AGI to inspect the `ChatOpenAI` span.

For a long-running service, leave the provider running between requests. When the
service stops, finish in-flight chains, await `awaitAllCallbacks()`, then shut down
the provider. Before an explicit `forceFlush()`, drain callbacks in the same order.

## Features

- Automatic tracing of LangChain operations
- Support for both ESM and CommonJS modules
- Compatible with LangChain.js v0.2.0 and v0.3.0
- Compatible with `@langchain/core` 0.2.x, 0.3.x, and 1.x
- Integration with TraceAI's observability platform

## Peer Dependencies

This package requires the following peer dependencies:
- `@langchain/core`: ^0.2.0 || ^0.3.0
- `@langchain/core`: ^0.2.0 || ^0.3.0 || ^1.0.0

Keep your LangChain adapters on a release compatible with your core version. For
example, use `@langchain/openai` 1.x with core 1.x. On core 0.3.80 and 1.x,
CommonJS applications can use the automatic require hook when instrumentation is
initialized before importing LangChain. For ESM, an already-loaded module, core
0.2.x, or earlier 0.3.x, use `manuallyInstrument()` as above.

## Development

Expand Down
118 changes: 118 additions & 0 deletions typescript/packages/traceai_langchain/contract/lifecycle.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
// Copy this fixture beside an isolated package.json after installing the built
// tarballs and the desired @langchain/core version. No model or vendor calls.
import assert from "node:assert/strict";
import { createRequire } from "node:module";
import { createServer } from "node:http";
import { once } from "node:events";

const require = createRequire(import.meta.url);
const mode = process.argv[2] ?? "automatic";
const hidden = process.argv[3] === "private";
assert.ok(["automatic", "manual-cjs", "manual-esm"].includes(mode));
// Exercise the public ESM export as well as CommonJS, without crossing the two
// separate LangChain module graphs when manually instrumenting them.
const load = mode === "manual-esm" ? (name) => import(name) : async (name) => require(name);
const { LangChainInstrumentation, isPatched } = await load("@traceai/langchain");
const { BasicTracerProvider, SimpleSpanProcessor } = await load("@opentelemetry/sdk-trace-base");
const { OTLPTraceExporter } = await load("@opentelemetry/exporter-trace-otlp-http");
const { SemanticConventions } = await load("@traceai/fi-semantic-conventions");

const requests = [];
const collector = createServer(async (request, response) => {
const parts = [];
for await (const part of request) parts.push(part);
requests.push({ url: request.url, body: JSON.parse(Buffer.concat(parts).toString()) });
response.writeHead(200, { "content-type": "application/json" });
response.end("{}");
});
collector.listen(0, "127.0.0.1");
await once(collector, "listening");
const unfinished = new Set();
const provider = new BasicTracerProvider({
spanProcessors: [
{
onStart(span) { unfinished.add(span.spanContext().spanId); },
onEnd(span) { unfinished.delete(span.spanContext().spanId); },
async forceFlush() {},
async shutdown() {},
},
new SimpleSpanProcessor(new OTLPTraceExporter({
url: `http://127.0.0.1:${collector.address().port}/v1/traces`,
})),
],
});
const instrumentation = new LangChainInstrumentation({
tracerProvider: provider,
traceConfig: hidden ? { hideInputs: true, hideOutputs: true, hideErrorDetails: true } : {},
});

try {
// Loading these AFTER initialization is essential to the automatic hook test.
const manager = await load("@langchain/core/callbacks/manager");
if (mode !== "automatic") {
instrumentation.manuallyInstrument(manager);
instrumentation.manuallyInstrument(manager); // Repeated setup must not duplicate spans.
}
assert.equal(isPatched(), true, `${mode} must patch core ${require("@langchain/core/package.json").version}`);
const { RunnableLambda } = await load("@langchain/core/runnables");
const { awaitAllCallbacks } = await load("@langchain/core/callbacks/promises");
const secret = "synthetic-private-input";
const makeChain = (name) => RunnableLambda.from((input) => `${input}-first`)
.withConfig({ runName: `${name}_first` })
.pipe(RunnableLambda.from((input) => `${input}-second`).withConfig({ runName: `${name}_second` }))
.withConfig({ runName: `${name}_root` });

assert.equal(await makeChain("invoke").invoke(secret), `${secret}-first-second`);
{
const chunks = [];
for await (const chunk of await makeChain("stream").stream(secret)) chunks.push(chunk);
assert.deepEqual(chunks, [`${secret}-first-second`]);
const originalError = new Error(`failure for ${secret}`);
const failing = RunnableLambda.from((input) => input)
.withConfig({ runName: "error_first" })
.pipe(RunnableLambda.from(() => { throw originalError; }).withConfig({ runName: "error_second" }))
.withConfig({ runName: "error_root" });
await assert.rejects(failing.invoke(secret), (error) => error === originalError);
// This is the documented shutdown order, including the failure path.
await awaitAllCallbacks();
await provider.shutdown();
}

assert.equal(unfinished.size, 0, "every started span must end, including on transitional cores");
assert.ok(requests.length > 0, "the real OTLP exporter must reach the local collector");
assert.ok(requests.every((request) => request.url === "/v1/traces"));
const spans = requests.flatMap(({ body }) => body.resourceSpans.flatMap(
(resource) => resource.scopeSpans.flatMap((scope) => scope.spans),
));
const names = spans.map((span) => span.name);
{
assert.equal(spans.length, 9, "three nested spans per invocation, stream, and error; no duplicates");
assert.equal(new Set(spans.map((span) => span.spanId)).size, 9);
for (const operation of ["invoke", "stream", "error"]) {
const root = spans.find((span) => span.name === `${operation}_root`);
assert.ok(root, `${operation} root must survive shutdown`);
assert.ok(!root.parentSpanId, "a root must have no parent");
for (const step of ["first", "second"]) {
const child = spans.find((span) => span.name === `${operation}_${step}`);
assert.ok(child);
assert.equal(child.parentSpanId, root.spanId);
assert.equal(child.traceId, root.traceId);
}
}
for (const span of spans) {
const failed = span.name === "error_root" || span.name === "error_second";
assert.equal(span.status.code, failed ? 2 : 1);
assert.ok(BigInt(span.endTimeUnixNano) >= BigInt(span.startTimeUnixNano));
const attributes = Object.fromEntries(span.attributes.map(({ key, value }) => [key, value]));
assert.equal(attributes[SemanticConventions.FI_SPAN_KIND]?.stringValue, "CHAIN");
if (!hidden) assert.ok(attributes[SemanticConventions.INPUT_VALUE]);
}
if (hidden) assert.ok(!JSON.stringify(requests).includes(secret), "private content must not reach OTLP");
else assert.ok(JSON.stringify(requests).includes(secret));
}
console.log(JSON.stringify({ core: require("@langchain/core/package.json").version, mode, hidden, spans: names, result: "PASS" }));
} finally {
instrumentation.disable();
await provider.shutdown();
await new Promise((resolve, reject) => collector.close((error) => error ? reject(error) : resolve()));
}
Loading
Loading