Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions .github/workflows/langchain-typescript-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -109,3 +109,54 @@ jobs:
"$(command -v node)" "$RUNNER_TEMP/langchain-consumer/lifecycle.mjs" "$mode" "$privacy"
done
done
- name: LangGraph pause and handoff spans, localhost only
if: matrix.core == '0.3.80' || matrix.core == '1.3.2'
env:
CORE_VERSION: ${{ matrix.core }}
run: |
# The last 0.2 and 0.x releases on core 0.3, the first 1.x and the latest on core 1.x.
case "$CORE_VERSION" in
0.3.80) langgraph_versions='0.2.74 0.4.10' ;;
1.3.2) langgraph_versions='1.0.0 1.6.0' ;;
esac
for langgraph in $langgraph_versions; do
consumer="$RUNNER_TEMP/langgraph-$langgraph"
mkdir -p "$consumer"
npm install --prefix "$consumer" --ignore-scripts --no-audit --no-fund \
"$RUNNER_TEMP/langchain-tarballs/"*.tgz "@langchain/core@$CORE_VERSION" \
"@langchain/langgraph@$langgraph" @opentelemetry/sdk-trace-base@2.0.1
cp typescript/packages/traceai_langchain/contract/langgraph.mjs "$consumer/"
for mode in manual-cjs manual-esm; do
sudo unshare --net bash -c 'ip link set lo up; exec "$@"' -- \
"$(command -v node)" "$consumer/langgraph.mjs" "$mode"
done
done
- name: Load the packed package with strict pnpm (no hoisting)
if: matrix.core == '1.3.2'
working-directory: typescript
env:
CORE_VERSION: ${{ matrix.core }}
run: |
# Only declared dependencies are resolvable, as with Yarn PnP.
consumer="$RUNNER_TEMP/langchain-strict"
mkdir -p "$consumer"
node -e '
const [consumer, tarballs, core] = process.argv.slice(1);
const fs = require("node:fs");
const tarball = (prefix) => "file:" + tarballs + "/" +
fs.readdirSync(tarballs).find((name) => name.startsWith(prefix) && /^[0-9]/.test(name.slice(prefix.length)));
const local = {
"@traceai/fi-core": tarball("traceai-fi-core-"),
"@traceai/fi-semantic-conventions": tarball("traceai-fi-semantic-conventions-"),
};
fs.writeFileSync(consumer + "/package.json", JSON.stringify({
private: true,
dependencies: { ...local, "@traceai/langchain": tarball("traceai-langchain-"), "@langchain/core": core },
pnpm: { overrides: local },
}));
' "$consumer" "$RUNNER_TEMP/langchain-tarballs" "$CORE_VERSION"
printf 'hoist=false\n' > "$consumer/.npmrc"
corepack pnpm --dir "$consumer" install --ignore-scripts
cd "$consumer"
node -e 'require("@traceai/langchain").LangChainInstrumentation.name'
node --input-type=module -e 'await import("@traceai/langchain")'
3 changes: 2 additions & 1 deletion typescript/packages/traceai_langchain/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ lcInstrumentation.manuallyInstrument(CallbackManagerModule);

// 4. Use LangChain as normal
const chatModel = new ChatOpenAI({
openAIApiKey: process.env.OPENAI_API_KEY,
apiKey: process.env.OPENAI_API_KEY,
metadata: {
session_id: "your-session-id",
},
Expand Down Expand Up @@ -93,6 +93,7 @@ the provider. Before an explicit `forceFlush()`, drain callbacks in the same ord
## Features

- Automatic tracing of LangChain operations
- LangGraph graphs: an `interrupt()` pause or a `Command` handoff is a successful span, not an error; a paused node is marked `langgraph.interrupt`
- Support for both ESM and CommonJS modules
- Compatible with `@langchain/core` 0.2.x, 0.3.x, and 1.x
- Integration with TraceAI's observability platform
Expand Down
113 changes: 113 additions & 0 deletions typescript/packages/traceai_langchain/contract/langgraph.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
// Copy this fixture beside an isolated package.json after installing the built
// tarballs with @langchain/core and a compatible @langchain/langgraph. No model
// or vendor calls. Usage: node langgraph.mjs manual-cjs|manual-esm
//
// LangGraph pauses (interrupt()) and hands off (Command.PARENT) by throwing
// control-flow signals through the chain callbacks. Those runs succeeded, so their
// spans must not be ERROR, carry exception events or leak the interrupt payload.
import assert from "node:assert/strict";
import { createRequire } from "node:module";

const require = createRequire(import.meta.url);
const mode = process.argv[2] ?? "manual-cjs";
assert.ok(["manual-cjs", "manual-esm"].includes(mode));
const load = mode === "manual-esm" ? (name) => import(name) : async (name) => require(name);
const { LangChainInstrumentation } = await load("@traceai/langchain");
const { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } =
await load("@opentelemetry/sdk-trace-base");
const { SpanStatusCode } = await load("@opentelemetry/api");

const exporter = new InMemorySpanExporter();
const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] });
new LangChainInstrumentation({ tracerProvider: provider })
.manuallyInstrument(await load("@langchain/core/callbacks/manager"));
const { StateGraph, Annotation, START, END, MemorySaver, interrupt, Command } =
await load("@langchain/langgraph");
const { awaitAllCallbacks } = await load("@langchain/core/callbacks/promises");
const versions = {
core: require("@langchain/core/package.json").version,
langgraph: require("@langchain/langgraph/package.json").version,
};

const State = Annotation.Root({ n: Annotation({ reducer: (_, next) => next, default: () => 0 }) });
const payload = "synthetic-private-approval-question";
const exceptions = (span) => span.events.filter((event) => event.name === "exception");
const isInterruptMarked = (span) => span.attributes["langgraph.interrupt"] === true;
async function finished() {
await awaitAllCallbacks();
const spans = exporter.getFinishedSpans();
exporter.reset();
return spans;
}
function assertSucceeded(spans, label) {
assert.ok(spans.length > 0, `${label}: no spans exported`);
for (const span of spans) {
assert.notEqual(span.status.code, SpanStatusCode.ERROR,
`${label}: ${span.name} exported as ERROR (${span.status.message})`);
assert.deepEqual(exceptions(span), [], `${label}: ${span.name} recorded an exception`);
assert.ok(!JSON.stringify(span.status).includes(payload), `${label}: status leaks the interrupt payload`);
}
}

try {
// 1. Human-in-the-loop pause, then resume.
const hitl = new StateGraph(State)
.addNode("ask_human", async (state) => ({ n: state.n + (interrupt(payload) ? 1 : 0) }))
.addEdge(START, "ask_human")
.addEdge("ask_human", END)
.compile({ checkpointer: new MemorySaver() });
const thread = { configurable: { thread_id: "synthetic-thread" } };
await hitl.invoke({ n: 0 }, thread);
const pending = await hitl.getState(thread);
assert.ok(pending.tasks.some((task) => task.interrupts?.length), "graph must pause on the interrupt");
let spans = await finished();
assertSucceeded(spans, "interrupt");
const pausedNode = spans.filter((span) => span.name === "ask_human");
assert.equal(pausedNode.length, 1, "one paused node span");
assert.ok(isInterruptMarked(pausedNode[0]), "the paused node is marked langgraph.interrupt");
assert.ok(pausedNode[0].events.some((event) => event.name === "interrupt"), "the paused node has an interrupt event");
assert.deepEqual(spans.filter(isInterruptMarked).map((span) => span.name), ["ask_human"]);

assert.deepEqual(await hitl.invoke(new Command({ resume: true }), thread), { n: 1 });
spans = await finished();
assertSucceeded(spans, "resume");
assert.deepEqual(spans.filter(isInterruptMarked), [], "a resumed run is not an interrupt");

// 2. Subgraph node hands off to its parent graph.
const subgraph = new StateGraph(State)
.addNode("route", async () => new Command({ graph: Command.PARENT, goto: "done", update: { n: 5 } }))
.addEdge(START, "route")
.compile();
const parent = new StateGraph(State)
.addNode("sub", subgraph, { ends: ["done"] })
.addNode("done", async (state) => ({ n: state.n + 1 }))
.addEdge(START, "sub")
.addEdge("done", END)
.compile();
assert.deepEqual(await parent.invoke({ n: 0 }), { n: 6 });
spans = await finished();
assertSucceeded(spans, "handoff");
for (const name of ["route", "sub", "done"]) {
assert.ok(spans.some((span) => span.name === name), `handoff: missing ${name} span`);
}
assert.deepEqual(spans.filter(isInterruptMarked), [], "a handoff is not an interrupt");

// 3. Negative control: a real node failure is still an error.
const failure = new Error("synthetic node failure");
const failing = new StateGraph(State)
.addNode("broken", async () => { throw failure; })
.addEdge(START, "broken")
.addEdge("broken", END)
.compile();
await assert.rejects(failing.invoke({ n: 0 }), (error) => error.message === failure.message);
spans = await finished();
const broken = spans.filter((span) => span.name === "broken");
assert.equal(broken.length, 1);
assert.equal(broken[0].status.code, SpanStatusCode.ERROR);
assert.equal(exceptions(broken[0]).length, 1);
assert.ok(!isInterruptMarked(broken[0]));

console.log(`PASS: LangGraph control flow (${mode}, core ${versions.core}, langgraph ${versions.langgraph})`);
} finally {
await provider.shutdown();
}
2 changes: 2 additions & 0 deletions typescript/packages/traceai_langchain/contract/readme.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ for (const [, specifier] of source.matchAll(/from "([^"]+)"/g)) {
`README must explicitly install ${packageName(specifier)}`);
}
assert.ok(dependencies.some((dep) => packageName(dep) === "tsx"), "README must install its TypeScript runner");
// ChatOpenAI 1.x ignores the old openAIApiKey alias; only apiKey reaches the client.
assert.doesNotMatch(source, /\bopenAIApiKey\b/, "README must pass the OpenAI key as apiKey");
for (const name of ["FI_API_KEY", "FI_SECRET_KEY", "OPENAI_API_KEY"]) {
assert.match(setup, new RegExp(`^export ${name}=`, "m"), `README must export ${name}`);
}
Expand Down
1 change: 1 addition & 0 deletions typescript/packages/traceai_langchain/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
"homepage": "https://docs.futureagi.com/docs/integrations/traceai/langchain",
"dependencies": {
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/core": "^2.0.1",
"@opentelemetry/instrumentation": "^0.202.0",
"@traceai/fi-core": "workspace:^",
"@traceai/fi-semantic-conventions": "workspace:^"
Expand Down
117 changes: 117 additions & 0 deletions typescript/packages/traceai_langchain/src/__tests__/bubbleUp.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
import { describe, it, expect, beforeEach } from "@jest/globals";
import { SpanStatusCode } from "@opentelemetry/api";
import {
BasicTracerProvider,
InMemorySpanExporter,
ReadableSpan,
SimpleSpanProcessor,
} from "@opentelemetry/sdk-trace-base";
import { BaseTracer } from "@langchain/core/tracers/base";
import { awaitAllCallbacks } from "@langchain/core/callbacks/promises";
import { RunnableLambda } from "@langchain/core/runnables";
import { FITracer } from "@traceai/fi-core";
import { LangChainTracer } from "../tracer";

const PAYLOAD = "synthetic-private-approval-question";

// Same shape as @langchain/langgraph's errors: control-flow signals expose a
// prototype `is_bubble_up` getter and set `name` in the constructor.
class GraphBubbleUp extends Error {
get is_bubble_up() {
return true;
}
}
class GraphInterrupt extends GraphBubbleUp {
constructor(public interrupts: unknown[]) {
super(JSON.stringify(interrupts, null, 2));
this.name = "GraphInterrupt";
}
}
class ParentCommand extends GraphBubbleUp {
constructor() {
super("Command(graph=PARENT)");
this.name = "ParentCommand";
}
}

describe("LangGraph control-flow bubble-ups", () => {
const exporter = new InMemorySpanExporter();
const provider = new BasicTracerProvider({
spanProcessors: [new SimpleSpanProcessor(exporter)],
});
beforeEach(() => exporter.reset());

const langChainTracer = () =>
new LangChainTracer(new FITracer({ tracer: provider.getTracer("test") }));

async function spanFor(error: Error, runName: string): Promise<ReadableSpan> {
const node = RunnableLambda.from(async (_input: string) => {
throw error;
}).withConfig({ runName });
await expect(node.invoke("x", { callbacks: [langChainTracer()] })).rejects.toBe(error);
await awaitAllCallbacks();
const spans = exporter.getFinishedSpans().filter((span) => span.name === runName);
expect(spans).toHaveLength(1);
return spans[0];
}

const exceptionEvents = (span: ReadableSpan) =>
span.events.filter((event) => event.name === "exception");

it("an interrupt() pause is OK, marked as an interrupt, and does not leak the payload", async () => {
const span = await spanFor(
new GraphInterrupt([{ id: "synthetic-id", value: PAYLOAD }]),
"ask_human",
);
expect(span.status).toEqual({ code: SpanStatusCode.OK });
expect(exceptionEvents(span)).toEqual([]);
expect(span.attributes["langgraph.interrupt"]).toBe(true);
expect(span.events.map((event) => event.name)).toContain("interrupt");
expect(span.attributes["error.type"]).toBeUndefined();
expect(
JSON.stringify({ status: span.status, attributes: span.attributes, events: span.events }),
).not.toContain(PAYLOAD);
});

it("a Command.PARENT handoff is OK and not marked as an interrupt", async () => {
const span = await spanFor(new ParentCommand(), "route");
expect(span.status).toEqual({ code: SpanStatusCode.OK });
expect(exceptionEvents(span)).toEqual([]);
expect(span.attributes["langgraph.interrupt"]).toBeUndefined();
expect(span.events).toEqual([]);
});

it("recognises a bubble-up from run.error when the error object was not seen", async () => {
const tracer = langChainTracer();
const runId = "00000000-0000-4000-8000-000000000101";
await tracer.handleChainStart(
{ lc: 1, type: "not_implemented", id: ["ask_human"] },
{ input: "x" },
runId,
);
const error = new GraphInterrupt([{ id: "synthetic-id", value: PAYLOAD }]);
// Bypass the override, as an older BaseTracer path would: only run.error is available
await BaseTracer.prototype.handleChainError.call(tracer, error, runId);
const [span] = exporter.getFinishedSpans();
expect(span.status).toEqual({ code: SpanStatusCode.OK });
expect(exceptionEvents(span)).toEqual([]);
expect(span.attributes["langgraph.interrupt"]).toBe(true);
});

it("a real error is still an ERROR span with an exception", async () => {
const span = await spanFor(new TypeError("synthetic node failure"), "broken");
expect(span.status).toEqual({
code: SpanStatusCode.ERROR,
message: "synthetic node failure",
});
expect(exceptionEvents(span)).toHaveLength(1);
expect(span.attributes["langgraph.interrupt"]).toBeUndefined();
});

it("an error that only borrows a falsy is_bubble_up is still an error", async () => {
const error = Object.assign(new Error("synthetic failure"), { is_bubble_up: false });
const span = await spanFor(error, "not_a_bubble_up");
expect(span.status.code).toBe(SpanStatusCode.ERROR);
expect(exceptionEvents(span)).toHaveLength(1);
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import { describe, it, expect } from "@jest/globals";
import { readdirSync, readFileSync } from "node:fs";
import { builtinModules } from "node:module";
import { join } from "node:path";

// The workspace hoists every package, so an undeclared import still resolves here.
// Strict installs (Yarn PnP, pnpm with hoist=false) only expose declared packages,
// and an undeclared import makes `@traceai/langchain` fail to load there.
const packageRoot = join(__dirname, "..", "..");
const manifest = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8"));
const declared = new Set([
...Object.keys(manifest.dependencies ?? {}),
...Object.keys(manifest.peerDependencies ?? {}),
]);

function packageName(specifier: string) {
const parts = specifier.split("/");
return specifier.startsWith("@") ? parts.slice(0, 2).join("/") : parts[0];
}

describe("published dependencies", () => {
it("declares every package the shipped source imports", () => {
const sourceDir = join(packageRoot, "src");
const imported = new Set<string>();
for (const file of readdirSync(sourceDir)) {
if (!file.endsWith(".ts")) continue;
const source = readFileSync(join(sourceDir, file), "utf8");
for (const [, specifier] of source.matchAll(
/(?:\bfrom\s+|\bimport\s*\(\s*|\brequire\s*\(\s*)["']([^"']+)["']/g,
)) {
if (specifier.startsWith(".") || specifier.startsWith("node:")) continue;
if (builtinModules.includes(packageName(specifier))) continue;
imported.add(packageName(specifier));
}
}
expect(imported.size).toBeGreaterThan(0);
expect([...imported].filter((name) => !declared.has(name)).sort()).toEqual([]);
});
});
Loading
Loading