From 971977f318809c512a81e1d606e0dbfcd6148ff1 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 1 Nov 2026 15:05:27 -0800 Subject: [PATCH] fix(langchain): keep LangGraph pauses and handoffs out of error spans LangGraph's interrupt() and Command.PARENT handoffs throw GraphBubbleUp control-flow signals through handleChainError. The tracer exported those runs as ERROR spans with an exception event, and the interrupt() status message carried the interrupt payload. Treat bubble-ups as successful runs, mark interrupt() pauses with langgraph.interrupt (as the Python integration does), and keep real errors as errors. Declare @opentelemetry/core, which the tracer imports: under Yarn PnP and pnpm without hoisting the package failed to load. Pass the README's OpenAI key as apiKey, since ChatOpenAI 1.x ignores openAIApiKey. CI runs the LangGraph contract on LangGraph 0.2.74, 0.4.10, 1.0.0 and 1.6.0, and loads the packed package with strict pnpm. Co-Authored-By: Claude Opus 5.5 --- .../workflows/langchain-typescript-tests.yml | 51 ++++++++ .../packages/traceai_langchain/README.md | 3 +- .../traceai_langchain/contract/langgraph.mjs | 113 +++++++++++++++++ .../traceai_langchain/contract/readme.mjs | 2 + .../packages/traceai_langchain/package.json | 1 + .../src/__tests__/bubbleUp.test.ts | 117 ++++++++++++++++++ .../src/__tests__/dependencies.test.ts | 39 ++++++ .../packages/traceai_langchain/src/tracer.ts | 54 +++++++- typescript/pnpm-lock.yaml | 3 + 9 files changed, 381 insertions(+), 2 deletions(-) create mode 100644 typescript/packages/traceai_langchain/contract/langgraph.mjs create mode 100644 typescript/packages/traceai_langchain/src/__tests__/bubbleUp.test.ts create mode 100644 typescript/packages/traceai_langchain/src/__tests__/dependencies.test.ts diff --git a/.github/workflows/langchain-typescript-tests.yml b/.github/workflows/langchain-typescript-tests.yml index a3f9e6906..3dd54039e 100644 --- a/.github/workflows/langchain-typescript-tests.yml +++ b/.github/workflows/langchain-typescript-tests.yml @@ -109,3 +109,54 @@ jobs: "$(command -v node)" "$RUNNER_TEMP/langchain-consumer/lifecycle.mjs" "$mode" "$privacy" done done + - name: LangGraph pause and handoff spans, localhost only + if: matrix.core == '0.3.80' || matrix.core == '1.3.2' + env: + CORE_VERSION: ${{ matrix.core }} + run: | + # The last 0.2 and 0.x releases on core 0.3, the first 1.x and the latest on core 1.x. + case "$CORE_VERSION" in + 0.3.80) langgraph_versions='0.2.74 0.4.10' ;; + 1.3.2) langgraph_versions='1.0.0 1.6.0' ;; + esac + for langgraph in $langgraph_versions; do + consumer="$RUNNER_TEMP/langgraph-$langgraph" + mkdir -p "$consumer" + npm install --prefix "$consumer" --ignore-scripts --no-audit --no-fund \ + "$RUNNER_TEMP/langchain-tarballs/"*.tgz "@langchain/core@$CORE_VERSION" \ + "@langchain/langgraph@$langgraph" @opentelemetry/sdk-trace-base@2.0.1 + cp typescript/packages/traceai_langchain/contract/langgraph.mjs "$consumer/" + for mode in manual-cjs manual-esm; do + sudo unshare --net bash -c 'ip link set lo up; exec "$@"' -- \ + "$(command -v node)" "$consumer/langgraph.mjs" "$mode" + done + done + - name: Load the packed package with strict pnpm (no hoisting) + if: matrix.core == '1.3.2' + working-directory: typescript + env: + CORE_VERSION: ${{ matrix.core }} + run: | + # Only declared dependencies are resolvable, as with Yarn PnP. + consumer="$RUNNER_TEMP/langchain-strict" + mkdir -p "$consumer" + node -e ' + const [consumer, tarballs, core] = process.argv.slice(1); + const fs = require("node:fs"); + const tarball = (prefix) => "file:" + tarballs + "/" + + fs.readdirSync(tarballs).find((name) => name.startsWith(prefix) && /^[0-9]/.test(name.slice(prefix.length))); + const local = { + "@traceai/fi-core": tarball("traceai-fi-core-"), + "@traceai/fi-semantic-conventions": tarball("traceai-fi-semantic-conventions-"), + }; + fs.writeFileSync(consumer + "/package.json", JSON.stringify({ + private: true, + dependencies: { ...local, "@traceai/langchain": tarball("traceai-langchain-"), "@langchain/core": core }, + pnpm: { overrides: local }, + })); + ' "$consumer" "$RUNNER_TEMP/langchain-tarballs" "$CORE_VERSION" + printf 'hoist=false\n' > "$consumer/.npmrc" + corepack pnpm --dir "$consumer" install --ignore-scripts + cd "$consumer" + node -e 'require("@traceai/langchain").LangChainInstrumentation.name' + node --input-type=module -e 'await import("@traceai/langchain")' diff --git a/typescript/packages/traceai_langchain/README.md b/typescript/packages/traceai_langchain/README.md index d0031d441..4fffae3dc 100644 --- a/typescript/packages/traceai_langchain/README.md +++ b/typescript/packages/traceai_langchain/README.md @@ -60,7 +60,7 @@ lcInstrumentation.manuallyInstrument(CallbackManagerModule); // 4. Use LangChain as normal const chatModel = new ChatOpenAI({ - openAIApiKey: process.env.OPENAI_API_KEY, + apiKey: process.env.OPENAI_API_KEY, metadata: { session_id: "your-session-id", }, @@ -93,6 +93,7 @@ the provider. Before an explicit `forceFlush()`, drain callbacks in the same ord ## Features - Automatic tracing of LangChain operations +- LangGraph graphs: an `interrupt()` pause or a `Command` handoff is a successful span, not an error; a paused node is marked `langgraph.interrupt` - Support for both ESM and CommonJS modules - Compatible with `@langchain/core` 0.2.x, 0.3.x, and 1.x - Integration with TraceAI's observability platform diff --git a/typescript/packages/traceai_langchain/contract/langgraph.mjs b/typescript/packages/traceai_langchain/contract/langgraph.mjs new file mode 100644 index 000000000..9b4c9e416 --- /dev/null +++ b/typescript/packages/traceai_langchain/contract/langgraph.mjs @@ -0,0 +1,113 @@ +// Copy this fixture beside an isolated package.json after installing the built +// tarballs with @langchain/core and a compatible @langchain/langgraph. No model +// or vendor calls. Usage: node langgraph.mjs manual-cjs|manual-esm +// +// LangGraph pauses (interrupt()) and hands off (Command.PARENT) by throwing +// control-flow signals through the chain callbacks. Those runs succeeded, so their +// spans must not be ERROR, carry exception events or leak the interrupt payload. +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; + +const require = createRequire(import.meta.url); +const mode = process.argv[2] ?? "manual-cjs"; +assert.ok(["manual-cjs", "manual-esm"].includes(mode)); +const load = mode === "manual-esm" ? (name) => import(name) : async (name) => require(name); +const { LangChainInstrumentation } = await load("@traceai/langchain"); +const { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } = + await load("@opentelemetry/sdk-trace-base"); +const { SpanStatusCode } = await load("@opentelemetry/api"); + +const exporter = new InMemorySpanExporter(); +const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); +new LangChainInstrumentation({ tracerProvider: provider }) + .manuallyInstrument(await load("@langchain/core/callbacks/manager")); +const { StateGraph, Annotation, START, END, MemorySaver, interrupt, Command } = + await load("@langchain/langgraph"); +const { awaitAllCallbacks } = await load("@langchain/core/callbacks/promises"); +const versions = { + core: require("@langchain/core/package.json").version, + langgraph: require("@langchain/langgraph/package.json").version, +}; + +const State = Annotation.Root({ n: Annotation({ reducer: (_, next) => next, default: () => 0 }) }); +const payload = "synthetic-private-approval-question"; +const exceptions = (span) => span.events.filter((event) => event.name === "exception"); +const isInterruptMarked = (span) => span.attributes["langgraph.interrupt"] === true; +async function finished() { + await awaitAllCallbacks(); + const spans = exporter.getFinishedSpans(); + exporter.reset(); + return spans; +} +function assertSucceeded(spans, label) { + assert.ok(spans.length > 0, `${label}: no spans exported`); + for (const span of spans) { + assert.notEqual(span.status.code, SpanStatusCode.ERROR, + `${label}: ${span.name} exported as ERROR (${span.status.message})`); + assert.deepEqual(exceptions(span), [], `${label}: ${span.name} recorded an exception`); + assert.ok(!JSON.stringify(span.status).includes(payload), `${label}: status leaks the interrupt payload`); + } +} + +try { + // 1. Human-in-the-loop pause, then resume. + const hitl = new StateGraph(State) + .addNode("ask_human", async (state) => ({ n: state.n + (interrupt(payload) ? 1 : 0) })) + .addEdge(START, "ask_human") + .addEdge("ask_human", END) + .compile({ checkpointer: new MemorySaver() }); + const thread = { configurable: { thread_id: "synthetic-thread" } }; + await hitl.invoke({ n: 0 }, thread); + const pending = await hitl.getState(thread); + assert.ok(pending.tasks.some((task) => task.interrupts?.length), "graph must pause on the interrupt"); + let spans = await finished(); + assertSucceeded(spans, "interrupt"); + const pausedNode = spans.filter((span) => span.name === "ask_human"); + assert.equal(pausedNode.length, 1, "one paused node span"); + assert.ok(isInterruptMarked(pausedNode[0]), "the paused node is marked langgraph.interrupt"); + assert.ok(pausedNode[0].events.some((event) => event.name === "interrupt"), "the paused node has an interrupt event"); + assert.deepEqual(spans.filter(isInterruptMarked).map((span) => span.name), ["ask_human"]); + + assert.deepEqual(await hitl.invoke(new Command({ resume: true }), thread), { n: 1 }); + spans = await finished(); + assertSucceeded(spans, "resume"); + assert.deepEqual(spans.filter(isInterruptMarked), [], "a resumed run is not an interrupt"); + + // 2. Subgraph node hands off to its parent graph. + const subgraph = new StateGraph(State) + .addNode("route", async () => new Command({ graph: Command.PARENT, goto: "done", update: { n: 5 } })) + .addEdge(START, "route") + .compile(); + const parent = new StateGraph(State) + .addNode("sub", subgraph, { ends: ["done"] }) + .addNode("done", async (state) => ({ n: state.n + 1 })) + .addEdge(START, "sub") + .addEdge("done", END) + .compile(); + assert.deepEqual(await parent.invoke({ n: 0 }), { n: 6 }); + spans = await finished(); + assertSucceeded(spans, "handoff"); + for (const name of ["route", "sub", "done"]) { + assert.ok(spans.some((span) => span.name === name), `handoff: missing ${name} span`); + } + assert.deepEqual(spans.filter(isInterruptMarked), [], "a handoff is not an interrupt"); + + // 3. Negative control: a real node failure is still an error. + const failure = new Error("synthetic node failure"); + const failing = new StateGraph(State) + .addNode("broken", async () => { throw failure; }) + .addEdge(START, "broken") + .addEdge("broken", END) + .compile(); + await assert.rejects(failing.invoke({ n: 0 }), (error) => error.message === failure.message); + spans = await finished(); + const broken = spans.filter((span) => span.name === "broken"); + assert.equal(broken.length, 1); + assert.equal(broken[0].status.code, SpanStatusCode.ERROR); + assert.equal(exceptions(broken[0]).length, 1); + assert.ok(!isInterruptMarked(broken[0])); + + console.log(`PASS: LangGraph control flow (${mode}, core ${versions.core}, langgraph ${versions.langgraph})`); +} finally { + await provider.shutdown(); +} diff --git a/typescript/packages/traceai_langchain/contract/readme.mjs b/typescript/packages/traceai_langchain/contract/readme.mjs index ac0f7a39a..1dd7a77b3 100644 --- a/typescript/packages/traceai_langchain/contract/readme.mjs +++ b/typescript/packages/traceai_langchain/contract/readme.mjs @@ -23,6 +23,8 @@ for (const [, specifier] of source.matchAll(/from "([^"]+)"/g)) { `README must explicitly install ${packageName(specifier)}`); } assert.ok(dependencies.some((dep) => packageName(dep) === "tsx"), "README must install its TypeScript runner"); +// ChatOpenAI 1.x ignores the old openAIApiKey alias; only apiKey reaches the client. +assert.doesNotMatch(source, /\bopenAIApiKey\b/, "README must pass the OpenAI key as apiKey"); for (const name of ["FI_API_KEY", "FI_SECRET_KEY", "OPENAI_API_KEY"]) { assert.match(setup, new RegExp(`^export ${name}=`, "m"), `README must export ${name}`); } diff --git a/typescript/packages/traceai_langchain/package.json b/typescript/packages/traceai_langchain/package.json index 616bc54fd..c6ff71eed 100644 --- a/typescript/packages/traceai_langchain/package.json +++ b/typescript/packages/traceai_langchain/package.json @@ -43,6 +43,7 @@ "homepage": "https://docs.futureagi.com/docs/integrations/traceai/langchain", "dependencies": { "@opentelemetry/api": "^1.9.0", + "@opentelemetry/core": "^2.0.1", "@opentelemetry/instrumentation": "^0.202.0", "@traceai/fi-core": "workspace:^", "@traceai/fi-semantic-conventions": "workspace:^" diff --git a/typescript/packages/traceai_langchain/src/__tests__/bubbleUp.test.ts b/typescript/packages/traceai_langchain/src/__tests__/bubbleUp.test.ts new file mode 100644 index 000000000..5ba4a3809 --- /dev/null +++ b/typescript/packages/traceai_langchain/src/__tests__/bubbleUp.test.ts @@ -0,0 +1,117 @@ +import { describe, it, expect, beforeEach } from "@jest/globals"; +import { SpanStatusCode } from "@opentelemetry/api"; +import { + BasicTracerProvider, + InMemorySpanExporter, + ReadableSpan, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; +import { BaseTracer } from "@langchain/core/tracers/base"; +import { awaitAllCallbacks } from "@langchain/core/callbacks/promises"; +import { RunnableLambda } from "@langchain/core/runnables"; +import { FITracer } from "@traceai/fi-core"; +import { LangChainTracer } from "../tracer"; + +const PAYLOAD = "synthetic-private-approval-question"; + +// Same shape as @langchain/langgraph's errors: control-flow signals expose a +// prototype `is_bubble_up` getter and set `name` in the constructor. +class GraphBubbleUp extends Error { + get is_bubble_up() { + return true; + } +} +class GraphInterrupt extends GraphBubbleUp { + constructor(public interrupts: unknown[]) { + super(JSON.stringify(interrupts, null, 2)); + this.name = "GraphInterrupt"; + } +} +class ParentCommand extends GraphBubbleUp { + constructor() { + super("Command(graph=PARENT)"); + this.name = "ParentCommand"; + } +} + +describe("LangGraph control-flow bubble-ups", () => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ + spanProcessors: [new SimpleSpanProcessor(exporter)], + }); + beforeEach(() => exporter.reset()); + + const langChainTracer = () => + new LangChainTracer(new FITracer({ tracer: provider.getTracer("test") })); + + async function spanFor(error: Error, runName: string): Promise { + const node = RunnableLambda.from(async (_input: string) => { + throw error; + }).withConfig({ runName }); + await expect(node.invoke("x", { callbacks: [langChainTracer()] })).rejects.toBe(error); + await awaitAllCallbacks(); + const spans = exporter.getFinishedSpans().filter((span) => span.name === runName); + expect(spans).toHaveLength(1); + return spans[0]; + } + + const exceptionEvents = (span: ReadableSpan) => + span.events.filter((event) => event.name === "exception"); + + it("an interrupt() pause is OK, marked as an interrupt, and does not leak the payload", async () => { + const span = await spanFor( + new GraphInterrupt([{ id: "synthetic-id", value: PAYLOAD }]), + "ask_human", + ); + expect(span.status).toEqual({ code: SpanStatusCode.OK }); + expect(exceptionEvents(span)).toEqual([]); + expect(span.attributes["langgraph.interrupt"]).toBe(true); + expect(span.events.map((event) => event.name)).toContain("interrupt"); + expect(span.attributes["error.type"]).toBeUndefined(); + expect( + JSON.stringify({ status: span.status, attributes: span.attributes, events: span.events }), + ).not.toContain(PAYLOAD); + }); + + it("a Command.PARENT handoff is OK and not marked as an interrupt", async () => { + const span = await spanFor(new ParentCommand(), "route"); + expect(span.status).toEqual({ code: SpanStatusCode.OK }); + expect(exceptionEvents(span)).toEqual([]); + expect(span.attributes["langgraph.interrupt"]).toBeUndefined(); + expect(span.events).toEqual([]); + }); + + it("recognises a bubble-up from run.error when the error object was not seen", async () => { + const tracer = langChainTracer(); + const runId = "00000000-0000-4000-8000-000000000101"; + await tracer.handleChainStart( + { lc: 1, type: "not_implemented", id: ["ask_human"] }, + { input: "x" }, + runId, + ); + const error = new GraphInterrupt([{ id: "synthetic-id", value: PAYLOAD }]); + // Bypass the override, as an older BaseTracer path would: only run.error is available + await BaseTracer.prototype.handleChainError.call(tracer, error, runId); + const [span] = exporter.getFinishedSpans(); + expect(span.status).toEqual({ code: SpanStatusCode.OK }); + expect(exceptionEvents(span)).toEqual([]); + expect(span.attributes["langgraph.interrupt"]).toBe(true); + }); + + it("a real error is still an ERROR span with an exception", async () => { + const span = await spanFor(new TypeError("synthetic node failure"), "broken"); + expect(span.status).toEqual({ + code: SpanStatusCode.ERROR, + message: "synthetic node failure", + }); + expect(exceptionEvents(span)).toHaveLength(1); + expect(span.attributes["langgraph.interrupt"]).toBeUndefined(); + }); + + it("an error that only borrows a falsy is_bubble_up is still an error", async () => { + const error = Object.assign(new Error("synthetic failure"), { is_bubble_up: false }); + const span = await spanFor(error, "not_a_bubble_up"); + expect(span.status.code).toBe(SpanStatusCode.ERROR); + expect(exceptionEvents(span)).toHaveLength(1); + }); +}); diff --git a/typescript/packages/traceai_langchain/src/__tests__/dependencies.test.ts b/typescript/packages/traceai_langchain/src/__tests__/dependencies.test.ts new file mode 100644 index 000000000..908094538 --- /dev/null +++ b/typescript/packages/traceai_langchain/src/__tests__/dependencies.test.ts @@ -0,0 +1,39 @@ +import { describe, it, expect } from "@jest/globals"; +import { readdirSync, readFileSync } from "node:fs"; +import { builtinModules } from "node:module"; +import { join } from "node:path"; + +// The workspace hoists every package, so an undeclared import still resolves here. +// Strict installs (Yarn PnP, pnpm with hoist=false) only expose declared packages, +// and an undeclared import makes `@traceai/langchain` fail to load there. +const packageRoot = join(__dirname, "..", ".."); +const manifest = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8")); +const declared = new Set([ + ...Object.keys(manifest.dependencies ?? {}), + ...Object.keys(manifest.peerDependencies ?? {}), +]); + +function packageName(specifier: string) { + const parts = specifier.split("/"); + return specifier.startsWith("@") ? parts.slice(0, 2).join("/") : parts[0]; +} + +describe("published dependencies", () => { + it("declares every package the shipped source imports", () => { + const sourceDir = join(packageRoot, "src"); + const imported = new Set(); + for (const file of readdirSync(sourceDir)) { + if (!file.endsWith(".ts")) continue; + const source = readFileSync(join(sourceDir, file), "utf8"); + for (const [, specifier] of source.matchAll( + /(?:\bfrom\s+|\bimport\s*\(\s*|\brequire\s*\(\s*)["']([^"']+)["']/g, + )) { + if (specifier.startsWith(".") || specifier.startsWith("node:")) continue; + if (builtinModules.includes(packageName(specifier))) continue; + imported.add(packageName(specifier)); + } + } + expect(imported.size).toBeGreaterThan(0); + expect([...imported].filter((name) => !declared.has(name)).sort()).toEqual([]); + }); +}); diff --git a/typescript/packages/traceai_langchain/src/tracer.ts b/typescript/packages/traceai_langchain/src/tracer.ts index f0e9921a3..b28bf1820 100644 --- a/typescript/packages/traceai_langchain/src/tracer.ts +++ b/typescript/packages/traceai_langchain/src/tracer.ts @@ -79,6 +79,46 @@ function recordRunError(span: Span, error: unknown, runError: string) { span.setStatus({ code: SpanStatusCode.ERROR, message }); } +/** LangGraph control-flow signals, for errors that do not expose `is_bubble_up` */ +const GRAPH_BUBBLE_UP_NAMES = new Set([ + "GraphBubbleUp", + "GraphInterrupt", + "NodeInterrupt", + "ParentCommand", + "GraphDrained", +]); +/** The bubble-ups that pause a graph for human input (`interrupt()`, `NodeInterrupt`) */ +const GRAPH_INTERRUPT_NAMES = new Set(["GraphInterrupt", "NodeInterrupt"]); + +/** + * The name of a LangGraph control-flow signal, or undefined for a real error. + * LangGraph pauses (`interrupt()`) and hands off (`Command.PARENT`) by throwing a + * `GraphBubbleUp` through the chain callbacks; the run itself did not fail. + * Matches LangGraph's own `isGraphBubbleUp` without importing LangGraph. + */ +function getGraphBubbleUpName( + error: unknown, + runError: string, +): string | undefined { + if (error != null && typeof error === "object") { + const { name, is_bubble_up } = error as { + name?: unknown; + is_bubble_up?: unknown; + }; + if (is_bubble_up === true) { + return typeof name === "string" ? name : "GraphBubbleUp"; + } + return typeof name === "string" && GRAPH_BUBBLE_UP_NAMES.has(name) + ? name + : undefined; + } + if (error != null) { + return undefined; + } + const { name } = parseRunError(runError); + return name != null && GRAPH_BUBBLE_UP_NAMES.has(name) ? name : undefined; +} + export class LangChainTracer extends BaseTracer { private tracer: FITracer; private runs: Record = {}; @@ -218,7 +258,19 @@ export class LangChainTracer extends BaseTracer { return; } const { span } = runWithSpan; - if (run.error != null) { + const bubbleUp = + run.error != null + ? getGraphBubbleUpName(this.runErrors[run.id], run.error) + : undefined; + if (bubbleUp != null) { + // A pause or handoff, not a failure: no exception, and no status message that + // could carry the interrupt payload. + if (GRAPH_INTERRUPT_NAMES.has(bubbleUp)) { + span.setAttribute("langgraph.interrupt", true); + span.addEvent("interrupt", { "langgraph.interrupt": true }); + } + span.setStatus({ code: SpanStatusCode.OK }); + } else if (run.error != null) { recordRunError(span, this.runErrors[run.id], run.error); } else { span.setStatus({ code: SpanStatusCode.OK }); diff --git a/typescript/pnpm-lock.yaml b/typescript/pnpm-lock.yaml index 3eae5818d..1ad32abdd 100644 --- a/typescript/pnpm-lock.yaml +++ b/typescript/pnpm-lock.yaml @@ -1190,6 +1190,9 @@ importers: '@opentelemetry/api': specifier: ^1.9.0 version: 1.9.0 + '@opentelemetry/core': + specifier: ^2.0.1 + version: 2.0.1(@opentelemetry/api@1.9.0) '@opentelemetry/instrumentation': specifier: ^0.202.0 version: 0.202.0(@opentelemetry/api@1.9.0)