diff --git a/.github/workflows/langchain-typescript-tests.yml b/.github/workflows/langchain-typescript-tests.yml index a3f9e6906..3dd54039e 100644 --- a/.github/workflows/langchain-typescript-tests.yml +++ b/.github/workflows/langchain-typescript-tests.yml @@ -109,3 +109,54 @@ jobs: "$(command -v node)" "$RUNNER_TEMP/langchain-consumer/lifecycle.mjs" "$mode" "$privacy" done done + - name: LangGraph pause and handoff spans, localhost only + if: matrix.core == '0.3.80' || matrix.core == '1.3.2' + env: + CORE_VERSION: ${{ matrix.core }} + run: | + # The last 0.2 and 0.x releases on core 0.3, the first 1.x and the latest on core 1.x. + case "$CORE_VERSION" in + 0.3.80) langgraph_versions='0.2.74 0.4.10' ;; + 1.3.2) langgraph_versions='1.0.0 1.6.0' ;; + esac + for langgraph in $langgraph_versions; do + consumer="$RUNNER_TEMP/langgraph-$langgraph" + mkdir -p "$consumer" + npm install --prefix "$consumer" --ignore-scripts --no-audit --no-fund \ + "$RUNNER_TEMP/langchain-tarballs/"*.tgz "@langchain/core@$CORE_VERSION" \ + "@langchain/langgraph@$langgraph" @opentelemetry/sdk-trace-base@2.0.1 + cp typescript/packages/traceai_langchain/contract/langgraph.mjs "$consumer/" + for mode in manual-cjs manual-esm; do + sudo unshare --net bash -c 'ip link set lo up; exec "$@"' -- \ + "$(command -v node)" "$consumer/langgraph.mjs" "$mode" + done + done + - name: Load the packed package with strict pnpm (no hoisting) + if: matrix.core == '1.3.2' + working-directory: typescript + env: + CORE_VERSION: ${{ matrix.core }} + run: | + # Only declared dependencies are resolvable, as with Yarn PnP. + consumer="$RUNNER_TEMP/langchain-strict" + mkdir -p "$consumer" + node -e ' + const [consumer, tarballs, core] = process.argv.slice(1); + const fs = require("node:fs"); + const tarball = (prefix) => "file:" + tarballs + "/" + + fs.readdirSync(tarballs).find((name) => name.startsWith(prefix) && /^[0-9]/.test(name.slice(prefix.length))); + const local = { + "@traceai/fi-core": tarball("traceai-fi-core-"), + "@traceai/fi-semantic-conventions": tarball("traceai-fi-semantic-conventions-"), + }; + fs.writeFileSync(consumer + "/package.json", JSON.stringify({ + private: true, + dependencies: { ...local, "@traceai/langchain": tarball("traceai-langchain-"), "@langchain/core": core }, + pnpm: { overrides: local }, + })); + ' "$consumer" "$RUNNER_TEMP/langchain-tarballs" "$CORE_VERSION" + printf 'hoist=false\n' > "$consumer/.npmrc" + corepack pnpm --dir "$consumer" install --ignore-scripts + cd "$consumer" + node -e 'require("@traceai/langchain").LangChainInstrumentation.name' + node --input-type=module -e 'await import("@traceai/langchain")' diff --git a/typescript/packages/traceai_langchain/README.md b/typescript/packages/traceai_langchain/README.md index d0031d441..4fffae3dc 100644 --- a/typescript/packages/traceai_langchain/README.md +++ b/typescript/packages/traceai_langchain/README.md @@ -60,7 +60,7 @@ lcInstrumentation.manuallyInstrument(CallbackManagerModule); // 4. Use LangChain as normal const chatModel = new ChatOpenAI({ - openAIApiKey: process.env.OPENAI_API_KEY, + apiKey: process.env.OPENAI_API_KEY, metadata: { session_id: "your-session-id", }, @@ -93,6 +93,7 @@ the provider. Before an explicit `forceFlush()`, drain callbacks in the same ord ## Features - Automatic tracing of LangChain operations +- LangGraph graphs: an `interrupt()` pause or a `Command` handoff is a successful span, not an error; a paused node is marked `langgraph.interrupt` - Support for both ESM and CommonJS modules - Compatible with `@langchain/core` 0.2.x, 0.3.x, and 1.x - Integration with TraceAI's observability platform diff --git a/typescript/packages/traceai_langchain/contract/langgraph.mjs b/typescript/packages/traceai_langchain/contract/langgraph.mjs new file mode 100644 index 000000000..9b4c9e416 --- /dev/null +++ b/typescript/packages/traceai_langchain/contract/langgraph.mjs @@ -0,0 +1,113 @@ +// Copy this fixture beside an isolated package.json after installing the built +// tarballs with @langchain/core and a compatible @langchain/langgraph. No model +// or vendor calls. Usage: node langgraph.mjs manual-cjs|manual-esm +// +// LangGraph pauses (interrupt()) and hands off (Command.PARENT) by throwing +// control-flow signals through the chain callbacks. Those runs succeeded, so their +// spans must not be ERROR, carry exception events or leak the interrupt payload. +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; + +const require = createRequire(import.meta.url); +const mode = process.argv[2] ?? "manual-cjs"; +assert.ok(["manual-cjs", "manual-esm"].includes(mode)); +const load = mode === "manual-esm" ? (name) => import(name) : async (name) => require(name); +const { LangChainInstrumentation } = await load("@traceai/langchain"); +const { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } = + await load("@opentelemetry/sdk-trace-base"); +const { SpanStatusCode } = await load("@opentelemetry/api"); + +const exporter = new InMemorySpanExporter(); +const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); +new LangChainInstrumentation({ tracerProvider: provider }) + .manuallyInstrument(await load("@langchain/core/callbacks/manager")); +const { StateGraph, Annotation, START, END, MemorySaver, interrupt, Command } = + await load("@langchain/langgraph"); +const { awaitAllCallbacks } = await load("@langchain/core/callbacks/promises"); +const versions = { + core: require("@langchain/core/package.json").version, + langgraph: require("@langchain/langgraph/package.json").version, +}; + +const State = Annotation.Root({ n: Annotation({ reducer: (_, next) => next, default: () => 0 }) }); +const payload = "synthetic-private-approval-question"; +const exceptions = (span) => span.events.filter((event) => event.name === "exception"); +const isInterruptMarked = (span) => span.attributes["langgraph.interrupt"] === true; +async function finished() { + await awaitAllCallbacks(); + const spans = exporter.getFinishedSpans(); + exporter.reset(); + return spans; +} +function assertSucceeded(spans, label) { + assert.ok(spans.length > 0, `${label}: no spans exported`); + for (const span of spans) { + assert.notEqual(span.status.code, SpanStatusCode.ERROR, + `${label}: ${span.name} exported as ERROR (${span.status.message})`); + assert.deepEqual(exceptions(span), [], `${label}: ${span.name} recorded an exception`); + assert.ok(!JSON.stringify(span.status).includes(payload), `${label}: status leaks the interrupt payload`); + } +} + +try { + // 1. Human-in-the-loop pause, then resume. + const hitl = new StateGraph(State) + .addNode("ask_human", async (state) => ({ n: state.n + (interrupt(payload) ? 1 : 0) })) + .addEdge(START, "ask_human") + .addEdge("ask_human", END) + .compile({ checkpointer: new MemorySaver() }); + const thread = { configurable: { thread_id: "synthetic-thread" } }; + await hitl.invoke({ n: 0 }, thread); + const pending = await hitl.getState(thread); + assert.ok(pending.tasks.some((task) => task.interrupts?.length), "graph must pause on the interrupt"); + let spans = await finished(); + assertSucceeded(spans, "interrupt"); + const pausedNode = spans.filter((span) => span.name === "ask_human"); + assert.equal(pausedNode.length, 1, "one paused node span"); + assert.ok(isInterruptMarked(pausedNode[0]), "the paused node is marked langgraph.interrupt"); + assert.ok(pausedNode[0].events.some((event) => event.name === "interrupt"), "the paused node has an interrupt event"); + assert.deepEqual(spans.filter(isInterruptMarked).map((span) => span.name), ["ask_human"]); + + assert.deepEqual(await hitl.invoke(new Command({ resume: true }), thread), { n: 1 }); + spans = await finished(); + assertSucceeded(spans, "resume"); + assert.deepEqual(spans.filter(isInterruptMarked), [], "a resumed run is not an interrupt"); + + // 2. Subgraph node hands off to its parent graph. + const subgraph = new StateGraph(State) + .addNode("route", async () => new Command({ graph: Command.PARENT, goto: "done", update: { n: 5 } })) + .addEdge(START, "route") + .compile(); + const parent = new StateGraph(State) + .addNode("sub", subgraph, { ends: ["done"] }) + .addNode("done", async (state) => ({ n: state.n + 1 })) + .addEdge(START, "sub") + .addEdge("done", END) + .compile(); + assert.deepEqual(await parent.invoke({ n: 0 }), { n: 6 }); + spans = await finished(); + assertSucceeded(spans, "handoff"); + for (const name of ["route", "sub", "done"]) { + assert.ok(spans.some((span) => span.name === name), `handoff: missing ${name} span`); + } + assert.deepEqual(spans.filter(isInterruptMarked), [], "a handoff is not an interrupt"); + + // 3. Negative control: a real node failure is still an error. + const failure = new Error("synthetic node failure"); + const failing = new StateGraph(State) + .addNode("broken", async () => { throw failure; }) + .addEdge(START, "broken") + .addEdge("broken", END) + .compile(); + await assert.rejects(failing.invoke({ n: 0 }), (error) => error.message === failure.message); + spans = await finished(); + const broken = spans.filter((span) => span.name === "broken"); + assert.equal(broken.length, 1); + assert.equal(broken[0].status.code, SpanStatusCode.ERROR); + assert.equal(exceptions(broken[0]).length, 1); + assert.ok(!isInterruptMarked(broken[0])); + + console.log(`PASS: LangGraph control flow (${mode}, core ${versions.core}, langgraph ${versions.langgraph})`); +} finally { + await provider.shutdown(); +} diff --git a/typescript/packages/traceai_langchain/contract/readme.mjs b/typescript/packages/traceai_langchain/contract/readme.mjs index ac0f7a39a..1dd7a77b3 100644 --- a/typescript/packages/traceai_langchain/contract/readme.mjs +++ b/typescript/packages/traceai_langchain/contract/readme.mjs @@ -23,6 +23,8 @@ for (const [, specifier] of source.matchAll(/from "([^"]+)"/g)) { `README must explicitly install ${packageName(specifier)}`); } assert.ok(dependencies.some((dep) => packageName(dep) === "tsx"), "README must install its TypeScript runner"); +// ChatOpenAI 1.x ignores the old openAIApiKey alias; only apiKey reaches the client. +assert.doesNotMatch(source, /\bopenAIApiKey\b/, "README must pass the OpenAI key as apiKey"); for (const name of ["FI_API_KEY", "FI_SECRET_KEY", "OPENAI_API_KEY"]) { assert.match(setup, new RegExp(`^export ${name}=`, "m"), `README must export ${name}`); } diff --git a/typescript/packages/traceai_langchain/package.json b/typescript/packages/traceai_langchain/package.json index 616bc54fd..c6ff71eed 100644 --- a/typescript/packages/traceai_langchain/package.json +++ b/typescript/packages/traceai_langchain/package.json @@ -43,6 +43,7 @@ "homepage": "https://docs.futureagi.com/docs/integrations/traceai/langchain", "dependencies": { "@opentelemetry/api": "^1.9.0", + "@opentelemetry/core": "^2.0.1", "@opentelemetry/instrumentation": "^0.202.0", "@traceai/fi-core": "workspace:^", "@traceai/fi-semantic-conventions": "workspace:^" diff --git a/typescript/packages/traceai_langchain/src/__tests__/bubbleUp.test.ts b/typescript/packages/traceai_langchain/src/__tests__/bubbleUp.test.ts new file mode 100644 index 000000000..5ba4a3809 --- /dev/null +++ b/typescript/packages/traceai_langchain/src/__tests__/bubbleUp.test.ts @@ -0,0 +1,117 @@ +import { describe, it, expect, beforeEach } from "@jest/globals"; +import { SpanStatusCode } from "@opentelemetry/api"; +import { + BasicTracerProvider, + InMemorySpanExporter, + ReadableSpan, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; +import { BaseTracer } from "@langchain/core/tracers/base"; +import { awaitAllCallbacks } from "@langchain/core/callbacks/promises"; +import { RunnableLambda } from "@langchain/core/runnables"; +import { FITracer } from "@traceai/fi-core"; +import { LangChainTracer } from "../tracer"; + +const PAYLOAD = "synthetic-private-approval-question"; + +// Same shape as @langchain/langgraph's errors: control-flow signals expose a +// prototype `is_bubble_up` getter and set `name` in the constructor. +class GraphBubbleUp extends Error { + get is_bubble_up() { + return true; + } +} +class GraphInterrupt extends GraphBubbleUp { + constructor(public interrupts: unknown[]) { + super(JSON.stringify(interrupts, null, 2)); + this.name = "GraphInterrupt"; + } +} +class ParentCommand extends GraphBubbleUp { + constructor() { + super("Command(graph=PARENT)"); + this.name = "ParentCommand"; + } +} + +describe("LangGraph control-flow bubble-ups", () => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ + spanProcessors: [new SimpleSpanProcessor(exporter)], + }); + beforeEach(() => exporter.reset()); + + const langChainTracer = () => + new LangChainTracer(new FITracer({ tracer: provider.getTracer("test") })); + + async function spanFor(error: Error, runName: string): Promise { + const node = RunnableLambda.from(async (_input: string) => { + throw error; + }).withConfig({ runName }); + await expect(node.invoke("x", { callbacks: [langChainTracer()] })).rejects.toBe(error); + await awaitAllCallbacks(); + const spans = exporter.getFinishedSpans().filter((span) => span.name === runName); + expect(spans).toHaveLength(1); + return spans[0]; + } + + const exceptionEvents = (span: ReadableSpan) => + span.events.filter((event) => event.name === "exception"); + + it("an interrupt() pause is OK, marked as an interrupt, and does not leak the payload", async () => { + const span = await spanFor( + new GraphInterrupt([{ id: "synthetic-id", value: PAYLOAD }]), + "ask_human", + ); + expect(span.status).toEqual({ code: SpanStatusCode.OK }); + expect(exceptionEvents(span)).toEqual([]); + expect(span.attributes["langgraph.interrupt"]).toBe(true); + expect(span.events.map((event) => event.name)).toContain("interrupt"); + expect(span.attributes["error.type"]).toBeUndefined(); + expect( + JSON.stringify({ status: span.status, attributes: span.attributes, events: span.events }), + ).not.toContain(PAYLOAD); + }); + + it("a Command.PARENT handoff is OK and not marked as an interrupt", async () => { + const span = await spanFor(new ParentCommand(), "route"); + expect(span.status).toEqual({ code: SpanStatusCode.OK }); + expect(exceptionEvents(span)).toEqual([]); + expect(span.attributes["langgraph.interrupt"]).toBeUndefined(); + expect(span.events).toEqual([]); + }); + + it("recognises a bubble-up from run.error when the error object was not seen", async () => { + const tracer = langChainTracer(); + const runId = "00000000-0000-4000-8000-000000000101"; + await tracer.handleChainStart( + { lc: 1, type: "not_implemented", id: ["ask_human"] }, + { input: "x" }, + runId, + ); + const error = new GraphInterrupt([{ id: "synthetic-id", value: PAYLOAD }]); + // Bypass the override, as an older BaseTracer path would: only run.error is available + await BaseTracer.prototype.handleChainError.call(tracer, error, runId); + const [span] = exporter.getFinishedSpans(); + expect(span.status).toEqual({ code: SpanStatusCode.OK }); + expect(exceptionEvents(span)).toEqual([]); + expect(span.attributes["langgraph.interrupt"]).toBe(true); + }); + + it("a real error is still an ERROR span with an exception", async () => { + const span = await spanFor(new TypeError("synthetic node failure"), "broken"); + expect(span.status).toEqual({ + code: SpanStatusCode.ERROR, + message: "synthetic node failure", + }); + expect(exceptionEvents(span)).toHaveLength(1); + expect(span.attributes["langgraph.interrupt"]).toBeUndefined(); + }); + + it("an error that only borrows a falsy is_bubble_up is still an error", async () => { + const error = Object.assign(new Error("synthetic failure"), { is_bubble_up: false }); + const span = await spanFor(error, "not_a_bubble_up"); + expect(span.status.code).toBe(SpanStatusCode.ERROR); + expect(exceptionEvents(span)).toHaveLength(1); + }); +}); diff --git a/typescript/packages/traceai_langchain/src/__tests__/dependencies.test.ts b/typescript/packages/traceai_langchain/src/__tests__/dependencies.test.ts new file mode 100644 index 000000000..908094538 --- /dev/null +++ b/typescript/packages/traceai_langchain/src/__tests__/dependencies.test.ts @@ -0,0 +1,39 @@ +import { describe, it, expect } from "@jest/globals"; +import { readdirSync, readFileSync } from "node:fs"; +import { builtinModules } from "node:module"; +import { join } from "node:path"; + +// The workspace hoists every package, so an undeclared import still resolves here. +// Strict installs (Yarn PnP, pnpm with hoist=false) only expose declared packages, +// and an undeclared import makes `@traceai/langchain` fail to load there. +const packageRoot = join(__dirname, "..", ".."); +const manifest = JSON.parse(readFileSync(join(packageRoot, "package.json"), "utf8")); +const declared = new Set([ + ...Object.keys(manifest.dependencies ?? {}), + ...Object.keys(manifest.peerDependencies ?? {}), +]); + +function packageName(specifier: string) { + const parts = specifier.split("/"); + return specifier.startsWith("@") ? parts.slice(0, 2).join("/") : parts[0]; +} + +describe("published dependencies", () => { + it("declares every package the shipped source imports", () => { + const sourceDir = join(packageRoot, "src"); + const imported = new Set(); + for (const file of readdirSync(sourceDir)) { + if (!file.endsWith(".ts")) continue; + const source = readFileSync(join(sourceDir, file), "utf8"); + for (const [, specifier] of source.matchAll( + /(?:\bfrom\s+|\bimport\s*\(\s*|\brequire\s*\(\s*)["']([^"']+)["']/g, + )) { + if (specifier.startsWith(".") || specifier.startsWith("node:")) continue; + if (builtinModules.includes(packageName(specifier))) continue; + imported.add(packageName(specifier)); + } + } + expect(imported.size).toBeGreaterThan(0); + expect([...imported].filter((name) => !declared.has(name)).sort()).toEqual([]); + }); +}); diff --git a/typescript/packages/traceai_langchain/src/tracer.ts b/typescript/packages/traceai_langchain/src/tracer.ts index f0e9921a3..b28bf1820 100644 --- a/typescript/packages/traceai_langchain/src/tracer.ts +++ b/typescript/packages/traceai_langchain/src/tracer.ts @@ -79,6 +79,46 @@ function recordRunError(span: Span, error: unknown, runError: string) { span.setStatus({ code: SpanStatusCode.ERROR, message }); } +/** LangGraph control-flow signals, for errors that do not expose `is_bubble_up` */ +const GRAPH_BUBBLE_UP_NAMES = new Set([ + "GraphBubbleUp", + "GraphInterrupt", + "NodeInterrupt", + "ParentCommand", + "GraphDrained", +]); +/** The bubble-ups that pause a graph for human input (`interrupt()`, `NodeInterrupt`) */ +const GRAPH_INTERRUPT_NAMES = new Set(["GraphInterrupt", "NodeInterrupt"]); + +/** + * The name of a LangGraph control-flow signal, or undefined for a real error. + * LangGraph pauses (`interrupt()`) and hands off (`Command.PARENT`) by throwing a + * `GraphBubbleUp` through the chain callbacks; the run itself did not fail. + * Matches LangGraph's own `isGraphBubbleUp` without importing LangGraph. + */ +function getGraphBubbleUpName( + error: unknown, + runError: string, +): string | undefined { + if (error != null && typeof error === "object") { + const { name, is_bubble_up } = error as { + name?: unknown; + is_bubble_up?: unknown; + }; + if (is_bubble_up === true) { + return typeof name === "string" ? name : "GraphBubbleUp"; + } + return typeof name === "string" && GRAPH_BUBBLE_UP_NAMES.has(name) + ? name + : undefined; + } + if (error != null) { + return undefined; + } + const { name } = parseRunError(runError); + return name != null && GRAPH_BUBBLE_UP_NAMES.has(name) ? name : undefined; +} + export class LangChainTracer extends BaseTracer { private tracer: FITracer; private runs: Record = {}; @@ -218,7 +258,19 @@ export class LangChainTracer extends BaseTracer { return; } const { span } = runWithSpan; - if (run.error != null) { + const bubbleUp = + run.error != null + ? getGraphBubbleUpName(this.runErrors[run.id], run.error) + : undefined; + if (bubbleUp != null) { + // A pause or handoff, not a failure: no exception, and no status message that + // could carry the interrupt payload. + if (GRAPH_INTERRUPT_NAMES.has(bubbleUp)) { + span.setAttribute("langgraph.interrupt", true); + span.addEvent("interrupt", { "langgraph.interrupt": true }); + } + span.setStatus({ code: SpanStatusCode.OK }); + } else if (run.error != null) { recordRunError(span, this.runErrors[run.id], run.error); } else { span.setStatus({ code: SpanStatusCode.OK }); diff --git a/typescript/pnpm-lock.yaml b/typescript/pnpm-lock.yaml index 3eae5818d..1ad32abdd 100644 --- a/typescript/pnpm-lock.yaml +++ b/typescript/pnpm-lock.yaml @@ -1190,6 +1190,9 @@ importers: '@opentelemetry/api': specifier: ^1.9.0 version: 1.9.0 + '@opentelemetry/core': + specifier: ^2.0.1 + version: 2.0.1(@opentelemetry/api@1.9.0) '@opentelemetry/instrumentation': specifier: ^0.202.0 version: 0.202.0(@opentelemetry/api@1.9.0)