From 49dd5085eb0236f2238b559734055f3849087093 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 14 Oct 2026 20:00:55 -0700 Subject: [PATCH 1/7] docs(coding-agents): Gemini CLI application session tracing (TH-8285) Recipe for Gemini CLI's built-in OpenTelemetry export (telemetry.enabled, traces: true, logPrompts: false, target local, OTLP/HTTP) through a local otelcol-contrib in front of Future AGI. The collector adds the Future AGI headers and project_name (Gemini CLI 0.64.0 has no OTLP header setting), drops logs and metrics (Future AGI ingests traces only), strips the system prompt, tool schemas, message attributes and process.command_args (N-06 privacy default), maps Gemini CLI's operation names to span kinds, names tool spans after the tool, copies the session id to session.id, and exports uncompressed (fi-collector answers gzip with 400). Includes run_e2e.sh (isolated HOME) and a contract test that runs the shipped config in a real otelcol-contrib against spans recorded from Gemini CLI 0.64.0. Tested end to end against a local Future AGI. Co-Authored-By: Claude Opus 5.5 --- .../gemini-cli/otel-collector.yaml | 92 ++++ examples/coding-agents/gemini-cli/run_e2e.sh | 75 ++++ .../coding-agents/gemini-cli/settings.json | 10 + .../tests/collector-test-overrides.yaml | 16 + .../fixtures/gemini-cli-0.64.0-traces.json | 419 ++++++++++++++++++ .../gemini-cli/tests/test_collector.py | 128 ++++++ src/lib/navigation.ts | 1 + .../integrations/coding-agents/gemini-cli.mdx | 152 +++++++ 8 files changed, 893 insertions(+) create mode 100644 examples/coding-agents/gemini-cli/otel-collector.yaml create mode 100755 examples/coding-agents/gemini-cli/run_e2e.sh create mode 100644 examples/coding-agents/gemini-cli/settings.json create mode 100644 examples/coding-agents/gemini-cli/tests/collector-test-overrides.yaml create mode 100644 examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json create mode 100644 examples/coding-agents/gemini-cli/tests/test_collector.py create mode 100644 src/pages/docs/integrations/coding-agents/gemini-cli.mdx diff --git a/examples/coding-agents/gemini-cli/otel-collector.yaml b/examples/coding-agents/gemini-cli/otel-collector.yaml new file mode 100644 index 000000000..8fefeaba8 --- /dev/null +++ b/examples/coding-agents/gemini-cli/otel-collector.yaml @@ -0,0 +1,92 @@ +# OpenTelemetry Collector between Gemini CLI and Future AGI. +# +# Gemini CLI exports OTLP traces, logs and metrics to this collector. The +# collector keeps the traces, removes prompt-shaped attributes, labels the span +# kinds Future AGI shows, and forwards the traces with your Future AGI keys. +# Logs and metrics are dropped: the Future AGI collector accepts traces only. +# +# FI_API_KEY=... FI_SECRET_KEY=... otelcol-contrib --config otel-collector.yaml +# +# Environment: +# FI_API_KEY, FI_SECRET_KEY required +# FI_BASE_URL default https://api.futureagi.com (traces go to $FI_BASE_URL/tracer/v1/traces) +# FI_PROJECT_NAME default gemini-cli +# GEMINI_OTLP_LISTEN default 127.0.0.1:4318 (Gemini CLI's otlpEndpoint must match) + +receivers: + otlp: + protocols: + http: + endpoint: ${env:GEMINI_OTLP_LISTEN:-127.0.0.1:4318} + +processors: + transform/gemini-cli: + error_mode: ignore + trace_statements: + - context: resource + statements: + - set(attributes["project_name"], "${env:FI_PROJECT_NAME:-gemini-cli}") + # Node's process detector records the full command line, which for + # `gemini -p "..."` contains the prompt. + - delete_key(attributes, "process.command_args") + - delete_key(attributes, "process.command") + - context: span + statements: + # Gemini CLI names its operations llm_call, tool_call, agent_call and + # schedule_tool_calls. Map them to the span kinds Future AGI uses. + - set(attributes["gen_ai.span.kind"], "LLM") where attributes["gen_ai.operation.name"] == "llm_call" + - set(attributes["gen_ai.span.kind"], "TOOL") where attributes["gen_ai.operation.name"] == "tool_call" + - set(attributes["gen_ai.span.kind"], "AGENT") where attributes["gen_ai.operation.name"] == "agent_call" + - set(attributes["gen_ai.span.kind"], "CHAIN") where attributes["gen_ai.operation.name"] == "schedule_tool_calls" + # Name tool spans after the tool instead of "tool_call". + - set(name, attributes["gen_ai.tool.name"]) where name == "tool_call" and attributes["gen_ai.tool.name"] != nil + # Every span carries the Gemini CLI session id; as session.id it groups + # the session's traces into one Future AGI session. + - set(attributes["session.id"], attributes["gen_ai.conversation.id"]) where attributes["gen_ai.conversation.id"] != nil + # Only present with the content opt-in (logPrompts: true): show the + # messages as the span's input and output in Future AGI. + - set(attributes["input.value"], attributes["gen_ai.input.messages"]) where attributes["gen_ai.input.messages"] != nil + - set(attributes["output.value"], attributes["gen_ai.output.messages"]) where attributes["gen_ai.output.messages"] != nil + + # Privacy default. With logPrompts false Gemini CLI already leaves out the + # message attributes; this also drops the system prompt (it embeds your + # GEMINI.md memory) and the tool schemas. Remove this processor from the + # pipeline below only to opt in to content. + transform/strip-content: + error_mode: ignore + trace_statements: + - context: span + statements: + - delete_key(attributes, "gen_ai.input.messages") + - delete_key(attributes, "gen_ai.output.messages") + - delete_key(attributes, "input.value") + - delete_key(attributes, "output.value") + - delete_key(attributes, "gen_ai.system_instructions") + - delete_key(attributes, "gen_ai.tool.definitions") + + batch: + timeout: 2s + +exporters: + otlphttp/futureagi: + traces_endpoint: ${env:FI_BASE_URL:-https://api.futureagi.com}/tracer/v1/traces + # The Future AGI collector reads the request body as-is; the exporter's + # default gzip encoding is answered with HTTP 400. + compression: none + headers: + X-Api-Key: ${env:FI_API_KEY} + X-Secret-Key: ${env:FI_SECRET_KEY} + nop: {} + +service: + pipelines: + traces: + receivers: [otlp] + processors: [transform/gemini-cli, transform/strip-content, batch] + exporters: [otlphttp/futureagi] + logs: + receivers: [otlp] + exporters: [nop] + metrics: + receivers: [otlp] + exporters: [nop] diff --git a/examples/coding-agents/gemini-cli/run_e2e.sh b/examples/coding-agents/gemini-cli/run_e2e.sh new file mode 100755 index 000000000..e69dc6374 --- /dev/null +++ b/examples/coding-agents/gemini-cli/run_e2e.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# One real Gemini CLI session, exported through otel-collector.yaml to Future AGI. +# +# FI_API_KEY=... FI_SECRET_KEY=... GEMINI_API_KEY=... ./run_e2e.sh +# +# Optional: FI_BASE_URL (default https://api.futureagi.com), FI_PROJECT_NAME +# (default gemini-cli), GEMINI_MODEL (default gemini-3.8-flash), +# GEMINI_BIN / OTELCOL_BIN (default: gemini / otelcol-contrib on PATH), +# CAPTURE_CONTENT=1 to run the documented content opt-in instead. +# +# Gemini CLI runs with a throwaway HOME and project directory, so your own +# ~/.gemini settings and sessions are not read or changed. The prompt asks for +# a directory listing and a file read, so the session has tool calls. The run +# prints the session id (gen_ai.conversation.id) and a marker string that is +# only in the prompt: with the default config it must not appear in Future AGI. +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +: "${FI_API_KEY:?set FI_API_KEY}" "${FI_SECRET_KEY:?set FI_SECRET_KEY}" +: "${GEMINI_API_KEY:?set GEMINI_API_KEY}" +GEMINI_BIN="${GEMINI_BIN:-gemini}" +OTELCOL_BIN="${OTELCOL_BIN:-otelcol-contrib}" +PORT="${GEMINI_OTLP_PORT:-24318}" +export FI_PROJECT_NAME="${FI_PROJECT_NAME:-gemini-cli}" +export GEMINI_OTLP_LISTEN="127.0.0.1:$PORT" + +WORK="$(mktemp -d)" +COLLECTOR_ARGS=(--config "$HERE/otel-collector.yaml") +LOG_PROMPTS=false +if [ "${CAPTURE_CONTENT:-0}" = "1" ]; then + LOG_PROMPTS=true + COLLECTOR_ARGS+=(--set "service.pipelines.traces.processors=[transform/gemini-cli,batch]") +fi + +"$OTELCOL_BIN" "${COLLECTOR_ARGS[@]}" > "$WORK/collector.log" 2>&1 & +COLLECTOR_PID=$! +trap 'kill "$COLLECTOR_PID" 2>/dev/null || true' EXIT +for _ in $(seq 1 50); do + (exec 3<>"/dev/tcp/127.0.0.1/$PORT") 2>/dev/null && break + sleep 0.2 +done + +mkdir -p "$WORK/home/.gemini" "$WORK/project" +python3 - "$HERE/settings.json" "$WORK/home/.gemini/settings.json" "$PORT" "$LOG_PROMPTS" <<'EOF' +import json, sys +src, dst, port, log_prompts = sys.argv[1:] +settings = json.load(open(src)) +settings["telemetry"]["otlpEndpoint"] = "http://127.0.0.1:" + port +settings["telemetry"]["logPrompts"] = log_prompts == "true" +settings["security"] = {"auth": {"selectedType": "gemini-api-key"}} +json.dump(settings, open(dst, "w"), indent=2) +EOF +printf '# Gemini CLI E2E fixture\n\nA throwaway project for the Future AGI recipe.\n' > "$WORK/project/README.md" + +SESSION_ID="$(python3 -c 'import uuid; print(uuid.uuid4())')" +MARKER="FI-E2E-MARKER-$(date +%s)" +( + cd "$WORK/project" + HOME="$WORK/home" GEMINI_CLI_TRUST_WORKSPACE=true OTEL_NODE_RESOURCE_DETECTORS=env \ + "$GEMINI_BIN" -m "${GEMINI_MODEL:-gemini-3.8-flash}" --session-id "$SESSION_ID" \ + -p "List the files in this folder and read README.md. Reply with its first heading. $MARKER" +) + +# Gemini CLI flushes on exit; give the collector's batch processor time to send. +sleep 5 +kill "$COLLECTOR_PID" +wait "$COLLECTOR_PID" 2>/dev/null || true +trap - EXIT +if grep -q -i -E "error|failed" "$WORK/collector.log"; then + echo "collector log ($WORK/collector.log) has errors:" >&2 + grep -i -E "error|failed" "$WORK/collector.log" | tail -5 >&2 +fi +echo "session_id=$SESSION_ID" +echo "marker=$MARKER" +echo "project=$FI_PROJECT_NAME" diff --git a/examples/coding-agents/gemini-cli/settings.json b/examples/coding-agents/gemini-cli/settings.json new file mode 100644 index 000000000..8a399e483 --- /dev/null +++ b/examples/coding-agents/gemini-cli/settings.json @@ -0,0 +1,10 @@ +{ + "telemetry": { + "enabled": true, + "target": "local", + "otlpEndpoint": "http://127.0.0.1:4318", + "otlpProtocol": "http", + "traces": true, + "logPrompts": false + } +} diff --git a/examples/coding-agents/gemini-cli/tests/collector-test-overrides.yaml b/examples/coding-agents/gemini-cli/tests/collector-test-overrides.yaml new file mode 100644 index 000000000..372890d7b --- /dev/null +++ b/examples/coding-agents/gemini-cli/tests/collector-test-overrides.yaml @@ -0,0 +1,16 @@ +# Test-only overlay for otel-collector.yaml: write forwarded traces to a file +# instead of sending them to Future AGI. +exporters: + file/test: + path: ${env:TEST_OUT} + flush_interval: 100ms +processors: + batch: + timeout: 100ms +service: + telemetry: + metrics: + level: none + pipelines: + traces: + exporters: [file/test] diff --git a/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json b/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json new file mode 100644 index 000000000..337150572 --- /dev/null +++ b/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json @@ -0,0 +1,419 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "v26.8.1" + } + }, + { + "key": "session.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "process.command_args", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "node" + }, + { + "stringValue": "gemini" + }, + { + "stringValue": "-p" + }, + { + "stringValue": "FIXTURE-CONTENT-MARKER prompt on the command line" + } + ] + } + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "gemini-cli", + "version": "v1" + }, + "spans": [ + { + "traceId": "45dc5d4d5ecd078b2b9c925d48995b57", + "spanId": "81ea502ba18b0eff", + "flags": 257, + "name": "llm_call", + "kind": 1, + "startTimeUnixNano": "1792027795983000000", + "endTimeUnixNano": "1792027802035021583", + "attributes": [ + { + "key": "gen_ai.request.model", + "value": { + "stringValue": "gemini-3.8-flash" + } + }, + { + "key": "gen_ai.prompt.name", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.system_instructions", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.system_instructions\"}]" + } + }, + { + "key": "gen_ai.tool.definitions", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.tool.definitions\"}]" + } + }, + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.input.messages\"}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.output.messages\"}]" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "llm_call" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power " + } + }, + { + "key": "gen_ai.usage.input_tokens", + "value": { + "intValue": "8816" + } + }, + { + "key": "gen_ai.usage.output_tokens", + "value": { + "intValue": "38" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "44fef0931f271d79c16809bc24382afb", + "spanId": "577562294772ba39", + "parentSpanId": "1669e7ece2fceb8b", + "flags": 257, + "name": "tool_call", + "kind": 1, + "startTimeUnixNano": "1792027802044000000", + "endTimeUnixNano": "1792027802057322208", + "attributes": [ + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "list_directory" + } + }, + { + "key": "gen_ai.tool.call_id", + "value": { + "stringValue": "list_directory__call_131963" + } + }, + { + "key": "gen_ai.tool.description", + "value": { + "stringValue": "Lists the names of files and subdirectories directly within " + } + }, + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.input.messages\"}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.output.messages\"}]" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "tool_call" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power " + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "44fef0931f271d79c16809bc24382afb", + "spanId": "234a34b7405f5ec8", + "parentSpanId": "1669e7ece2fceb8b", + "flags": 257, + "name": "tool_call", + "kind": 1, + "startTimeUnixNano": "1792027802047000000", + "endTimeUnixNano": "1792027802064669833", + "attributes": [ + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "read_file" + } + }, + { + "key": "gen_ai.tool.call_id", + "value": { + "stringValue": "read_file__call_131969" + } + }, + { + "key": "gen_ai.tool.description", + "value": { + "stringValue": "Reads and returns the content of a specified file. If the fi" + } + }, + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.input.messages\"}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.output.messages\"}]" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "tool_call" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power " + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "44fef0931f271d79c16809bc24382afb", + "spanId": "1669e7ece2fceb8b", + "flags": 257, + "name": "schedule_tool_calls", + "kind": 1, + "startTimeUnixNano": "1792027802038000000", + "endTimeUnixNano": "1792027802070371583", + "attributes": [ + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.input.messages\"}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.output.messages\"}]" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "schedule_tool_calls" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power " + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "a4ee9efb18dd4ee74e57eba8f8d30a82", + "spanId": "f10e2357106088c0", + "flags": 257, + "name": "llm_call", + "kind": 1, + "startTimeUnixNano": "1792027802079000000", + "endTimeUnixNano": "1792027804392816500", + "attributes": [ + { + "key": "gen_ai.request.model", + "value": { + "stringValue": "gemini-3.8-flash" + } + }, + { + "key": "gen_ai.prompt.name", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.system_instructions", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.system_instructions\"}]" + } + }, + { + "key": "gen_ai.tool.definitions", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.tool.definitions\"}]" + } + }, + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.input.messages\"}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.output.messages\"}]" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "llm_call" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power " + } + }, + { + "key": "gen_ai.usage.input_tokens", + "value": { + "intValue": "9255" + } + }, + { + "key": "gen_ai.usage.output_tokens", + "value": { + "intValue": "38" + } + } + ], + "status": { + "code": 1 + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/examples/coding-agents/gemini-cli/tests/test_collector.py b/examples/coding-agents/gemini-cli/tests/test_collector.py new file mode 100644 index 000000000..dbb95346b --- /dev/null +++ b/examples/coding-agents/gemini-cli/tests/test_collector.py @@ -0,0 +1,128 @@ +"""Run otel-collector.yaml in a real otelcol-contrib and check what it forwards. + +The input is spans recorded from Gemini CLI 0.64.0 with content capture on +(``logPrompts: true``), with the content values replaced by a marker +(tests/fixtures/gemini-cli-0.64.0-traces.json). The Future AGI exporter is +swapped for a file exporter (tests/collector-test-overrides.yaml); everything +else is the shipped config. + + OTELCOL_BIN=/path/to/otelcol-contrib python3 -m unittest discover -s tests + +Skipped when no otelcol-contrib binary is found. +""" +from __future__ import annotations + +import json +import os +import shutil +import socket +import subprocess +import tempfile +import time +import unittest +import urllib.request +from pathlib import Path + +HERE = Path(__file__).resolve().parent +CONFIG = HERE.parent / "otel-collector.yaml" +OVERRIDES = HERE / "collector-test-overrides.yaml" +FIXTURE = HERE / "fixtures" / "gemini-cli-0.64.0-traces.json" +MARKER = "FIXTURE-CONTENT-MARKER" +OTELCOL = os.environ.get("OTELCOL_BIN") or shutil.which("otelcol-contrib") + + +def free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def attrs(items): + out = {} + for a in items or []: + value = a["value"] + out[a["key"]] = next(iter(value.values())) if value else None + return out + + +@unittest.skipUnless(OTELCOL, "otelcol-contrib not found (set OTELCOL_BIN)") +class CollectorTest(unittest.TestCase): + def run_collector(self, *extra_config: str): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, True) + out = Path(tmp) / "out.json" + port = free_port() + env = dict(os.environ, GEMINI_OTLP_LISTEN="127.0.0.1:%d" % port, TEST_OUT=str(out), + FI_API_KEY="test-key", FI_SECRET_KEY="test-secret", FI_PROJECT_NAME="gemini-cli-test") + args = [OTELCOL, "--config", "file:" + str(CONFIG), "--config", "file:" + str(OVERRIDES)] + for item in extra_config: + args += ["--set", item] + proc = subprocess.Popen(args, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + self.addCleanup(proc.stderr.close) + self.addCleanup(proc.wait, 10) + self.addCleanup(proc.terminate) + body = FIXTURE.read_bytes() + for _ in range(100): + try: + req = urllib.request.Request("http://127.0.0.1:%d/v1/traces" % port, data=body, + headers={"Content-Type": "application/json"}) + urllib.request.urlopen(req, timeout=2).read() + break + except OSError: + if proc.poll() is not None: + self.fail("collector exited: " + proc.stderr.read().decode()[-2000:]) + time.sleep(0.2) + for _ in range(100): + if out.exists() and out.stat().st_size: + break + time.sleep(0.2) + text = out.read_text() + self.assertTrue(text, "collector wrote nothing") + return text, [json.loads(line) for line in text.splitlines() if line.strip()] + + def spans(self, requests): + result = [] + for req in requests: + for rs in req["resourceSpans"]: + resource = attrs(rs["resource"]["attributes"]) + for ss in rs["scopeSpans"]: + for span in ss["spans"]: + result.append((resource, span["name"], attrs(span["attributes"]))) + return result + + def test_default_strips_content_and_labels_kinds(self): + text, requests = self.run_collector() + self.assertNotIn(MARKER, text) + spans = self.spans(requests) + self.assertEqual(len(spans), 5) + kinds = {} + for resource, name, a in spans: + self.assertEqual(resource["project_name"], "gemini-cli-test") + self.assertNotIn("process.command_args", resource) + for key in ("gen_ai.input.messages", "gen_ai.output.messages", "input.value", + "output.value", "gen_ai.system_instructions", "gen_ai.tool.definitions"): + self.assertNotIn(key, a) + self.assertEqual(a["gen_ai.conversation.id"], "fixture-session-0001") + self.assertEqual(a["session.id"], "fixture-session-0001") + kinds.setdefault(name, set()).add(a.get("gen_ai.span.kind")) + self.assertEqual(kinds, {"llm_call": {"LLM"}, "list_directory": {"TOOL"}, + "read_file": {"TOOL"}, "schedule_tool_calls": {"CHAIN"}}) + llm = [a for _, name, a in spans if name == "llm_call"] + self.assertTrue(all(a["gen_ai.request.model"] == "gemini-3.8-flash" for a in llm)) + self.assertTrue(all(int(a["gen_ai.usage.input_tokens"]) > 0 for a in llm)) + + def test_opt_in_pipeline_keeps_messages(self): + # The documented opt-in: drop transform/strip-content from the pipeline. + text, requests = self.run_collector( + "service.pipelines.traces.processors=[transform/gemini-cli,batch]") + self.assertIn(MARKER + " gen_ai.input.messages", text) + self.assertIn(MARKER + " gen_ai.output.messages", text) + for _, _, a in self.spans(requests): + self.assertIn("gen_ai.input.messages", a["input.value"]) + self.assertIn("gen_ai.output.messages", a["output.value"]) + # The command line is removed either way. + self.assertNotIn("prompt on the command line", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/lib/navigation.ts b/src/lib/navigation.ts index 9ebda640c..329e266cc 100644 --- a/src/lib/navigation.ts +++ b/src/lib/navigation.ts @@ -968,6 +968,7 @@ export const tabNavigation: NavTab[] = [ // coding-agents:devin // coding-agents:devin-desktop // coding-agents:gemini-cli + { title: 'Gemini CLI', href: '/docs/integrations/coding-agents/gemini-cli' }, // coding-agents:github-copilot // coding-agents:google-antigravity // coding-agents:goose diff --git a/src/pages/docs/integrations/coding-agents/gemini-cli.mdx b/src/pages/docs/integrations/coding-agents/gemini-cli.mdx new file mode 100644 index 000000000..905f58d82 --- /dev/null +++ b/src/pages/docs/integrations/coding-agents/gemini-cli.mdx @@ -0,0 +1,152 @@ +--- +title: "Gemini CLI session tracing with Future AGI" +description: "Trace Gemini CLI sessions in Future AGI with Gemini CLI's built-in OpenTelemetry export and a local OpenTelemetry Collector: model calls with tokens, tool calls, errors, grouped by session. Prompt and tool content stay off by default." +--- + +## About + +Gemini CLI has [built-in OpenTelemetry](https://geminicli.com/docs/cli/telemetry/). With traces turned on it emits a span for every model call and every tool call. This page sends those spans through a small OpenTelemetry Collector running next to Gemini CLI, which forwards them to Future AGI: + +``` +Gemini CLI ──OTLP/HTTP──▶ otelcol-contrib (localhost:4318) ──▶ Future AGI + keeps traces, drops logs and metrics, + strips prompt-shaped attributes, + labels span kinds, adds your keys +``` + +One Gemini CLI session shows up in Future AGI as one **session** made of several traces: + +``` +llm_call one model request: model, input/output tokens, status +schedule_tool_calls one batch of tool calls the model asked for +├── list_directory tool call: name, call id, duration, status +└── read_file +llm_call the next model request +``` + +Gemini CLI does not emit a root span for a session or a turn, so each model call and each tool batch is its own trace. Every span carries the Gemini CLI session id, which the collector copies to `session.id`, so the **Sessions** tab groups them. + +This is not the same as: + +- **The Google GenAI SDK instrumentor** (`traceAI-google-genai`), which traces your own code that calls the Gemini API. It does not see Gemini CLI's tool scheduler or session. +- **Google Antigravity** or Gemini in other IDEs. Those are separate integrations. +- **Gemini CLI's `target: "gcp"` export**, which sends telemetry to Google Cloud, not to Future AGI. + +| | | +| --- | --- | +| Tested with | Gemini CLI 0.64.0 (`@google/gemini-cli`, macOS), OpenTelemetry Collector Contrib 0.162.0 | +| Gemini CLI docs used | [Observability with OpenTelemetry](https://geminicli.com/docs/cli/telemetry/) and [configuration](https://www.geminicli.com/docs/reference/configuration), read 2026-10-14 | +| Requires | [`otelcol-contrib`](https://github.com/open-telemetry/opentelemetry-collector-releases/releases) (the contrib build: the config uses the `transform` processor) | +| Source | [examples/coding-agents/gemini-cli](https://github.com/future-agi/docs/tree/main/examples/coding-agents/gemini-cli) | + +## 1. Run the collector + +Download the example collector config and start `otelcol-contrib` with your Future AGI keys: + +```bash +git clone --depth 1 https://github.com/future-agi/docs.git fi-docs +export FI_API_KEY=your-futureagi-api-key +export FI_SECRET_KEY=your-futureagi-secret-key +export FI_PROJECT_NAME=gemini-cli +otelcol-contrib --config fi-docs/examples/coding-agents/gemini-cli/otel-collector.yaml +``` + +It listens on `127.0.0.1:4318` (set `GEMINI_OTLP_LISTEN` to change it) and sends traces to `$FI_BASE_URL/tracer/v1/traces` (default `https://api.futureagi.com`; set `FI_BASE_URL` for a self-hosted Future AGI). Keep it running while you use Gemini CLI, or run it as a service. + +The collector does four things Gemini CLI cannot do on its own: + +| Step | Why | +| --- | --- | +| Adds `X-Api-Key`, `X-Secret-Key` and `project_name` | Gemini CLI 0.64.0 has no setting for OTLP headers. (Gemini CLI's main-branch docs list an `otlpHeaders` setting that is not in 0.64.0; the other three steps still need the collector.) | +| Drops logs and metrics | Gemini CLI always exports all three signals; Future AGI ingests traces. | +| Deletes `gen_ai.system_instructions`, `gen_ai.tool.definitions`, message attributes and `process.command_args` | Privacy default, see [Privacy](#privacy). | +| Sets `gen_ai.span.kind` from Gemini CLI's operation names, names tool spans after the tool, copies the session id to `session.id` | Gemini CLI uses its own operation names (`llm_call`, `tool_call`), which Future AGI does not map by itself. | + +It also sends uncompressed: the Future AGI collector answers gzip-encoded OTLP requests with HTTP 400. + +## 2. Turn on Gemini CLI telemetry + +Add the `telemetry` block to `~/.gemini/settings.json` (all projects) or `.gemini/settings.json` in one project: + +```json +{ + "telemetry": { + "enabled": true, + "target": "local", + "otlpEndpoint": "http://127.0.0.1:4318", + "otlpProtocol": "http", + "traces": true, + "logPrompts": false + } +} +``` + +- `traces: true` is required. With it off, Gemini CLI still emits the spans but drops every attribute except the operation name, the agent name and the session id: no model, no tokens, no tool names. +- `logPrompts: false` is not Gemini CLI's default. The vendor default is `true`, which puts prompts, responses, tool arguments and tool results on the spans. Keep it `false` unless you opt in to content. +- Do not set `outfile` (it replaces the OTLP export) or `target: "gcp"`. + +The same settings exist as environment variables (`GEMINI_TELEMETRY_ENABLED`, `GEMINI_TELEMETRY_TRACES_ENABLED`, `GEMINI_TELEMETRY_TARGET`, `GEMINI_TELEMETRY_OTLP_ENDPOINT`, `GEMINI_TELEMETRY_OTLP_PROTOCOL`, `GEMINI_TELEMETRY_LOG_PROMPTS`), which override the file. + +Optionally also export `OTEL_NODE_RESOURCE_DETECTORS=env` in the shell that runs `gemini`. Node's default resource detectors record the full command line, which for `gemini -p "..."` contains the prompt. The collector deletes it either way; this setting keeps it from leaving the Gemini CLI process at all. + +## 3. Run a session + +Start `gemini` in a project and send one prompt that uses a tool, for example *"List the files in this folder and read README.md"*. Exit Gemini CLI, or wait a few seconds: spans are batched. + +Open **Observe** in Future AGI and select the project (`gemini-cli` above). The **Trace** tab lists `llm_call` and `schedule_tool_calls` traces. The **Sessions** tab groups them by Gemini CLI session. + +`run_e2e.sh` in the examples folder does all three steps against a throwaway `HOME`, so it does not touch your own `~/.gemini`: + +```bash +FI_API_KEY=... FI_SECRET_KEY=... GEMINI_API_KEY=... \ + fi-docs/examples/coding-agents/gemini-cli/run_e2e.sh +``` + +## What is captured + +| Gemini CLI span | Future AGI span kind | Fields | +| --- | --- | --- | +| `llm_call` | `LLM` | `gen_ai.request.model`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.prompt.name` (Gemini CLI's prompt id), status `ERROR` when the API call fails (a retried 503 shows up as its own `ERROR` span) | +| `schedule_tool_calls` | `CHAIN` | parent of the tool calls the model requested in one response | +| `tool_call`, renamed to the tool (`read_file`, `list_directory`, `run_shell_command`, ...) | `TOOL` | `gen_ai.tool.name`, `gen_ai.tool.call_id`, `gen_ai.tool.description`, duration, status | +| `agent_call` | `AGENT` | subagent name and description. Present in Gemini CLI 0.64.0's source; not exercised in our test | +| every span | | `gen_ai.conversation.id` and `session.id` = Gemini CLI session id; resource `service.name=gemini-cli` | + +## What is not available + +- **One trace per session or turn.** Gemini CLI has no session or turn span; use the **Sessions** tab. +- **Cost.** Gemini CLI reports tokens, not cost. Future AGI computes cost from the model name only when it has pricing for that model. +- **Cached, thinking and tool-use token counts.** Only input and output tokens are on the spans. +- **Gemini CLI logs and metrics** (`gemini_cli.user_prompt`, `gemini_cli.api_request`, token and latency metrics). The collector drops them; Future AGI ingests traces only. +- **Telemetry in sandbox mode** was not tested. + +## Privacy + +With the settings above, only structure and metadata leave your machine: model, token counts, tool names, call ids, tool descriptions (static text from Gemini CLI), durations, statuses, and the session id. + +- **Not exported by default:** prompts and responses, tool arguments and results (Gemini CLI leaves them out with `logPrompts: false`), the system prompt, which includes your `GEMINI.md` memory, and the tool schemas (the collector deletes them), and the command line. +- **Never forwarded:** Gemini CLI's logs and metrics. + +To include content: + +1. Set `"logPrompts": true` in the `telemetry` block. +2. In `otel-collector.yaml`, remove `transform/strip-content` from the traces pipeline: + + ```yaml + processors: [transform/gemini-cli, batch] + ``` + +Model spans then carry the request and response messages, and tool spans the call request and result, as `gen_ai.input.messages` / `gen_ai.output.messages`; the collector also copies them to the span's input and output in Future AGI. Gemini CLI truncates each value to 10,000 characters. The system prompt and tool schemas are exported too in this mode. + +## Troubleshooting + +**No traces appear.** +1. Check that the collector is running and listening on the `otlpEndpoint` port. With `otlpProtocol: "http"` Gemini CLI posts to `/v1/traces`; the default protocol is `grpc`, which this collector config does not accept. +2. Check the collector's output for `Exporting failed`. `401`/`403` means wrong keys. `400` means a request the Future AGI collector could not decode: keep `compression: none` on the exporter. +3. `telemetry.enabled` must be `true`. `useCollector` together with `useCliAuth` disables telemetry. + +**Spans have no model, tokens or tool names.** `telemetry.traces` is `false` (or `GEMINI_TELEMETRY_TRACES_ENABLED` overrides it). + +**Tool spans show as `unknown` and are named `tool_call`.** The spans did not go through the `transform/gemini-cli` processor; send Gemini CLI's export to the collector, not straight to Future AGI. + +**Prompts show up although you did not opt in.** `logPrompts` defaults to `true` in Gemini CLI. Set it to `false`, and keep `transform/strip-content` in the pipeline. From 68174b31cc807e31c1a3f88a7afda38a0888cdf0 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 14 Oct 2026 21:38:25 -0700 Subject: [PATCH 2/7] fix(gemini-cli): strip Node detector identity attributes in the collector (TH-8285) The default config already removed prompts, system instructions, tool schemas and process.command_args, and a fresh default session confirms 0 spans with gen_ai.system_instructions and 0 '/Users/' hits. But when OTEL_NODE_RESOURCE_DETECTORS=env (documented as optional) is not set, Node's process and host detectors also exported process.executable.path, process.owner and host.name: local paths (under the home directory for nvm installs), the OS user name and the machine name. The collector now deletes process.command, process.executable.path/name, process.owner, host.name and host.id as well, keeping runtime and arch. The contract test fixture now carries these attributes with a /Users/ path; the test fails on the previous config and passes on this one. The page's privacy text names them and spells out that the content opt-in also exports the system prompt (GEMINI.md memory, working directory). Co-Authored-By: Claude Opus 5.5 --- .../gemini-cli/otel-collector.yaml | 10 ++++- .../fixtures/gemini-cli-0.64.0-traces.json | 42 +++++++++++++++++++ .../gemini-cli/tests/test_collector.py | 16 ++++++- .../integrations/coding-agents/gemini-cli.mdx | 8 ++-- 4 files changed, 68 insertions(+), 8 deletions(-) diff --git a/examples/coding-agents/gemini-cli/otel-collector.yaml b/examples/coding-agents/gemini-cli/otel-collector.yaml index 8fefeaba8..576895e90 100644 --- a/examples/coding-agents/gemini-cli/otel-collector.yaml +++ b/examples/coding-agents/gemini-cli/otel-collector.yaml @@ -26,10 +26,16 @@ processors: - context: resource statements: - set(attributes["project_name"], "${env:FI_PROJECT_NAME:-gemini-cli}") - # Node's process detector records the full command line, which for - # `gemini -p "..."` contains the prompt. + # Node's default resource detectors record the full command line + # (for `gemini -p "..."` it contains the prompt), local paths, the + # OS user name and the machine name. Keep only runtime and arch. - delete_key(attributes, "process.command_args") - delete_key(attributes, "process.command") + - delete_key(attributes, "process.executable.path") + - delete_key(attributes, "process.executable.name") + - delete_key(attributes, "process.owner") + - delete_key(attributes, "host.name") + - delete_key(attributes, "host.id") - context: span statements: # Gemini CLI names its operations llm_call, tool_call, agent_call and diff --git a/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json b/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json index 337150572..2b043682e 100644 --- a/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json +++ b/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json @@ -41,6 +41,48 @@ ] } } + }, + { + "key": "process.executable.path", + "value": { + "stringValue": "/Users/fixture-user/.nvm/versions/node/v22.18.0/bin/node" + } + }, + { + "key": "process.executable.name", + "value": { + "stringValue": "/Users/fixture-user/.nvm/versions/node/v22.18.0/bin/node" + } + }, + { + "key": "process.command", + "value": { + "stringValue": "/Users/fixture-user/.npm-global/bin/gemini" + } + }, + { + "key": "process.owner", + "value": { + "stringValue": "fixture-user" + } + }, + { + "key": "host.name", + "value": { + "stringValue": "fixture-users-laptop.local" + } + }, + { + "key": "host.arch", + "value": { + "stringValue": "arm64" + } + }, + { + "key": "process.runtime.name", + "value": { + "stringValue": "nodejs" + } } ] }, diff --git a/examples/coding-agents/gemini-cli/tests/test_collector.py b/examples/coding-agents/gemini-cli/tests/test_collector.py index dbb95346b..ed2fc4b45 100644 --- a/examples/coding-agents/gemini-cli/tests/test_collector.py +++ b/examples/coding-agents/gemini-cli/tests/test_collector.py @@ -28,6 +28,10 @@ OVERRIDES = HERE / "collector-test-overrides.yaml" FIXTURE = HERE / "fixtures" / "gemini-cli-0.64.0-traces.json" MARKER = "FIXTURE-CONTENT-MARKER" +# Added by Node's process and host resource detectors when +# OTEL_NODE_RESOURCE_DETECTORS is not set to env: paths, user and machine name. +LOCAL_IDENTITY = ("process.command_args", "process.command", "process.executable.path", + "process.executable.name", "process.owner", "host.name", "host.id") OTELCOL = os.environ.get("OTELCOL_BIN") or shutil.which("otelcol-contrib") @@ -93,12 +97,17 @@ def spans(self, requests): def test_default_strips_content_and_labels_kinds(self): text, requests = self.run_collector() self.assertNotIn(MARKER, text) + self.assertNotIn("/Users/", text) + self.assertNotIn("fixture-user", text) spans = self.spans(requests) self.assertEqual(len(spans), 5) kinds = {} for resource, name, a in spans: self.assertEqual(resource["project_name"], "gemini-cli-test") - self.assertNotIn("process.command_args", resource) + for key in LOCAL_IDENTITY: + self.assertNotIn(key, resource) + self.assertEqual(resource["host.arch"], "arm64") + self.assertEqual(resource["process.runtime.name"], "nodejs") for key in ("gen_ai.input.messages", "gen_ai.output.messages", "input.value", "output.value", "gen_ai.system_instructions", "gen_ai.tool.definitions"): self.assertNotIn(key, a) @@ -120,8 +129,11 @@ def test_opt_in_pipeline_keeps_messages(self): for _, _, a in self.spans(requests): self.assertIn("gen_ai.input.messages", a["input.value"]) self.assertIn("gen_ai.output.messages", a["output.value"]) - # The command line is removed either way. + # The command line, local paths, user and machine name are removed either way. self.assertNotIn("prompt on the command line", text) + self.assertNotIn("fixture-user", text) + # The opt-in exports the system prompt and tool schemas (documented). + self.assertIn(MARKER + " gen_ai.system_instructions", text) if __name__ == "__main__": diff --git a/src/pages/docs/integrations/coding-agents/gemini-cli.mdx b/src/pages/docs/integrations/coding-agents/gemini-cli.mdx index 905f58d82..e21074926 100644 --- a/src/pages/docs/integrations/coding-agents/gemini-cli.mdx +++ b/src/pages/docs/integrations/coding-agents/gemini-cli.mdx @@ -59,7 +59,7 @@ The collector does four things Gemini CLI cannot do on its own: | --- | --- | | Adds `X-Api-Key`, `X-Secret-Key` and `project_name` | Gemini CLI 0.64.0 has no setting for OTLP headers. (Gemini CLI's main-branch docs list an `otlpHeaders` setting that is not in 0.64.0; the other three steps still need the collector.) | | Drops logs and metrics | Gemini CLI always exports all three signals; Future AGI ingests traces. | -| Deletes `gen_ai.system_instructions`, `gen_ai.tool.definitions`, message attributes and `process.command_args` | Privacy default, see [Privacy](#privacy). | +| Deletes `gen_ai.system_instructions`, `gen_ai.tool.definitions`, message attributes, and the command line, executable paths, OS user name and machine name that Node's resource detectors add | Privacy default, see [Privacy](#privacy). | | Sets `gen_ai.span.kind` from Gemini CLI's operation names, names tool spans after the tool, copies the session id to `session.id` | Gemini CLI uses its own operation names (`llm_call`, `tool_call`), which Future AGI does not map by itself. | It also sends uncompressed: the Future AGI collector answers gzip-encoded OTLP requests with HTTP 400. @@ -87,7 +87,7 @@ Add the `telemetry` block to `~/.gemini/settings.json` (all projects) or `.gemin The same settings exist as environment variables (`GEMINI_TELEMETRY_ENABLED`, `GEMINI_TELEMETRY_TRACES_ENABLED`, `GEMINI_TELEMETRY_TARGET`, `GEMINI_TELEMETRY_OTLP_ENDPOINT`, `GEMINI_TELEMETRY_OTLP_PROTOCOL`, `GEMINI_TELEMETRY_LOG_PROMPTS`), which override the file. -Optionally also export `OTEL_NODE_RESOURCE_DETECTORS=env` in the shell that runs `gemini`. Node's default resource detectors record the full command line, which for `gemini -p "..."` contains the prompt. The collector deletes it either way; this setting keeps it from leaving the Gemini CLI process at all. +Optionally also export `OTEL_NODE_RESOURCE_DETECTORS=env` in the shell that runs `gemini`. Node's default resource detectors record the full command line (for `gemini -p "..."` it contains the prompt), the Node executable path (often under your home directory), your OS user name and the machine name. The collector deletes all of these either way (`process.command_args`, `process.command`, `process.executable.*`, `process.owner`, `host.name`, `host.id`); this setting keeps them from leaving the Gemini CLI process at all. ## 3. Run a session @@ -124,7 +124,7 @@ FI_API_KEY=... FI_SECRET_KEY=... GEMINI_API_KEY=... \ With the settings above, only structure and metadata leave your machine: model, token counts, tool names, call ids, tool descriptions (static text from Gemini CLI), durations, statuses, and the session id. -- **Not exported by default:** prompts and responses, tool arguments and results (Gemini CLI leaves them out with `logPrompts: false`), the system prompt, which includes your `GEMINI.md` memory, and the tool schemas (the collector deletes them), and the command line. +- **Not exported by default:** prompts and responses, tool arguments and results (Gemini CLI leaves them out with `logPrompts: false`), the system prompt, which includes your `GEMINI.md` memory and local paths, and the tool schemas (the collector deletes them), and the command line, executable paths, OS user name and machine name. - **Never forwarded:** Gemini CLI's logs and metrics. To include content: @@ -136,7 +136,7 @@ To include content: processors: [transform/gemini-cli, batch] ``` -Model spans then carry the request and response messages, and tool spans the call request and result, as `gen_ai.input.messages` / `gen_ai.output.messages`; the collector also copies them to the span's input and output in Future AGI. Gemini CLI truncates each value to 10,000 characters. The system prompt and tool schemas are exported too in this mode. +Model spans then carry the request and response messages, and tool spans the call request and result, as `gen_ai.input.messages` / `gen_ai.output.messages`; the collector also copies them to the span's input and output in Future AGI. Gemini CLI truncates each value to 10,000 characters. The system prompt (with your `GEMINI.md` memory and local paths such as the working directory) and the tool schemas are exported too in this mode. The command line, paths, user and machine name stay removed. ## Troubleshooting From 8bef2d7b4eec141204020faff116f0302359bbcf Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Thu, 15 Oct 2026 07:24:31 -0700 Subject: [PATCH 3/7] fix(gemini-cli): strip error text by default, correct page claims (TH-8285) Review fixes for docs#922. - Collector: error stack traces (Gemini CLI install paths, OS user name) are deleted always; by default exception messages are deleted and the status message of an ERROR span becomes the error class, plus the HTTP status for Gemini API errors ("ApiError HTTP 503", "ENOENT", or "error"). Full error text only with the content opt-in. Collector self-metrics off (service.telemetry.metrics.level: none) so port 8888 never collides. - Tests: fixture gains a real 0.64.0 API-error llm_call (exception event, stack trace) and a thrown tool error with a local path; default asserts no path/user/stack trace/error body, opt-in keeps messages but no stack traces; shipped config checked for metrics level none and no compression. - Page: tool failures a tool returns (not throws) end OK in 0.64.0; traces:false still emits model, tool names, system prompt and tool schemas (only tokens are dropped); agent_call carries gen_ai.agent.name=gemini-cli, not the subagent name; error text and stack traces in Privacy; GEMINI_TELEMETRY_ENABLED=false did not stop the export in 0.64.0 (settings file does); Vertex AI instrumentor in "not the same as"; curl for the collector config; port-in-use troubleshooting. - run_e2e.sh: fails if the collector is not listening, checks what the collector forwarded (marker, home, user, host) and exits non-zero on a leak, gemini stdin from /dev/null, deletes its work dir on success, documents GEMINI_OTLP_PORT. Co-Authored-By: Claude Opus 5.5 --- .../gemini-cli/otel-collector.yaml | 31 ++- examples/coding-agents/gemini-cli/run_e2e.sh | 60 +++++- .../tests/collector-test-overrides.yaml | 3 - .../fixtures/gemini-cli-0.64.0-traces.json | 190 ++++++++++++++++++ .../gemini-cli/tests/test_collector.py | 73 +++++-- .../integrations/coding-agents/gemini-cli.mdx | 51 +++-- 6 files changed, 361 insertions(+), 47 deletions(-) diff --git a/examples/coding-agents/gemini-cli/otel-collector.yaml b/examples/coding-agents/gemini-cli/otel-collector.yaml index 576895e90..4b22e34a2 100644 --- a/examples/coding-agents/gemini-cli/otel-collector.yaml +++ b/examples/coding-agents/gemini-cli/otel-collector.yaml @@ -12,6 +12,9 @@ # FI_BASE_URL default https://api.futureagi.com (traces go to $FI_BASE_URL/tracer/v1/traces) # FI_PROJECT_NAME default gemini-cli # GEMINI_OTLP_LISTEN default 127.0.0.1:4318 (Gemini CLI's otlpEndpoint must match) +# +# The collector's own metrics endpoint (port 8888) is turned off at the end of +# this file, so it does not collide with Jupyter or another collector. receivers: otlp: @@ -53,11 +56,18 @@ processors: # messages as the span's input and output in Future AGI. - set(attributes["input.value"], attributes["gen_ai.input.messages"]) where attributes["gen_ai.input.messages"] != nil - set(attributes["output.value"], attributes["gen_ai.output.messages"]) where attributes["gen_ai.output.messages"] != nil + - context: spanevent + statements: + # Gemini CLI records every error as an "exception" event. Its stack + # trace lists file:// paths of your Gemini CLI install (home + # directory, OS user name) and no content, so it is dropped even + # with the content opt-in. + - delete_key(attributes, "exception.stacktrace") # Privacy default. With logPrompts false Gemini CLI already leaves out the # message attributes; this also drops the system prompt (it embeds your - # GEMINI.md memory) and the tool schemas. Remove this processor from the - # pipeline below only to opt in to content. + # GEMINI.md memory), the tool schemas and error text. Remove this processor + # from the pipeline below only to opt in to content. transform/strip-content: error_mode: ignore trace_statements: @@ -69,6 +79,20 @@ processors: - delete_key(attributes, "output.value") - delete_key(attributes, "gen_ai.system_instructions") - delete_key(attributes, "gen_ai.tool.definitions") + # The status message of a failed span is the raw error text: an API + # error body, or for a tool that throws, file paths and command + # output. Keep the ERROR status, replace the text (refined from the + # exception event below). + - set(status.message, "error") where status.code == STATUS_CODE_ERROR + - set(status.message, "") where status.code != STATUS_CODE_ERROR + - context: spanevent + statements: + # Status message = the error class or code (ApiError, ENOENT, ...), + # plus the HTTP status for Gemini API errors ("ApiError HTTP 429"). + - set(span.status.message, attributes["exception.type"]) where name == "exception" and span.status.code == STATUS_CODE_ERROR and IsMatch(attributes["exception.type"], "^[A-Za-z0-9_.:-]{1,80}$") + - set(span.status.message, Concat([attributes["exception.type"], ExtractPatterns(attributes["exception.message"], "\"code\\\\?\":\\s*(?P[1-5][0-9][0-9])")["code"]], " HTTP ")) where name == "exception" and span.status.code == STATUS_CODE_ERROR and attributes["exception.type"] == "ApiError" and IsMatch(attributes["exception.message"], "\"code\\\\?\":\\s*[1-5][0-9][0-9]") + - delete_key(attributes, "exception.type") where not IsMatch(attributes["exception.type"], "^[A-Za-z0-9_.:-]{1,80}$") + - delete_key(attributes, "exception.message") batch: timeout: 2s @@ -85,6 +109,9 @@ exporters: nop: {} service: + telemetry: + metrics: + level: none pipelines: traces: receivers: [otlp] diff --git a/examples/coding-agents/gemini-cli/run_e2e.sh b/examples/coding-agents/gemini-cli/run_e2e.sh index e69dc6374..cafc981a8 100755 --- a/examples/coding-agents/gemini-cli/run_e2e.sh +++ b/examples/coding-agents/gemini-cli/run_e2e.sh @@ -5,14 +5,18 @@ # # Optional: FI_BASE_URL (default https://api.futureagi.com), FI_PROJECT_NAME # (default gemini-cli), GEMINI_MODEL (default gemini-3.8-flash), +# GEMINI_OTLP_PORT (port the collector listens on, default 24318), # GEMINI_BIN / OTELCOL_BIN (default: gemini / otelcol-contrib on PATH), # CAPTURE_CONTENT=1 to run the documented content opt-in instead. # # Gemini CLI runs with a throwaway HOME and project directory, so your own # ~/.gemini settings and sessions are not read or changed. The prompt asks for -# a directory listing and a file read, so the session has tool calls. The run -# prints the session id (gen_ai.conversation.id) and a marker string that is -# only in the prompt: with the default config it must not appear in Future AGI. +# a directory listing and a file read, so the session has tool calls. The +# collector also writes what it forwards to a local file, which the script +# checks: by default the marker string from the prompt, your user name, home +# directory and machine name must not be in it; with CAPTURE_CONTENT=1 the +# marker must be. The script exits non-zero if a check fails and then keeps its +# work directory for inspection; on success it deletes it. set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" @@ -25,7 +29,14 @@ export FI_PROJECT_NAME="${FI_PROJECT_NAME:-gemini-cli}" export GEMINI_OTLP_LISTEN="127.0.0.1:$PORT" WORK="$(mktemp -d)" -COLLECTOR_ARGS=(--config "$HERE/otel-collector.yaml") +fail() { + echo "FAIL: $*" >&2 + echo "work directory kept: $WORK" >&2 + exit 1 +} +COLLECTOR_ARGS=(--config "$HERE/otel-collector.yaml" + --set "exporters.file/e2e.path=$WORK/forwarded.json" + --set "service.pipelines.traces.exporters=[otlphttp/futureagi,file/e2e]") LOG_PROMPTS=false if [ "${CAPTURE_CONTENT:-0}" = "1" ]; then LOG_PROMPTS=true @@ -35,10 +46,16 @@ fi "$OTELCOL_BIN" "${COLLECTOR_ARGS[@]}" > "$WORK/collector.log" 2>&1 & COLLECTOR_PID=$! trap 'kill "$COLLECTOR_PID" 2>/dev/null || true' EXIT +listening=0 for _ in $(seq 1 50); do - (exec 3<>"/dev/tcp/127.0.0.1/$PORT") 2>/dev/null && break + if (exec 3<>"/dev/tcp/127.0.0.1/$PORT") 2>/dev/null; then listening=1; break; fi + kill -0 "$COLLECTOR_PID" 2>/dev/null || break sleep 0.2 done +if [ "$listening" != 1 ]; then + tail -5 "$WORK/collector.log" >&2 + fail "the collector is not listening on 127.0.0.1:$PORT (see $WORK/collector.log)" +fi mkdir -p "$WORK/home/.gemini" "$WORK/project" python3 - "$HERE/settings.json" "$WORK/home/.gemini/settings.json" "$PORT" "$LOG_PROMPTS" <<'EOF' @@ -58,18 +75,41 @@ MARKER="FI-E2E-MARKER-$(date +%s)" cd "$WORK/project" HOME="$WORK/home" GEMINI_CLI_TRUST_WORKSPACE=true OTEL_NODE_RESOURCE_DETECTORS=env \ "$GEMINI_BIN" -m "${GEMINI_MODEL:-gemini-3.8-flash}" --session-id "$SESSION_ID" \ - -p "List the files in this folder and read README.md. Reply with its first heading. $MARKER" -) + -p "List the files in this folder and read README.md. Reply with its first heading. $MARKER" \ + < /dev/null +) || echo "gemini exited non-zero; checking what was exported anyway" >&2 # Gemini CLI flushes on exit; give the collector's batch processor time to send. sleep 5 kill "$COLLECTOR_PID" wait "$COLLECTOR_PID" 2>/dev/null || true trap - EXIT -if grep -q -i -E "error|failed" "$WORK/collector.log"; then - echo "collector log ($WORK/collector.log) has errors:" >&2 - grep -i -E "error|failed" "$WORK/collector.log" | tail -5 >&2 +if grep -q -E $'\terror\t|Exporting failed' "$WORK/collector.log"; then + echo "collector log has errors:" >&2 + grep -E $'\terror\t|Exporting failed' "$WORK/collector.log" | tail -5 >&2 fi + echo "session_id=$SESSION_ID" echo "marker=$MARKER" echo "project=$FI_PROJECT_NAME" +touch "$WORK/forwarded.json" +python3 - "$WORK/forwarded.json" <<'EOF' +import json, sys +spans = [s for line in open(sys.argv[1]) if line.strip() + for rs in json.loads(line)["resourceSpans"] for ss in rs["scopeSpans"] for s in ss["spans"]] +print("spans_forwarded=%d" % len(spans)) +print("span_names=%s" % ",".join(sorted({s["name"] for s in spans}))) +EOF +if [ "$LOG_PROMPTS" = true ]; then + grep -q -F "$MARKER" "$WORK/forwarded.json" || fail "content opt-in: the marker was not forwarded" + echo "privacy: content opt-in, marker forwarded" +else + if grep -q -F "$MARKER" "$WORK/forwarded.json"; then fail "the prompt marker was forwarded"; fi + for needle in "$HOME" "/$(id -un)/" "$(hostname)" "$(hostname -s)"; do + if grep -q -F "$needle" "$WORK/forwarded.json"; then + fail "forwarded spans contain a local identity string" + fi + done + echo "privacy: marker, home directory, user name and machine name not forwarded" +fi +rm -rf "$WORK" diff --git a/examples/coding-agents/gemini-cli/tests/collector-test-overrides.yaml b/examples/coding-agents/gemini-cli/tests/collector-test-overrides.yaml index 372890d7b..5567f93fa 100644 --- a/examples/coding-agents/gemini-cli/tests/collector-test-overrides.yaml +++ b/examples/coding-agents/gemini-cli/tests/collector-test-overrides.yaml @@ -8,9 +8,6 @@ processors: batch: timeout: 100ms service: - telemetry: - metrics: - level: none pipelines: traces: exporters: [file/test] diff --git a/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json b/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json index 2b043682e..12e517cc8 100644 --- a/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json +++ b/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json @@ -452,6 +452,196 @@ "status": { "code": 1 } + }, + { + "traceId": "5a1e0f3c2b9d4e8f8a7b6c5d4e3f2a10", + "spanId": "9b8a7c6d5e4f3a21", + "flags": 257, + "name": "llm_call", + "kind": 1, + "startTimeUnixNano": "1792027810000000000", + "endTimeUnixNano": "1792027810472056458", + "attributes": [ + { + "key": "gen_ai.request.model", + "value": { + "stringValue": "gemini-2.5-flash" + } + }, + { + "key": "gen_ai.prompt.name", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.system_instructions", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.system_instructions\"}]" + } + }, + { + "key": "gen_ai.tool.definitions", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.tool.definitions\"}]" + } + }, + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.input.messages\"}]" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "llm_call" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power " + } + } + ], + "events": [ + { + "timeUnixNano": "1792027810472056458", + "name": "exception", + "attributes": [ + { + "key": "exception.type", + "value": { + "stringValue": "ApiError" + } + }, + { + "key": "exception.message", + "value": { + "stringValue": "{\"error\":{\"message\":\"{\\n \\\"error\\\": {\\n \\\"code\\\": 404,\\n \\\"message\\\": \\\"This model models/gemini-2.5-flash is no longer available to new users. Please update your code to use models/gemini-3.8-flash for the latest features and improvements. We recommend you to use the Interactions API (https://ai.google.dev/gemini-api/docs/get-started).\\\",\\n \\\"status\\\": \\\"NOT_FOUND\\\"\\n }\\n}\\n\",\"code\":404,\"status\":\"\"}}" + } + }, + { + "key": "exception.stacktrace", + "value": { + "stringValue": "ApiError: {\"error\":{\"message\":\"{\\n \\\"error\\\": {\\n \\\"code\\\": 404,\\n \\\"message\\\": \\\"This model models/gemini-2.5-flash is no longer available to new users. Please update your code to use models/gemini-3.8-flash for the latest features and improvements. We recommend you to use the Interactions API (https://ai.google.dev/gemini-api/docs/get-started).\\\",\\n \\\"status\\\": \\\"NOT_FOUND\\\"\\n }\\n}\\n\",\"code\":404,\"status\":\"\"}}\n at throwErrorIfNotOK (file:///Users/fixture-user/.npm-global/lib/node_modules/@google/gemini-cli/bundle/chunk-NR7L3FXV.js:267903:24)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async file:///Users/fixture-user/.npm-global/lib/node_modules/@google/gemini-cli/bundle/chunk-NR7L3FXV.js:267666:7\n at async Models.generateContentStream (file:///Users/fixture-user/.npm-global/lib/node_modules/@google/gemini-cli/bundle/chunk-NR7L3FXV.js:268762:16)\n at async file:///Users/fixture-user/.npm-global/lib/node_modules/@google/gemini-cli/bundle/chunk-NR7L3FXV.js:312364:19" + } + } + ] + } + ], + "status": { + "message": "{\"error\":{\"message\":\"{\\n \\\"error\\\": {\\n \\\"code\\\": 404,\\n \\\"message\\\": \\\"This model models/gemini-2.5-flash is no longer available to new users. Please update your code to use models/gemini-3.8-flash for the latest features and improvements. We recommend you to use the Interactions API (https://ai.google.dev/gemini-api/docs/get-started).\\\",\\n \\\"status\\\": \\\"NOT_FOUND\\\"\\n }\\n}\\n\",\"code\":404,\"status\":\"\"}}", + "code": 2 + } + }, + { + "traceId": "5a1e0f3c2b9d4e8f8a7b6c5d4e3f2a11", + "spanId": "9b8a7c6d5e4f3a22", + "flags": 257, + "name": "tool_call", + "kind": 1, + "startTimeUnixNano": "1792027811000000000", + "endTimeUnixNano": "1792027811012000000", + "attributes": [ + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "read_file" + } + }, + { + "key": "gen_ai.tool.call_id", + "value": { + "stringValue": "read_file__call_131990" + } + }, + { + "key": "gen_ai.tool.description", + "value": { + "stringValue": "Reads and returns the content of a specified file. " + } + }, + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.input.messages\"}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.output.messages\"}]" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "tool_call" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power " + } + } + ], + "events": [ + { + "timeUnixNano": "1792027811012000000", + "name": "exception", + "attributes": [ + { + "key": "exception.type", + "value": { + "stringValue": "ENOENT" + } + }, + { + "key": "exception.message", + "value": { + "stringValue": "ENOENT: no such file or directory, open '/Users/fixture-user/project/FIXTURE-CONTENT-MARKER.txt'" + } + }, + { + "key": "exception.stacktrace", + "value": { + "stringValue": "Error: ENOENT: no such file or directory, open '/Users/fixture-user/project/FIXTURE-CONTENT-MARKER.txt'\n at async open (node:internal/fs/promises:639:25)\n at async file:///Users/fixture-user/.npm-global/lib/node_modules/@google/gemini-cli/bundle/chunk-J7SXW2FI.js:316160:7" + } + } + ] + } + ], + "status": { + "message": "ENOENT: no such file or directory, open '/Users/fixture-user/project/FIXTURE-CONTENT-MARKER.txt'", + "code": 2 + } } ] } diff --git a/examples/coding-agents/gemini-cli/tests/test_collector.py b/examples/coding-agents/gemini-cli/tests/test_collector.py index ed2fc4b45..9993cfcfa 100644 --- a/examples/coding-agents/gemini-cli/tests/test_collector.py +++ b/examples/coding-agents/gemini-cli/tests/test_collector.py @@ -2,9 +2,13 @@ The input is spans recorded from Gemini CLI 0.64.0 with content capture on (``logPrompts: true``), with the content values replaced by a marker -(tests/fixtures/gemini-cli-0.64.0-traces.json). The Future AGI exporter is -swapped for a file exporter (tests/collector-test-overrides.yaml); everything -else is the shipped config. +(tests/fixtures/gemini-cli-0.64.0-traces.json). Two error spans are appended: +an ``llm_call`` that failed with a Gemini API error (status message, exception +event and stack trace from a real 0.64.0 run, install path rewritten to +/Users/fixture-user) and a synthetic ``tool_call`` that threw an fs error with +a local path in its message. The Future AGI exporter is swapped for a file +exporter (tests/collector-test-overrides.yaml); everything else is the shipped +config. OTELCOL_BIN=/path/to/otelcol-contrib python3 -m unittest discover -s tests @@ -33,6 +37,8 @@ LOCAL_IDENTITY = ("process.command_args", "process.command", "process.executable.path", "process.executable.name", "process.owner", "host.name", "host.id") OTELCOL = os.environ.get("OTELCOL_BIN") or shutil.which("otelcol-contrib") +CONTENT_KEYS = ("gen_ai.input.messages", "gen_ai.output.messages", "input.value", + "output.value", "gen_ai.system_instructions", "gen_ai.tool.definitions") def free_port() -> int: @@ -41,6 +47,16 @@ def free_port() -> int: return s.getsockname()[1] +class ShippedConfigTest(unittest.TestCase): + def test_collector_metrics_off_and_uncompressed(self): + text = CONFIG.read_text() + # Port 8888 (the collector's own metrics) collides with Jupyter or a + # second collector; Future AGI answers gzip with HTTP 400. + self.assertRegex(text, r"(?m)^service:\n telemetry:\n metrics:\n level: none$") + self.assertRegex(text, r"(?m)^ compression: none$") + self.assertNotIn("level: none", OVERRIDES.read_text()) + + def attrs(items): out = {} for a in items or []: @@ -94,13 +110,19 @@ def spans(self, requests): result.append((resource, span["name"], attrs(span["attributes"]))) return result + def raw_spans(self, requests): + return {span["spanId"]: span for req in requests for rs in req["resourceSpans"] + for ss in rs["scopeSpans"] for span in ss["spans"]} + def test_default_strips_content_and_labels_kinds(self): text, requests = self.run_collector() self.assertNotIn(MARKER, text) self.assertNotIn("/Users/", text) self.assertNotIn("fixture-user", text) + self.assertNotIn("file://", text) + self.assertNotIn("NOT_FOUND", text) # the API error body spans = self.spans(requests) - self.assertEqual(len(spans), 5) + self.assertEqual(len(spans), 7) kinds = {} for resource, name, a in spans: self.assertEqual(resource["project_name"], "gemini-cli-test") @@ -108,17 +130,30 @@ def test_default_strips_content_and_labels_kinds(self): self.assertNotIn(key, resource) self.assertEqual(resource["host.arch"], "arm64") self.assertEqual(resource["process.runtime.name"], "nodejs") - for key in ("gen_ai.input.messages", "gen_ai.output.messages", "input.value", - "output.value", "gen_ai.system_instructions", "gen_ai.tool.definitions"): + for key in CONTENT_KEYS: self.assertNotIn(key, a) self.assertEqual(a["gen_ai.conversation.id"], "fixture-session-0001") self.assertEqual(a["session.id"], "fixture-session-0001") kinds.setdefault(name, set()).add(a.get("gen_ai.span.kind")) self.assertEqual(kinds, {"llm_call": {"LLM"}, "list_directory": {"TOOL"}, "read_file": {"TOOL"}, "schedule_tool_calls": {"CHAIN"}}) - llm = [a for _, name, a in spans if name == "llm_call"] - self.assertTrue(all(a["gen_ai.request.model"] == "gemini-3.8-flash" for a in llm)) - self.assertTrue(all(int(a["gen_ai.usage.input_tokens"]) > 0 for a in llm)) + raw = self.raw_spans(requests) + ok_llm = [attrs(s["attributes"]) for s in raw.values() + if s["name"] == "llm_call" and s["status"].get("code") == 1] + self.assertEqual(len(ok_llm), 2) + self.assertTrue(all(a["gen_ai.request.model"] == "gemini-3.8-flash" for a in ok_llm)) + self.assertTrue(all(int(a["gen_ai.usage.input_tokens"]) > 0 for a in ok_llm)) + # Errors stay ERROR; only the error class (plus the HTTP status of a + # Gemini API error) is kept, not the message or the stack trace. + api_error, tool_error = raw["9b8a7c6d5e4f3a21"], raw["9b8a7c6d5e4f3a22"] + self.assertEqual(api_error["status"], {"code": 2, "message": "ApiError HTTP 404"}) + self.assertEqual(tool_error["status"], {"code": 2, "message": "ENOENT"}) + for span, error_type in ((api_error, "ApiError"), (tool_error, "ENOENT")): + self.assertEqual([(e["name"], attrs(e["attributes"])) for e in span["events"]], + [("exception", {"exception.type": error_type})]) + for span in raw.values(): + if span["status"].get("code") != 2: + self.assertFalse(span["status"].get("message")) def test_opt_in_pipeline_keeps_messages(self): # The documented opt-in: drop transform/strip-content from the pipeline. @@ -128,10 +163,24 @@ def test_opt_in_pipeline_keeps_messages(self): self.assertIn(MARKER + " gen_ai.output.messages", text) for _, _, a in self.spans(requests): self.assertIn("gen_ai.input.messages", a["input.value"]) - self.assertIn("gen_ai.output.messages", a["output.value"]) - # The command line, local paths, user and machine name are removed either way. + if "gen_ai.output.messages" in a: # a failed model call has no output + self.assertIn("gen_ai.output.messages", a["output.value"]) + # Error text is content: kept with the opt-in. Stack traces are not + # (install paths only), with or without it. + raw = self.raw_spans(requests) + self.assertIn("NOT_FOUND", raw["9b8a7c6d5e4f3a21"]["status"]["message"]) + self.assertIn("FIXTURE-CONTENT-MARKER.txt", raw["9b8a7c6d5e4f3a22"]["status"]["message"]) + event = attrs(raw["9b8a7c6d5e4f3a22"]["events"][0]["attributes"]) + self.assertEqual(sorted(event), ["exception.message", "exception.type"]) + self.assertNotIn("file://", text) + # The command line, executable paths, user and machine name are removed + # either way. (The opted-in tool error message above carries a path.) self.assertNotIn("prompt on the command line", text) - self.assertNotIn("fixture-user", text) + self.assertNotIn("fixture-users-laptop", text) + self.assertNotIn(".nvm", text) + for resource, _, _ in self.spans(requests): + for key in LOCAL_IDENTITY: + self.assertNotIn(key, resource) # The opt-in exports the system prompt and tool schemas (documented). self.assertIn(MARKER + " gen_ai.system_instructions", text) diff --git a/src/pages/docs/integrations/coding-agents/gemini-cli.mdx b/src/pages/docs/integrations/coding-agents/gemini-cli.mdx index e21074926..62c2d82cc 100644 --- a/src/pages/docs/integrations/coding-agents/gemini-cli.mdx +++ b/src/pages/docs/integrations/coding-agents/gemini-cli.mdx @@ -1,6 +1,6 @@ --- title: "Gemini CLI session tracing with Future AGI" -description: "Trace Gemini CLI sessions in Future AGI with Gemini CLI's built-in OpenTelemetry export and a local OpenTelemetry Collector: model calls with tokens, tool calls, errors, grouped by session. Prompt and tool content stay off by default." +description: "Trace Gemini CLI sessions in Future AGI with Gemini CLI's built-in OpenTelemetry export and a local OpenTelemetry Collector: model calls with tokens, tool calls, errors, grouped by session. Prompt, tool and error content stay off by default." --- ## About @@ -10,7 +10,7 @@ Gemini CLI has [built-in OpenTelemetry](https://geminicli.com/docs/cli/telemetry ``` Gemini CLI ──OTLP/HTTP──▶ otelcol-contrib (localhost:4318) ──▶ Future AGI keeps traces, drops logs and metrics, - strips prompt-shaped attributes, + strips prompt-shaped attributes and error text, labels span kinds, adds your keys ``` @@ -28,7 +28,7 @@ Gemini CLI does not emit a root span for a session or a turn, so each model call This is not the same as: -- **The Google GenAI SDK instrumentor** (`traceAI-google-genai`), which traces your own code that calls the Gemini API. It does not see Gemini CLI's tool scheduler or session. +- **The [Google GenAI](/docs/integrations/traceai/google_genai) and [Vertex AI](/docs/integrations/traceai/vertexai) SDK instrumentors** (`traceAI-google-genai`, `traceAI-vertexai`), which trace your own code that calls the Gemini API. They do not see Gemini CLI's tool scheduler or session. - **Google Antigravity** or Gemini in other IDEs. Those are separate integrations. - **Gemini CLI's `target: "gcp"` export**, which sends telemetry to Google Cloud, not to Future AGI. @@ -44,14 +44,14 @@ This is not the same as: Download the example collector config and start `otelcol-contrib` with your Future AGI keys: ```bash -git clone --depth 1 https://github.com/future-agi/docs.git fi-docs +curl -fsSLO https://raw.githubusercontent.com/future-agi/docs/main/examples/coding-agents/gemini-cli/otel-collector.yaml export FI_API_KEY=your-futureagi-api-key export FI_SECRET_KEY=your-futureagi-secret-key export FI_PROJECT_NAME=gemini-cli -otelcol-contrib --config fi-docs/examples/coding-agents/gemini-cli/otel-collector.yaml +otelcol-contrib --config otel-collector.yaml ``` -It listens on `127.0.0.1:4318` (set `GEMINI_OTLP_LISTEN` to change it) and sends traces to `$FI_BASE_URL/tracer/v1/traces` (default `https://api.futureagi.com`; set `FI_BASE_URL` for a self-hosted Future AGI). Keep it running while you use Gemini CLI, or run it as a service. +It listens on `127.0.0.1:4318` (set `GEMINI_OTLP_LISTEN` to change it) and sends traces to `$FI_BASE_URL/tracer/v1/traces` (default `https://api.futureagi.com`; set `FI_BASE_URL` for a self-hosted Future AGI). Keep it running while you use Gemini CLI, or run it as a service. The collector's own metrics endpoint is turned off in the config (`service.telemetry.metrics.level: none`), so it does not take port 8888 from Jupyter or another collector. The collector does four things Gemini CLI cannot do on its own: @@ -59,7 +59,7 @@ The collector does four things Gemini CLI cannot do on its own: | --- | --- | | Adds `X-Api-Key`, `X-Secret-Key` and `project_name` | Gemini CLI 0.64.0 has no setting for OTLP headers. (Gemini CLI's main-branch docs list an `otlpHeaders` setting that is not in 0.64.0; the other three steps still need the collector.) | | Drops logs and metrics | Gemini CLI always exports all three signals; Future AGI ingests traces. | -| Deletes `gen_ai.system_instructions`, `gen_ai.tool.definitions`, message attributes, and the command line, executable paths, OS user name and machine name that Node's resource detectors add | Privacy default, see [Privacy](#privacy). | +| Deletes `gen_ai.system_instructions`, `gen_ai.tool.definitions`, message attributes, error messages and stack traces, and the command line, executable paths, OS user name and machine name that Node's resource detectors add | Privacy default, see [Privacy](#privacy). | | Sets `gen_ai.span.kind` from Gemini CLI's operation names, names tool spans after the tool, copies the session id to `session.id` | Gemini CLI uses its own operation names (`llm_call`, `tool_call`), which Future AGI does not map by itself. | It also sends uncompressed: the Future AGI collector answers gzip-encoded OTLP requests with HTTP 400. @@ -81,11 +81,11 @@ Add the `telemetry` block to `~/.gemini/settings.json` (all projects) or `.gemin } ``` -- `traces: true` is required. With it off, Gemini CLI still emits the spans but drops every attribute except the operation name, the agent name and the session id: no model, no tokens, no tool names. +- `traces: true` is required for token counts. With it off, Gemini CLI still emits the spans with the model, the tool names, the system prompt and the tool schemas, but without token counts. It is not a privacy setting on its own: keep `transform/strip-content` in the collector either way. - `logPrompts: false` is not Gemini CLI's default. The vendor default is `true`, which puts prompts, responses, tool arguments and tool results on the spans. Keep it `false` unless you opt in to content. - Do not set `outfile` (it replaces the OTLP export) or `target: "gcp"`. -The same settings exist as environment variables (`GEMINI_TELEMETRY_ENABLED`, `GEMINI_TELEMETRY_TRACES_ENABLED`, `GEMINI_TELEMETRY_TARGET`, `GEMINI_TELEMETRY_OTLP_ENDPOINT`, `GEMINI_TELEMETRY_OTLP_PROTOCOL`, `GEMINI_TELEMETRY_LOG_PROMPTS`), which override the file. +Gemini CLI also documents environment variables for these settings (`GEMINI_TELEMETRY_ENABLED`, `GEMINI_TELEMETRY_TRACES_ENABLED`, `GEMINI_TELEMETRY_TARGET`, `GEMINI_TELEMETRY_OTLP_ENDPOINT`, `GEMINI_TELEMETRY_OTLP_PROTOCOL`, `GEMINI_TELEMETRY_LOG_PROMPTS`). Prefer the settings file: in our 0.64.0 test, `GEMINI_TELEMETRY_ENABLED=false` did not stop the export when the settings file had `"enabled": true`. Setting `"enabled": false` in the file did: no spans reached the collector. Optionally also export `OTEL_NODE_RESOURCE_DETECTORS=env` in the shell that runs `gemini`. Node's default resource detectors record the full command line (for `gemini -p "..."` it contains the prompt), the Node executable path (often under your home directory), your OS user name and the machine name. The collector deletes all of these either way (`process.command_args`, `process.command`, `process.executable.*`, `process.owner`, `host.name`, `host.id`); this setting keeps them from leaving the Gemini CLI process at all. @@ -95,26 +95,32 @@ Start `gemini` in a project and send one prompt that uses a tool, for example *" Open **Observe** in Future AGI and select the project (`gemini-cli` above). The **Trace** tab lists `llm_call` and `schedule_tool_calls` traces. The **Sessions** tab groups them by Gemini CLI session. -`run_e2e.sh` in the examples folder does all three steps against a throwaway `HOME`, so it does not touch your own `~/.gemini`: +`run_e2e.sh` in the [examples folder](https://github.com/future-agi/docs/tree/main/examples/coding-agents/gemini-cli) does all three steps against a throwaway `HOME`, so it does not touch your own `~/.gemini`. It also checks what the collector forwarded: the prompt's marker string, your user name, home directory and machine name must not be in it (with `CAPTURE_CONTENT=1`, the marker must be). Its collector listens on port 24318 (`GEMINI_OTLP_PORT`): ```bash +git clone --depth 1 --filter=blob:none --sparse https://github.com/future-agi/docs.git fi-docs +git -C fi-docs sparse-checkout set examples/coding-agents/gemini-cli FI_API_KEY=... FI_SECRET_KEY=... GEMINI_API_KEY=... \ fi-docs/examples/coding-agents/gemini-cli/run_e2e.sh ``` +`tests/test_collector.py` in the same folder runs the shipped config in a real `otelcol-contrib` and checks what it forwards; it is skipped when no `otelcol-contrib` binary is found (set `OTELCOL_BIN`). + ## What is captured | Gemini CLI span | Future AGI span kind | Fields | | --- | --- | --- | -| `llm_call` | `LLM` | `gen_ai.request.model`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.prompt.name` (Gemini CLI's prompt id), status `ERROR` when the API call fails (a retried 503 shows up as its own `ERROR` span) | +| `llm_call` | `LLM` | `gen_ai.request.model`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.prompt.name` (Gemini CLI's prompt id), status `ERROR` when the API call fails, with the error class and HTTP status as the status message (for example `ApiError HTTP 503`; a retried 503 shows up as its own `ERROR` span) | | `schedule_tool_calls` | `CHAIN` | parent of the tool calls the model requested in one response | -| `tool_call`, renamed to the tool (`read_file`, `list_directory`, `run_shell_command`, ...) | `TOOL` | `gen_ai.tool.name`, `gen_ai.tool.call_id`, `gen_ai.tool.description`, duration, status | -| `agent_call` | `AGENT` | subagent name and description. Present in Gemini CLI 0.64.0's source; not exercised in our test | +| `tool_call`, renamed to the tool (`read_file`, `list_directory`, `run_shell_command`, ...) | `TOOL` | `gen_ai.tool.name`, `gen_ai.tool.call_id`, `gen_ai.tool.description`, duration. Status `ERROR` (status message: the error class, for example `ENOENT`) only when the tool throws; see [What is not available](#what-is-not-available) | +| `agent_call` | `AGENT` | a subagent run; its model and tool spans nest under it. Gemini CLI 0.64.0 overwrites the subagent's name with `gen_ai.agent.name=gemini-cli`, so the subagent's own name is not available. Read from Gemini CLI 0.64.0's source; not exercised in our test | | every span | | `gen_ai.conversation.id` and `session.id` = Gemini CLI session id; resource `service.name=gemini-cli` | ## What is not available - **One trace per session or turn.** Gemini CLI has no session or turn span; use the **Sessions** tab. +- **Tool failures the tool reports itself.** A tool that returns an error instead of throwing (file not found, path outside the workspace, a shell execution error) ends its span with status `OK` in Gemini CLI 0.64.0. Only an exception thrown while the tool runs is marked `ERROR`. The failure is visible only in the tool result, which is exported only with the [content opt-in](#privacy). +- **Error messages and stack traces** by default. The collector keeps the `ERROR` status and the error class (plus the HTTP status of a Gemini API error) and removes the text; see [Privacy](#privacy). - **Cost.** Gemini CLI reports tokens, not cost. Future AGI computes cost from the model name only when it has pricing for that model. - **Cached, thinking and tool-use token counts.** Only input and output tokens are on the spans. - **Gemini CLI logs and metrics** (`gemini_cli.user_prompt`, `gemini_cli.api_request`, token and latency metrics). The collector drops them; Future AGI ingests traces only. @@ -122,10 +128,10 @@ FI_API_KEY=... FI_SECRET_KEY=... GEMINI_API_KEY=... \ ## Privacy -With the settings above, only structure and metadata leave your machine: model, token counts, tool names, call ids, tool descriptions (static text from Gemini CLI), durations, statuses, and the session id. +With the settings above, only structure and metadata leave your machine: model, token counts, tool names, call ids, tool descriptions (static text from Gemini CLI or your MCP servers), durations, statuses, error classes (`ApiError HTTP 429`, `ENOENT`), and the session id. -- **Not exported by default:** prompts and responses, tool arguments and results (Gemini CLI leaves them out with `logPrompts: false`), the system prompt, which includes your `GEMINI.md` memory and local paths, and the tool schemas (the collector deletes them), and the command line, executable paths, OS user name and machine name. -- **Never forwarded:** Gemini CLI's logs and metrics. +- **Not exported by default:** prompts and responses, tool arguments and results (Gemini CLI leaves them out with `logPrompts: false`); the system prompt, which includes your `GEMINI.md` memory and local paths, the tool schemas, and error messages, which carry API error bodies and, for a tool that throws, file paths and command output (the collector deletes them or replaces them with the error class); the command line, executable paths, OS user name and machine name. +- **Never forwarded:** Gemini CLI's logs and metrics, and error stack traces (they list your Gemini CLI install path, including your home directory; the collector deletes them even with the content opt-in). To include content: @@ -136,16 +142,21 @@ To include content: processors: [transform/gemini-cli, batch] ``` -Model spans then carry the request and response messages, and tool spans the call request and result, as `gen_ai.input.messages` / `gen_ai.output.messages`; the collector also copies them to the span's input and output in Future AGI. Gemini CLI truncates each value to 10,000 characters. The system prompt (with your `GEMINI.md` memory and local paths such as the working directory) and the tool schemas are exported too in this mode. The command line, paths, user and machine name stay removed. +Model spans then carry the request and response messages, and tool spans the call request and result, as `gen_ai.input.messages` / `gen_ai.output.messages`; the collector also copies them to the span's input and output in Future AGI. Gemini CLI truncates each value to 10,000 characters. The system prompt (with your `GEMINI.md` memory and local paths such as the working directory), the tool schemas and the full error messages (`exception.message` and the status message, which can contain file paths) are exported too in this mode. The command line, executable paths, user and machine name, and stack traces stay removed. ## Troubleshooting **No traces appear.** 1. Check that the collector is running and listening on the `otlpEndpoint` port. With `otlpProtocol: "http"` Gemini CLI posts to `/v1/traces`; the default protocol is `grpc`, which this collector config does not accept. -2. Check the collector's output for `Exporting failed`. `401`/`403` means wrong keys. `400` means a request the Future AGI collector could not decode: keep `compression: none` on the exporter. -3. `telemetry.enabled` must be `true`. `useCollector` together with `useCliAuth` disables telemetry. +2. If the collector exits with `address already in use`, another process owns port 4318 (often another OpenTelemetry Collector or agent). Start this one on a free port, for example `GEMINI_OTLP_LISTEN=127.0.0.1:24318`, and set the same port in `otlpEndpoint` (`http://127.0.0.1:24318`). +3. Check the collector's output for `Exporting failed`. `401`/`403` means wrong keys. `400` means a request the Future AGI collector could not decode: keep `compression: none` on the exporter. +4. `telemetry.enabled` must be `true` in the settings file. `useCollector` together with `useCliAuth` disables telemetry. + +**Model spans have no token counts.** `telemetry.traces` is `false` in the settings file. + +**A failed tool call shows status `OK`.** Expected in Gemini CLI 0.64.0 when the tool reports the failure in its result instead of throwing; see [What is not available](#what-is-not-available). -**Spans have no model, tokens or tool names.** `telemetry.traces` is `false` (or `GEMINI_TELEMETRY_TRACES_ENABLED` overrides it). +**Error spans show only `ApiError HTTP 503` or `error`.** The collector removes error text by default. Remove `transform/strip-content` (the content opt-in) to keep full error messages. **Tool spans show as `unknown` and are named `tool_call`.** The spans did not go through the `transform/gemini-cli` processor; send Gemini CLI's export to the collector, not straight to Future AGI. From 46e09d577f7023873afb36e860d459e8f0084e3e Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Thu, 15 Oct 2026 11:47:11 -0700 Subject: [PATCH 4/7] fix(gemini-cli): map interactive user_prompt/system_prompt turn spans (TH-8285) Gemini CLI 0.64.0 interactive mode wraps every turn in a user_prompt root span and the tool-result follow-up in a system_prompt child span. The collector now labels both CHAIN (same as schedule_tool_calls and the hook-to-otlp turn span). The contract test fixture gains a real user_prompt/system_prompt pair from an interactive session. The page now describes interactive mode (one trace per prompt, queued prompts nest under the running turn, slash commands give an empty user_prompt trace) and -p mode (no turn span) separately. Gemini CLI records no content on the turn spans, also with logPrompts true. run_e2e.sh fails when another process owns the port and when the collector forwarded no spans, instead of passing on an empty capture. Co-Authored-By: Claude Opus 5.5 --- .../gemini-cli/otel-collector.yaml | 9 ++- examples/coding-agents/gemini-cli/run_e2e.sh | 29 +++++-- .../fixtures/gemini-cli-0.64.0-traces.json | 77 +++++++++++++++++++ .../gemini-cli/tests/test_collector.py | 30 ++++++-- .../integrations/coding-agents/gemini-cli.mdx | 48 ++++++++---- 5 files changed, 163 insertions(+), 30 deletions(-) diff --git a/examples/coding-agents/gemini-cli/otel-collector.yaml b/examples/coding-agents/gemini-cli/otel-collector.yaml index 4b22e34a2..a7780f767 100644 --- a/examples/coding-agents/gemini-cli/otel-collector.yaml +++ b/examples/coding-agents/gemini-cli/otel-collector.yaml @@ -41,12 +41,17 @@ processors: - delete_key(attributes, "host.id") - context: span statements: - # Gemini CLI names its operations llm_call, tool_call, agent_call and - # schedule_tool_calls. Map them to the span kinds Future AGI uses. + # Gemini CLI names its operations llm_call, tool_call, agent_call, + # schedule_tool_calls, and in interactive mode user_prompt (one turn, + # the root of its model and tool spans) and system_prompt (the + # follow-up turn that sends tool results back to the model). Map them + # to the span kinds Future AGI uses. - set(attributes["gen_ai.span.kind"], "LLM") where attributes["gen_ai.operation.name"] == "llm_call" - set(attributes["gen_ai.span.kind"], "TOOL") where attributes["gen_ai.operation.name"] == "tool_call" - set(attributes["gen_ai.span.kind"], "AGENT") where attributes["gen_ai.operation.name"] == "agent_call" - set(attributes["gen_ai.span.kind"], "CHAIN") where attributes["gen_ai.operation.name"] == "schedule_tool_calls" + - set(attributes["gen_ai.span.kind"], "CHAIN") where attributes["gen_ai.operation.name"] == "user_prompt" + - set(attributes["gen_ai.span.kind"], "CHAIN") where attributes["gen_ai.operation.name"] == "system_prompt" # Name tool spans after the tool instead of "tool_call". - set(name, attributes["gen_ai.tool.name"]) where name == "tool_call" and attributes["gen_ai.tool.name"] != nil # Every span carries the Gemini CLI session id; as session.id it groups diff --git a/examples/coding-agents/gemini-cli/run_e2e.sh b/examples/coding-agents/gemini-cli/run_e2e.sh index cafc981a8..6a315ee56 100755 --- a/examples/coding-agents/gemini-cli/run_e2e.sh +++ b/examples/coding-agents/gemini-cli/run_e2e.sh @@ -15,8 +15,11 @@ # collector also writes what it forwards to a local file, which the script # checks: by default the marker string from the prompt, your user name, home # directory and machine name must not be in it; with CAPTURE_CONTENT=1 the -# marker must be. The script exits non-zero if a check fails and then keeps its -# work directory for inspection; on success it deletes it. +# marker must be. It also fails if the port is already taken by another process +# or the collector forwarded no spans. The script exits non-zero if a check +# fails and then keeps its work directory for inspection; on success it deletes +# it. It runs Gemini CLI non-interactively (-p), so there is no user_prompt turn +# span; see the page for the interactive trace shape. set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" @@ -28,6 +31,13 @@ PORT="${GEMINI_OTLP_PORT:-24318}" export FI_PROJECT_NAME="${FI_PROJECT_NAME:-gemini-cli}" export GEMINI_OTLP_LISTEN="127.0.0.1:$PORT" +# Another process on the port would receive Gemini CLI's spans instead of this +# collector, and the checks below would run on an empty capture. +if (exec 3<>"/dev/tcp/127.0.0.1/$PORT") 2>/dev/null; then + echo "FAIL: port $PORT is already in use by another process; set GEMINI_OTLP_PORT to a free port" >&2 + exit 1 +fi + WORK="$(mktemp -d)" fail() { echo "FAIL: $*" >&2 @@ -52,8 +62,11 @@ for _ in $(seq 1 50); do kill -0 "$COLLECTOR_PID" 2>/dev/null || break sleep 0.2 done -if [ "$listening" != 1 ]; then +if [ "$listening" != 1 ] || ! kill -0 "$COLLECTOR_PID" 2>/dev/null; then tail -5 "$WORK/collector.log" >&2 + if grep -q "address already in use" "$WORK/collector.log"; then + fail "port $PORT is already in use by another process; set GEMINI_OTLP_PORT to a free port" + fi fail "the collector is not listening on 127.0.0.1:$PORT (see $WORK/collector.log)" fi @@ -93,13 +106,17 @@ echo "session_id=$SESSION_ID" echo "marker=$MARKER" echo "project=$FI_PROJECT_NAME" touch "$WORK/forwarded.json" -python3 - "$WORK/forwarded.json" <<'EOF' +SPANS_FORWARDED="$(python3 - "$WORK/forwarded.json" <<'EOF' import json, sys spans = [s for line in open(sys.argv[1]) if line.strip() for rs in json.loads(line)["resourceSpans"] for ss in rs["scopeSpans"] for s in ss["spans"]] -print("spans_forwarded=%d" % len(spans)) -print("span_names=%s" % ",".join(sorted({s["name"] for s in spans}))) +print(len(spans)) +print("span_names=%s" % ",".join(sorted({s["name"] for s in spans})), file=sys.stderr) EOF +)" +echo "spans_forwarded=$SPANS_FORWARDED" +# Zero spans would make the privacy checks below pass on nothing. +[ "$SPANS_FORWARDED" -gt 0 ] || fail "the collector forwarded no spans (see $WORK/collector.log)" if [ "$LOG_PROMPTS" = true ]; then grep -q -F "$MARKER" "$WORK/forwarded.json" || fail "content opt-in: the marker was not forwarded" echo "privacy: content opt-in, marker forwarded" diff --git a/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json b/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json index 12e517cc8..ae72b1789 100644 --- a/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json +++ b/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json @@ -642,6 +642,83 @@ "message": "ENOENT: no such file or directory, open '/Users/fixture-user/project/FIXTURE-CONTENT-MARKER.txt'", "code": 2 } + }, + { + "traceId": "7c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f", + "spanId": "6e5f4a3b2c1d0e01", + "flags": 257, + "name": "user_prompt", + "kind": 1, + "startTimeUnixNano": "1792027900000000000", + "endTimeUnixNano": "1792028067923258250", + "attributes": [ + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "user_prompt" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power of Gemini directly into your terminal. It is designed to be a terminal-first, extensible, and powerful tool for developers, engineers, SREs, and beyond." + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "7c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f", + "spanId": "6e5f4a3b2c1d0e02", + "parentSpanId": "6e5f4a3b2c1d0e01", + "flags": 257, + "name": "system_prompt", + "kind": 1, + "startTimeUnixNano": "1792028067923000000", + "endTimeUnixNano": "1792028181319740708", + "attributes": [ + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "system_prompt" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power of Gemini directly into your terminal. It is designed to be a terminal-first, extensible, and powerful tool for developers, engineers, SREs, and beyond." + } + } + ], + "status": { + "code": 1 + } } ] } diff --git a/examples/coding-agents/gemini-cli/tests/test_collector.py b/examples/coding-agents/gemini-cli/tests/test_collector.py index 9993cfcfa..c09493da3 100644 --- a/examples/coding-agents/gemini-cli/tests/test_collector.py +++ b/examples/coding-agents/gemini-cli/tests/test_collector.py @@ -2,11 +2,14 @@ The input is spans recorded from Gemini CLI 0.64.0 with content capture on (``logPrompts: true``), with the content values replaced by a marker -(tests/fixtures/gemini-cli-0.64.0-traces.json). Two error spans are appended: -an ``llm_call`` that failed with a Gemini API error (status message, exception -event and stack trace from a real 0.64.0 run, install path rewritten to -/Users/fixture-user) and a synthetic ``tool_call`` that threw an fs error with -a local path in its message. The Future AGI exporter is swapped for a file +(tests/fixtures/gemini-cli-0.64.0-traces.json). Appended to that: two error +spans, an ``llm_call`` that failed with a Gemini API error (status message, +exception event and stack trace from a real 0.64.0 run, install path rewritten +to /Users/fixture-user) and a synthetic ``tool_call`` that threw an fs error +with a local path in its message; and the ``user_prompt`` turn span with its +``system_prompt`` follow-up child from a real interactive 0.64.0 session +(session id replaced). Gemini CLI puts no content on those two even with +``logPrompts: true``. The Future AGI exporter is swapped for a file exporter (tests/collector-test-overrides.yaml); everything else is the shipped config. @@ -122,7 +125,7 @@ def test_default_strips_content_and_labels_kinds(self): self.assertNotIn("file://", text) self.assertNotIn("NOT_FOUND", text) # the API error body spans = self.spans(requests) - self.assertEqual(len(spans), 7) + self.assertEqual(len(spans), 9) kinds = {} for resource, name, a in spans: self.assertEqual(resource["project_name"], "gemini-cli-test") @@ -135,9 +138,15 @@ def test_default_strips_content_and_labels_kinds(self): self.assertEqual(a["gen_ai.conversation.id"], "fixture-session-0001") self.assertEqual(a["session.id"], "fixture-session-0001") kinds.setdefault(name, set()).add(a.get("gen_ai.span.kind")) + # user_prompt / system_prompt: interactive mode's turn spans. self.assertEqual(kinds, {"llm_call": {"LLM"}, "list_directory": {"TOOL"}, - "read_file": {"TOOL"}, "schedule_tool_calls": {"CHAIN"}}) + "read_file": {"TOOL"}, "schedule_tool_calls": {"CHAIN"}, + "user_prompt": {"CHAIN"}, "system_prompt": {"CHAIN"}}) raw = self.raw_spans(requests) + turn, follow_up = raw["6e5f4a3b2c1d0e01"], raw["6e5f4a3b2c1d0e02"] + self.assertEqual((turn["name"], follow_up["name"]), ("user_prompt", "system_prompt")) + self.assertFalse(turn.get("parentSpanId")) + self.assertEqual(follow_up["parentSpanId"], turn["spanId"]) ok_llm = [attrs(s["attributes"]) for s in raw.values() if s["name"] == "llm_call" and s["status"].get("code") == 1] self.assertEqual(len(ok_llm), 2) @@ -161,7 +170,12 @@ def test_opt_in_pipeline_keeps_messages(self): "service.pipelines.traces.processors=[transform/gemini-cli,batch]") self.assertIn(MARKER + " gen_ai.input.messages", text) self.assertIn(MARKER + " gen_ai.output.messages", text) - for _, _, a in self.spans(requests): + for _, name, a in self.spans(requests): + if name in ("user_prompt", "system_prompt"): + # Gemini CLI 0.64.0 records no content on its turn spans. + self.assertEqual(a["gen_ai.span.kind"], "CHAIN") + self.assertNotIn("input.value", a) + continue self.assertIn("gen_ai.input.messages", a["input.value"]) if "gen_ai.output.messages" in a: # a failed model call has no output self.assertIn("gen_ai.output.messages", a["output.value"]) diff --git a/src/pages/docs/integrations/coding-agents/gemini-cli.mdx b/src/pages/docs/integrations/coding-agents/gemini-cli.mdx index 62c2d82cc..1bab51ecf 100644 --- a/src/pages/docs/integrations/coding-agents/gemini-cli.mdx +++ b/src/pages/docs/integrations/coding-agents/gemini-cli.mdx @@ -1,11 +1,11 @@ --- title: "Gemini CLI session tracing with Future AGI" -description: "Trace Gemini CLI sessions in Future AGI with Gemini CLI's built-in OpenTelemetry export and a local OpenTelemetry Collector: model calls with tokens, tool calls, errors, grouped by session. Prompt, tool and error content stay off by default." +description: "Trace Gemini CLI sessions in Future AGI with Gemini CLI's built-in OpenTelemetry export and a local OpenTelemetry Collector: turns, model calls with tokens, tool calls and errors, grouped by session. Prompt, tool and error content stay off by default." --- ## About -Gemini CLI has [built-in OpenTelemetry](https://geminicli.com/docs/cli/telemetry/). With traces turned on it emits a span for every model call and every tool call. This page sends those spans through a small OpenTelemetry Collector running next to Gemini CLI, which forwards them to Future AGI: +Gemini CLI has [built-in OpenTelemetry](https://geminicli.com/docs/cli/telemetry/). With traces turned on it emits a span for every model call and every tool call, and in interactive mode a span for every turn. This page sends those spans through a small OpenTelemetry Collector running next to Gemini CLI, which forwards them to Future AGI: ``` Gemini CLI ──OTLP/HTTP──▶ otelcol-contrib (localhost:4318) ──▶ Future AGI @@ -14,17 +14,33 @@ Gemini CLI ──OTLP/HTTP──▶ otelcol-contrib (localhost:4318) ── labels span kinds, adds your keys ``` -One Gemini CLI session shows up in Future AGI as one **session** made of several traces: +One Gemini CLI session shows up in Future AGI as one **session**. How it splits into traces depends on how you run Gemini CLI. +**Interactive `gemini`:** one trace per prompt you type, rooted at a `user_prompt` span: + +``` +user_prompt one turn: the prompt you typed +├── llm_call one model request: model, input/output tokens, status +├── schedule_tool_calls the tool calls the model asked for +│ ├── list_directory tool call: name, call id, duration, status +│ └── read_file +└── system_prompt follow-up turn: Gemini CLI sends the tool results back + └── llm_call the next model request ``` -llm_call one model request: model, input/output tokens, status -schedule_tool_calls one batch of tool calls the model asked for -├── list_directory tool call: name, call id, duration, status + +Two cases in Gemini CLI 0.64.0 differ from that. A prompt you type while a turn is still running is queued and sent when that turn ends; in our test it then nested under the running turn, in the same trace, instead of starting its own. A slash command you type (`/quit`, `/stats`, ...) also starts a `user_prompt` trace, with no spans under it. + +**Non-interactive `gemini -p "..."`:** there is no turn span, so each model call and each tool batch is its own trace: + +``` +llm_call one model request +schedule_tool_calls one batch of tool calls +├── list_directory └── read_file llm_call the next model request ``` -Gemini CLI does not emit a root span for a session or a turn, so each model call and each tool batch is its own trace. Every span carries the Gemini CLI session id, which the collector copies to `session.id`, so the **Sessions** tab groups them. +Gemini CLI has no span for the whole session. Every span carries the Gemini CLI session id, which the collector copies to `session.id`, so the **Sessions** tab groups a session's traces in both modes. This is not the same as: @@ -34,7 +50,7 @@ This is not the same as: | | | | --- | --- | -| Tested with | Gemini CLI 0.64.0 (`@google/gemini-cli`, macOS), OpenTelemetry Collector Contrib 0.162.0 | +| Tested with | Gemini CLI 0.64.0 (`@google/gemini-cli`, macOS; interactive and `-p`), OpenTelemetry Collector Contrib 0.162.0 | | Gemini CLI docs used | [Observability with OpenTelemetry](https://geminicli.com/docs/cli/telemetry/) and [configuration](https://www.geminicli.com/docs/reference/configuration), read 2026-10-14 | | Requires | [`otelcol-contrib`](https://github.com/open-telemetry/opentelemetry-collector-releases/releases) (the contrib build: the config uses the `transform` processor) | | Source | [examples/coding-agents/gemini-cli](https://github.com/future-agi/docs/tree/main/examples/coding-agents/gemini-cli) | @@ -60,7 +76,7 @@ The collector does four things Gemini CLI cannot do on its own: | Adds `X-Api-Key`, `X-Secret-Key` and `project_name` | Gemini CLI 0.64.0 has no setting for OTLP headers. (Gemini CLI's main-branch docs list an `otlpHeaders` setting that is not in 0.64.0; the other three steps still need the collector.) | | Drops logs and metrics | Gemini CLI always exports all three signals; Future AGI ingests traces. | | Deletes `gen_ai.system_instructions`, `gen_ai.tool.definitions`, message attributes, error messages and stack traces, and the command line, executable paths, OS user name and machine name that Node's resource detectors add | Privacy default, see [Privacy](#privacy). | -| Sets `gen_ai.span.kind` from Gemini CLI's operation names, names tool spans after the tool, copies the session id to `session.id` | Gemini CLI uses its own operation names (`llm_call`, `tool_call`), which Future AGI does not map by itself. | +| Sets `gen_ai.span.kind` from Gemini CLI's operation names, names tool spans after the tool, copies the session id to `session.id` | Gemini CLI uses its own operation names (`user_prompt`, `llm_call`, `tool_call`, ...), which Future AGI does not map by itself. | It also sends uncompressed: the Future AGI collector answers gzip-encoded OTLP requests with HTTP 400. @@ -93,9 +109,9 @@ Optionally also export `OTEL_NODE_RESOURCE_DETECTORS=env` in the shell that runs Start `gemini` in a project and send one prompt that uses a tool, for example *"List the files in this folder and read README.md"*. Exit Gemini CLI, or wait a few seconds: spans are batched. -Open **Observe** in Future AGI and select the project (`gemini-cli` above). The **Trace** tab lists `llm_call` and `schedule_tool_calls` traces. The **Sessions** tab groups them by Gemini CLI session. +Open **Observe** in Future AGI and select the project (`gemini-cli` above). For an interactive session the **Trace** tab lists one `user_prompt` trace per prompt; for `gemini -p` it lists separate `llm_call` and `schedule_tool_calls` traces. The **Sessions** tab groups them by Gemini CLI session. -`run_e2e.sh` in the [examples folder](https://github.com/future-agi/docs/tree/main/examples/coding-agents/gemini-cli) does all three steps against a throwaway `HOME`, so it does not touch your own `~/.gemini`. It also checks what the collector forwarded: the prompt's marker string, your user name, home directory and machine name must not be in it (with `CAPTURE_CONTENT=1`, the marker must be). Its collector listens on port 24318 (`GEMINI_OTLP_PORT`): +`run_e2e.sh` in the [examples folder](https://github.com/future-agi/docs/tree/main/examples/coding-agents/gemini-cli) does all three steps against a throwaway `HOME`, so it does not touch your own `~/.gemini`. It runs `gemini -p`, so its traces have no `user_prompt` span. It also checks what the collector forwarded: at least one span, and not the prompt's marker string, your user name, home directory or machine name (with `CAPTURE_CONTENT=1`, the marker must be there). It fails if another process already listens on its port, 24318 (`GEMINI_OTLP_PORT`): ```bash git clone --depth 1 --filter=blob:none --sparse https://github.com/future-agi/docs.git fi-docs @@ -110,6 +126,8 @@ FI_API_KEY=... FI_SECRET_KEY=... GEMINI_API_KEY=... \ | Gemini CLI span | Future AGI span kind | Fields | | --- | --- | --- | +| `user_prompt` (interactive mode only) | `CHAIN` | one turn, the root of its model and tool spans: `gen_ai.agent.name=gemini-cli`, duration, status. No prompt text, even with the content opt-in: Gemini CLI 0.64.0 does not record it on this span (with the opt-in, the prompt is on the first `llm_call`) | +| `system_prompt` (interactive mode only) | `CHAIN` | the follow-up turn Gemini CLI starts itself to send tool results back to the model, a child of `user_prompt`; same fields. It starts when `user_prompt` ends, so a turn's full duration is `user_prompt` plus its `system_prompt` children. Despite the name it does not hold the system prompt (that is `gen_ai.system_instructions` on `llm_call`, deleted by default) | | `llm_call` | `LLM` | `gen_ai.request.model`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.prompt.name` (Gemini CLI's prompt id), status `ERROR` when the API call fails, with the error class and HTTP status as the status message (for example `ApiError HTTP 503`; a retried 503 shows up as its own `ERROR` span) | | `schedule_tool_calls` | `CHAIN` | parent of the tool calls the model requested in one response | | `tool_call`, renamed to the tool (`read_file`, `list_directory`, `run_shell_command`, ...) | `TOOL` | `gen_ai.tool.name`, `gen_ai.tool.call_id`, `gen_ai.tool.description`, duration. Status `ERROR` (status message: the error class, for example `ENOENT`) only when the tool throws; see [What is not available](#what-is-not-available) | @@ -118,7 +136,7 @@ FI_API_KEY=... FI_SECRET_KEY=... GEMINI_API_KEY=... \ ## What is not available -- **One trace per session or turn.** Gemini CLI has no session or turn span; use the **Sessions** tab. +- **One trace per session.** Gemini CLI has no session span: interactive mode gives one trace per turn, `gemini -p` one trace per model call or tool batch. Use the **Sessions** tab for the whole session. - **Tool failures the tool reports itself.** A tool that returns an error instead of throwing (file not found, path outside the workspace, a shell execution error) ends its span with status `OK` in Gemini CLI 0.64.0. Only an exception thrown while the tool runs is marked `ERROR`. The failure is visible only in the tool result, which is exported only with the [content opt-in](#privacy). - **Error messages and stack traces** by default. The collector keeps the `ERROR` status and the error class (plus the HTTP status of a Gemini API error) and removes the text; see [Privacy](#privacy). - **Cost.** Gemini CLI reports tokens, not cost. Future AGI computes cost from the model name only when it has pricing for that model. @@ -142,7 +160,7 @@ To include content: processors: [transform/gemini-cli, batch] ``` -Model spans then carry the request and response messages, and tool spans the call request and result, as `gen_ai.input.messages` / `gen_ai.output.messages`; the collector also copies them to the span's input and output in Future AGI. Gemini CLI truncates each value to 10,000 characters. The system prompt (with your `GEMINI.md` memory and local paths such as the working directory), the tool schemas and the full error messages (`exception.message` and the status message, which can contain file paths) are exported too in this mode. The command line, executable paths, user and machine name, and stack traces stay removed. +Model spans then carry the request and response messages, and tool spans the call request and result, as `gen_ai.input.messages` / `gen_ai.output.messages`; the collector also copies them to the span's input and output in Future AGI. Gemini CLI truncates each value to 10,000 characters. The `user_prompt` and `system_prompt` turn spans stay without input and output: Gemini CLI 0.64.0 does not record content on them. The system prompt (with your `GEMINI.md` memory and local paths such as the working directory), the tool schemas and the full error messages (`exception.message` and the status message, which can contain file paths) are exported too in this mode. The command line, executable paths, user and machine name, and stack traces stay removed. ## Troubleshooting @@ -158,6 +176,8 @@ Model spans then carry the request and response messages, and tool spans the cal **Error spans show only `ApiError HTTP 503` or `error`.** The collector removes error text by default. Remove `transform/strip-content` (the content opt-in) to keep full error messages. -**Tool spans show as `unknown` and are named `tool_call`.** The spans did not go through the `transform/gemini-cli` processor; send Gemini CLI's export to the collector, not straight to Future AGI. +**Tool spans show as `unknown` and are named `tool_call`, or `user_prompt` spans have no span kind.** The spans did not go through the `transform/gemini-cli` processor; send Gemini CLI's export to the collector, not straight to Future AGI. + +**No `user_prompt` traces, only separate `llm_call` and `schedule_tool_calls` traces.** Expected for `gemini -p` (non-interactive): Gemini CLI emits the turn span only in interactive mode. **Prompts show up although you did not opt in.** `logPrompts` defaults to `true` in Gemini CLI. Set it to `false`, and keep `transform/strip-content` in the pipeline. From e371d19ec901ebcd9b5873cb17ca221e2add639d Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Thu, 5 Nov 2026 16:31:08 -0800 Subject: [PATCH 5/7] docs(gemini-cli): scope native session tracing and preserve privacy Replace public QA notes with a version-scoped non-sandbox setup, register the page in the shared catalog, and keep the shipped collector privacy filters intact. Add page contracts and cloud-only collector regression checks for default and opt-in exports. Co-Authored-By: Claude Code --- .github/workflows/gemini-cli-docs.yml | 43 ++++ scripts/gemini-cli-page.test.mjs | 60 ++++++ scripts/integrations.test.mjs | 1 + src/lib/integrations.mjs | 1 + .../integrations/coding-agents/gemini-cli.mdx | 186 +++++++----------- 5 files changed, 176 insertions(+), 115 deletions(-) create mode 100644 .github/workflows/gemini-cli-docs.yml create mode 100644 scripts/gemini-cli-page.test.mjs diff --git a/.github/workflows/gemini-cli-docs.yml b/.github/workflows/gemini-cli-docs.yml new file mode 100644 index 000000000..be57c2661 --- /dev/null +++ b/.github/workflows/gemini-cli-docs.yml @@ -0,0 +1,43 @@ +name: Gemini CLI collector privacy + +on: + pull_request: + paths: + - 'src/pages/docs/integrations/coding-agents/gemini-cli.mdx' + - 'examples/coding-agents/gemini-cli/**' + - 'scripts/gemini-cli-page.test.mjs' + - '.github/workflows/gemini-cli-docs.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + collector-privacy: + name: Gemini CLI collector privacy + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download released Collector Contrib + shell: bash + run: | + set -euo pipefail + mkdir "$RUNNER_TEMP/gemini-collector" + cd "$RUNNER_TEMP/gemini-collector" + archive=otelcol-contrib_0.162.0_linux_amd64.tar.gz + curl -fLsS "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.162.0/$archive" -o "$archive" + echo "fcc063749f730f8c21fe29f2d340ff174f5f1c5885bd3156fb6c985a3036fcc3 $archive" | sha256sum --check + tar -xzf "$archive" otelcol-contrib + test -x otelcol-contrib + - name: Check default and opt-in exports with loopback-only networking + shell: bash + run: | + sudo unshare --net -- bash -c ' + set -euo pipefail + ip link set lo up + exec env OTELCOL_BIN="$1" "$2" -I -m unittest discover -s "$3" -v + ' bash "$RUNNER_TEMP/gemini-collector/otelcol-contrib" "$(command -v python)" "$PWD/examples/coding-agents/gemini-cli/tests" diff --git a/scripts/gemini-cli-page.test.mjs b/scripts/gemini-cli-page.test.mjs new file mode 100644 index 000000000..fe1ddf8ee --- /dev/null +++ b/scripts/gemini-cli-page.test.mjs @@ -0,0 +1,60 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { integrations } from '../src/lib/integrations.mjs'; + +const read = file => fs.readFileSync(new URL(`../${file}`, import.meta.url), 'utf8'); +const page = read('src/pages/docs/integrations/coding-agents/gemini-cli.mdx'); +const collector = read('examples/coding-agents/gemini-cli/otel-collector.yaml'); +const settings = JSON.parse(read('examples/coding-agents/gemini-cli/settings.json')); + +test('Gemini CLI is discoverable as native OTLP with explicit non-sandbox scope', () => { + const entry = integrations.find(item => item.id === 'gemini-cli'); + assert.equal(entry?.href, '/docs/integrations/coding-agents/gemini-cli'); + assert.equal(entry.category, 'coding'); + assert.equal(entry.method, 'Native OTLP'); + const prerequisites = page.split('## 1. Start the collector')[0]; + assert.match(prerequisites, /Gemini CLI 0\.64\.0 on macOS/); + assert.match(prerequisites, /non-sandbox interactive \(`gemini`\) and non-interactive \(`gemini -p`\) sessions/); + assert.match(prerequisites, /does not cover sandbox runtimes/); + assert.match(prerequisites, /@google\/gemini-cli@0\.64\.0/); + assert.match(prerequisites, /releases\/tag\/v0\.162\.0/); + assert.doesNotMatch(page, /run_e2e\.sh|CAPTURE_CONTENT|test_collector\.py|in our test|not tested|read 2026-|verified live/i); +}); + +test('Gemini CLI setup matches the shipped native settings and preserves privacy controls', () => { + const snippet = page.match(/```json\n([\s\S]*?)\n```/); + assert.ok(snippet, 'Include a complete telemetry settings example'); + assert.deepEqual(JSON.parse(snippet[1]), settings); + assert.equal(settings.telemetry.logPrompts, false); + assert.equal(settings.telemetry.traces, true); + assert.equal(settings.telemetry.otlpProtocol, 'http'); + assert.match(collector, /processors: \[transform\/gemini-cli, transform\/strip-content, batch\]/); + assert.match(collector, /compression: none/); + assert.match(collector, /endpoint: \$\{env:GEMINI_OTLP_LISTEN:-127\.0\.0\.1:4318\}/); + for (const key of [ + 'gen_ai.input.messages', 'gen_ai.output.messages', 'input.value', 'output.value', + 'gen_ai.system_instructions', 'gen_ai.tool.definitions', 'exception.stacktrace', + 'exception.message', 'process.command_args', 'process.command', + 'process.executable.path', 'process.executable.name', 'process.owner', 'host.name', 'host.id', + ]) assert.ok(collector.includes(`delete_key(attributes, "${key}")`), `Preserve filtering for ${key}`); + assert.match(page, /local collector receives attributes before filtering/); + assert.match(page, /Tool descriptions can come from your MCP servers/); + assert.match(page, /remove sensitive text from tool names and descriptions/i); + assert.match(page, /This also exports system instructions, `GEMINI\.md` memory, tool schemas and full error messages/); + assert.match(page, /Stack traces and the local identity resource fields remain removed/); + assert.match(page, /processors: \[transform\/gemini-cli, batch\]/); +}); + +test('Gemini CLI explains session mapping and native trace limitations', () => { + assert.match(page, /copies `gen_ai\.conversation\.id` to `session\.id`/); + assert.match(page, /does not emit turn spans in this mode/); + assert.match(page, /returns an error instead of throwing/); + assert.match(page, /status `OK`/); + assert.match(page, /Gemini CLI has no whole-session span/); + assert.match(page, /Logs and metrics are not forwarded/); + assert.match(collector, /set\(attributes\["session.id"\], attributes\["gen_ai.conversation.id"\]\)/); + for (const operation of ['user_prompt', 'system_prompt', 'schedule_tool_calls']) { + assert.ok(collector.includes(`set(attributes["gen_ai.span.kind"], "CHAIN") where attributes["gen_ai.operation.name"] == "${operation}"`)); + } +}); diff --git a/scripts/integrations.test.mjs b/scripts/integrations.test.mjs index 30ff909ed..e838521d8 100644 --- a/scripts/integrations.test.mjs +++ b/scripts/integrations.test.mjs @@ -1,3 +1,4 @@ +import './gemini-cli-page.test.mjs'; import './ollama-java.test.mjs'; import './azure-openai-java.test.mjs'; import './langsmith-page.test.mjs'; diff --git a/src/lib/integrations.mjs b/src/lib/integrations.mjs index df0f390a2..c4e6eeb44 100644 --- a/src/lib/integrations.mjs +++ b/src/lib/integrations.mjs @@ -57,6 +57,7 @@ export const integrations = [ { id: 'gradio', name: 'Gradio', href: '/docs/integrations/traceai/gradio', category: 'frameworks', summary: 'Trace synchronous chat callbacks through their model client.', method: 'OpenAI client', logo: 'generic-framework' }, { id: 'z-ai', name: 'Z.ai', href: '/docs/integrations/providers/z-ai', category: 'providers', summary: 'Trace Python chat completions on the general-platform endpoint.', method: 'OpenAI client', logo: 'generic-model' }, { id: 'cursor', name: 'Cursor IDE', href: '/docs/integrations/coding-agents/cursor', category: 'coding', summary: 'Trace macOS IDE agent sessions through observation hooks.', method: 'Hook-to-OTLP', logo: 'generic-framework' }, + { id: 'gemini-cli', name: 'Gemini CLI', href: '/docs/integrations/coding-agents/gemini-cli', category: 'coding', summary: 'Trace non-sandbox CLI sessions through a privacy-filtering collector.', method: 'Native OTLP', logo: 'google' }, { id: 'langsmith', name: 'LangSmith', href: '/docs/integrations/langsmith', category: 'frameworks', summary: 'Forward supported LangSmith SDK traces over OTLP.', method: 'Native OTLP', logo: 'generic-framework' }, {"id": "mlflow", "name": "MLflow", "href": "/docs/integrations/mlflow", "category": "frameworks", "summary": "Export current MLflow traces with OTLP/HTTP.", "method": "Native OTLP", "logo": "generic-framework"}, { id: 'openai', name: 'OpenAI', href: '/docs/integrations/traceai/openai', category: 'providers', summary: 'Instrument OpenAI client calls in your application.', method: 'traceAI SDK', logo: 'openai' }, diff --git a/src/pages/docs/integrations/coding-agents/gemini-cli.mdx b/src/pages/docs/integrations/coding-agents/gemini-cli.mdx index 1bab51ecf..a3317fd99 100644 --- a/src/pages/docs/integrations/coding-agents/gemini-cli.mdx +++ b/src/pages/docs/integrations/coding-agents/gemini-cli.mdx @@ -1,88 +1,43 @@ --- title: "Gemini CLI session tracing with Future AGI" -description: "Trace Gemini CLI sessions in Future AGI with Gemini CLI's built-in OpenTelemetry export and a local OpenTelemetry Collector: turns, model calls with tokens, tool calls and errors, grouped by session. Prompt, tool and error content stay off by default." +description: "Send Gemini CLI's native OpenTelemetry traces through a local collector to inspect model calls, tools, tokens and sessions in Future AGI." --- -## About +Trace Gemini CLI model calls, tool calls and interactive turns in Future AGI using its [native OpenTelemetry export](https://geminicli.com/docs/cli/telemetry/) and a local OpenTelemetry Collector. -Gemini CLI has [built-in OpenTelemetry](https://geminicli.com/docs/cli/telemetry/). With traces turned on it emits a span for every model call and every tool call, and in interactive mode a span for every turn. This page sends those spans through a small OpenTelemetry Collector running next to Gemini CLI, which forwards them to Future AGI: +## Prerequisites -``` -Gemini CLI ──OTLP/HTTP──▶ otelcol-contrib (localhost:4318) ──▶ Future AGI - keeps traces, drops logs and metrics, - strips prompt-shaped attributes and error text, - labels span kinds, adds your keys -``` - -One Gemini CLI session shows up in Future AGI as one **session**. How it splits into traces depends on how you run Gemini CLI. - -**Interactive `gemini`:** one trace per prompt you type, rooted at a `user_prompt` span: - -``` -user_prompt one turn: the prompt you typed -├── llm_call one model request: model, input/output tokens, status -├── schedule_tool_calls the tool calls the model asked for -│ ├── list_directory tool call: name, call id, duration, status -│ └── read_file -└── system_prompt follow-up turn: Gemini CLI sends the tool results back - └── llm_call the next model request -``` +- **Gemini CLI 0.64.0 on macOS**, authenticated with your own Gemini account or API key, and Node.js 20 or newer. This guide covers **non-sandbox interactive (`gemini`) and non-interactive (`gemini -p`) sessions**. The loopback collector setup does not cover sandbox runtimes. +- **[OpenTelemetry Collector Contrib 0.162.0](https://github.com/open-telemetry/opentelemetry-collector-releases/releases/tag/v0.162.0)**, with `otelcol-contrib` on your PATH. Use the contrib distribution: the configuration requires its transform processor. +- **Future AGI API and secret keys** for the workspace where you want to collect traces. -Two cases in Gemini CLI 0.64.0 differ from that. A prompt you type while a turn is still running is queued and sent when that turn ends; in our test it then nested under the running turn, in the same trace, instead of starting its own. A slash command you type (`/quit`, `/stats`, ...) also starts a `user_prompt` trace, with no spans under it. +Install the version used by this guide: -**Non-interactive `gemini -p "..."`:** there is no turn span, so each model call and each tool batch is its own trace: - -``` -llm_call one model request -schedule_tool_calls one batch of tool calls -├── list_directory -└── read_file -llm_call the next model request +```bash +npm install -g @google/gemini-cli@0.64.0 ``` -Gemini CLI has no span for the whole session. Every span carries the Gemini CLI session id, which the collector copies to `session.id`, so the **Sessions** tab groups a session's traces in both modes. - -This is not the same as: - -- **The [Google GenAI](/docs/integrations/traceai/google_genai) and [Vertex AI](/docs/integrations/traceai/vertexai) SDK instrumentors** (`traceAI-google-genai`, `traceAI-vertexai`), which trace your own code that calls the Gemini API. They do not see Gemini CLI's tool scheduler or session. -- **Google Antigravity** or Gemini in other IDEs. Those are separate integrations. -- **Gemini CLI's `target: "gcp"` export**, which sends telemetry to Google Cloud, not to Future AGI. - -| | | -| --- | --- | -| Tested with | Gemini CLI 0.64.0 (`@google/gemini-cli`, macOS; interactive and `-p`), OpenTelemetry Collector Contrib 0.162.0 | -| Gemini CLI docs used | [Observability with OpenTelemetry](https://geminicli.com/docs/cli/telemetry/) and [configuration](https://www.geminicli.com/docs/reference/configuration), read 2026-10-14 | -| Requires | [`otelcol-contrib`](https://github.com/open-telemetry/opentelemetry-collector-releases/releases) (the contrib build: the config uses the `transform` processor) | -| Source | [examples/coding-agents/gemini-cli](https://github.com/future-agi/docs/tree/main/examples/coding-agents/gemini-cli) | +This integration traces the Gemini CLI application, not your own Gemini API code. For application SDK calls, use [Google GenAI](/docs/integrations/traceai/google_genai) or [Vertex AI](/docs/integrations/traceai/vertexai). -## 1. Run the collector +## 1. Start the collector -Download the example collector config and start `otelcol-contrib` with your Future AGI keys: +Download the collector configuration and start it with your Future AGI credentials: ```bash curl -fsSLO https://raw.githubusercontent.com/future-agi/docs/main/examples/coding-agents/gemini-cli/otel-collector.yaml -export FI_API_KEY=your-futureagi-api-key -export FI_SECRET_KEY=your-futureagi-secret-key -export FI_PROJECT_NAME=gemini-cli +export FI_API_KEY="your-futureagi-api-key" +export FI_SECRET_KEY="your-futureagi-secret-key" +export FI_PROJECT_NAME="gemini-cli" otelcol-contrib --config otel-collector.yaml ``` -It listens on `127.0.0.1:4318` (set `GEMINI_OTLP_LISTEN` to change it) and sends traces to `$FI_BASE_URL/tracer/v1/traces` (default `https://api.futureagi.com`; set `FI_BASE_URL` for a self-hosted Future AGI). Keep it running while you use Gemini CLI, or run it as a service. The collector's own metrics endpoint is turned off in the config (`service.telemetry.metrics.level: none`), so it does not take port 8888 from Jupyter or another collector. +Keep this terminal open while using Gemini CLI. The collector listens on `127.0.0.1:4318` and forwards traces to `https://api.futureagi.com/tracer/v1/traces`. For self-hosted Future AGI, set `FI_BASE_URL` to your API base URL before starting it. -The collector does four things Gemini CLI cannot do on its own: +The collector adds authentication, maps span kinds and session IDs, drops logs and metrics, and removes content before forwarding traces. Keep both transform processors in the default pipeline; Gemini CLI's `logPrompts: false` setting alone does not remove system instructions, tool schemas or error text. See [Content and privacy](#content-and-privacy). -| Step | Why | -| --- | --- | -| Adds `X-Api-Key`, `X-Secret-Key` and `project_name` | Gemini CLI 0.64.0 has no setting for OTLP headers. (Gemini CLI's main-branch docs list an `otlpHeaders` setting that is not in 0.64.0; the other three steps still need the collector.) | -| Drops logs and metrics | Gemini CLI always exports all three signals; Future AGI ingests traces. | -| Deletes `gen_ai.system_instructions`, `gen_ai.tool.definitions`, message attributes, error messages and stack traces, and the command line, executable paths, OS user name and machine name that Node's resource detectors add | Privacy default, see [Privacy](#privacy). | -| Sets `gen_ai.span.kind` from Gemini CLI's operation names, names tool spans after the tool, copies the session id to `session.id` | Gemini CLI uses its own operation names (`user_prompt`, `llm_call`, `tool_call`, ...), which Future AGI does not map by itself. | +## 2. Configure native telemetry -It also sends uncompressed: the Future AGI collector answers gzip-encoded OTLP requests with HTTP 400. - -## 2. Turn on Gemini CLI telemetry - -Add the `telemetry` block to `~/.gemini/settings.json` (all projects) or `.gemini/settings.json` in one project: +Merge this `telemetry` block into `~/.gemini/settings.json` for all projects, or `.gemini/settings.json` for one project. Preserve your other settings: ```json { @@ -97,87 +52,88 @@ Add the `telemetry` block to `~/.gemini/settings.json` (all projects) or `.gemin } ``` -- `traces: true` is required for token counts. With it off, Gemini CLI still emits the spans with the model, the tool names, the system prompt and the tool schemas, but without token counts. It is not a privacy setting on its own: keep `transform/strip-content` in the collector either way. -- `logPrompts: false` is not Gemini CLI's default. The vendor default is `true`, which puts prompts, responses, tool arguments and tool results on the spans. Keep it `false` unless you opt in to content. -- Do not set `outfile` (it replaces the OTLP export) or `target: "gcp"`. +- Keep `traces: true` for input and output token counts. Turning it off is **not** a privacy control: Gemini CLI 0.64.0 still emits spans with model, tool and system-instruction attributes. +- Set `logPrompts: false` explicitly; Gemini CLI defaults to `true`. +- Do not set `outfile`, which replaces OTLP export, or `target: "gcp"`, which routes telemetry to Google Cloud. +- Use the settings file for this setup and remove conflicting `GEMINI_TELEMETRY_*` environment overrides. To stop exporting, set `telemetry.enabled` to `false` in the file and restart Gemini CLI. -Gemini CLI also documents environment variables for these settings (`GEMINI_TELEMETRY_ENABLED`, `GEMINI_TELEMETRY_TRACES_ENABLED`, `GEMINI_TELEMETRY_TARGET`, `GEMINI_TELEMETRY_OTLP_ENDPOINT`, `GEMINI_TELEMETRY_OTLP_PROTOCOL`, `GEMINI_TELEMETRY_LOG_PROMPTS`). Prefer the settings file: in our 0.64.0 test, `GEMINI_TELEMETRY_ENABLED=false` did not stop the export when the settings file had `"enabled": true`. Setting `"enabled": false` in the file did: no spans reached the collector. - -Optionally also export `OTEL_NODE_RESOURCE_DETECTORS=env` in the shell that runs `gemini`. Node's default resource detectors record the full command line (for `gemini -p "..."` it contains the prompt), the Node executable path (often under your home directory), your OS user name and the machine name. The collector deletes all of these either way (`process.command_args`, `process.command`, `process.executable.*`, `process.owner`, `host.name`, `host.id`); this setting keeps them from leaving the Gemini CLI process at all. +Optionally set `OTEL_NODE_RESOURCE_DETECTORS=env` in the shell that runs Gemini CLI. This prevents Node's resource detectors from adding the command line, executable paths, OS user name and machine name. The collector strips those fields either way, including when content capture is enabled. ## 3. Run a session -Start `gemini` in a project and send one prompt that uses a tool, for example *"List the files in this folder and read README.md"*. Exit Gemini CLI, or wait a few seconds: spans are batched. +In a separate terminal, open a project you trust and start an interactive session: -Open **Observe** in Future AGI and select the project (`gemini-cli` above). For an interactive session the **Trace** tab lists one `user_prompt` trace per prompt; for `gemini -p` it lists separate `llm_call` and `schedule_tool_calls` traces. The **Sessions** tab groups them by Gemini CLI session. +```bash +gemini +``` -`run_e2e.sh` in the [examples folder](https://github.com/future-agi/docs/tree/main/examples/coding-agents/gemini-cli) does all three steps against a throwaway `HOME`, so it does not touch your own `~/.gemini`. It runs `gemini -p`, so its traces have no `user_prompt` span. It also checks what the collector forwarded: at least one span, and not the prompt's marker string, your user name, home directory or machine name (with `CAPTURE_CONTENT=1`, the marker must be there). It fails if another process already listens on its port, 24318 (`GEMINI_OTLP_PORT`): +Ask **"List the files in this folder and read README.md"**, and approve the tool calls you want to run. Alternatively, run the same task non-interactively: ```bash -git clone --depth 1 --filter=blob:none --sparse https://github.com/future-agi/docs.git fi-docs -git -C fi-docs sparse-checkout set examples/coding-agents/gemini-cli -FI_API_KEY=... FI_SECRET_KEY=... GEMINI_API_KEY=... \ - fi-docs/examples/coding-agents/gemini-cli/run_e2e.sh +gemini -p "List the files in this folder and read README.md" ``` -`tests/test_collector.py` in the same folder runs the shipped config in a real `otelcol-contrib` and checks what it forwards; it is skipped when no `otelcol-contrib` binary is found (set `OTELCOL_BIN`). +Keep Gemini CLI and the collector on the same host, outside a sandbox. Exit Gemini CLI or wait a few seconds for batched spans to arrive. + +## 4. Inspect the session + +Open **Observe** in Future AGI and select the project named by `FI_PROJECT_NAME` (`gemini-cli` above). Use **Sessions** to view a whole Gemini CLI session; the collector copies `gen_ai.conversation.id` to `session.id` to group its traces. -## What is captured +In the **Trace** tab, the trace shape depends on the mode: -| Gemini CLI span | Future AGI span kind | Fields | +- **Interactive:** a `user_prompt` span groups a turn's model and tool spans. A `system_prompt` child contains follow-up model calls after tools return. Despite its name, that span does not contain the system prompt. It starts when `user_prompt` ends, so include both when inspecting a turn's duration. +- **Non-interactive (`gemini -p`):** each `llm_call` and each `schedule_tool_calls` batch is a separate trace. Gemini CLI does not emit turn spans in this mode. + +In Gemini CLI 0.64.0, a prompt queued during a running turn can nest under that turn instead of starting a new trace. Typed slash commands can create empty `user_prompt` traces. + +### Captured fields + +| Gemini CLI operation | Future AGI span kind | Fields | | --- | --- | --- | -| `user_prompt` (interactive mode only) | `CHAIN` | one turn, the root of its model and tool spans: `gen_ai.agent.name=gemini-cli`, duration, status. No prompt text, even with the content opt-in: Gemini CLI 0.64.0 does not record it on this span (with the opt-in, the prompt is on the first `llm_call`) | -| `system_prompt` (interactive mode only) | `CHAIN` | the follow-up turn Gemini CLI starts itself to send tool results back to the model, a child of `user_prompt`; same fields. It starts when `user_prompt` ends, so a turn's full duration is `user_prompt` plus its `system_prompt` children. Despite the name it does not hold the system prompt (that is `gen_ai.system_instructions` on `llm_call`, deleted by default) | -| `llm_call` | `LLM` | `gen_ai.request.model`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.prompt.name` (Gemini CLI's prompt id), status `ERROR` when the API call fails, with the error class and HTTP status as the status message (for example `ApiError HTTP 503`; a retried 503 shows up as its own `ERROR` span) | -| `schedule_tool_calls` | `CHAIN` | parent of the tool calls the model requested in one response | -| `tool_call`, renamed to the tool (`read_file`, `list_directory`, `run_shell_command`, ...) | `TOOL` | `gen_ai.tool.name`, `gen_ai.tool.call_id`, `gen_ai.tool.description`, duration. Status `ERROR` (status message: the error class, for example `ENOENT`) only when the tool throws; see [What is not available](#what-is-not-available) | -| `agent_call` | `AGENT` | a subagent run; its model and tool spans nest under it. Gemini CLI 0.64.0 overwrites the subagent's name with `gen_ai.agent.name=gemini-cli`, so the subagent's own name is not available. Read from Gemini CLI 0.64.0's source; not exercised in our test | -| every span | | `gen_ai.conversation.id` and `session.id` = Gemini CLI session id; resource `service.name=gemini-cli` | +| `user_prompt`, `system_prompt` | `CHAIN` | Interactive turn structure, duration, status; no input or output, even with content capture | +| `llm_call` | `LLM` | Model, input/output token counts, prompt ID, duration, status | +| `schedule_tool_calls` | `CHAIN` | Parent of the tool calls requested in one model response | +| `tool_call` | `TOOL` | Tool name, call ID, description, duration, status; the collector names the span after the tool | +| `agent_call` | `AGENT` | Agent operation; Gemini CLI 0.64.0 records `gemini-cli` rather than the individual subagent name | -## What is not available +### Compatibility limits -- **One trace per session.** Gemini CLI has no session span: interactive mode gives one trace per turn, `gemini -p` one trace per model call or tool batch. Use the **Sessions** tab for the whole session. -- **Tool failures the tool reports itself.** A tool that returns an error instead of throwing (file not found, path outside the workspace, a shell execution error) ends its span with status `OK` in Gemini CLI 0.64.0. Only an exception thrown while the tool runs is marked `ERROR`. The failure is visible only in the tool result, which is exported only with the [content opt-in](#privacy). -- **Error messages and stack traces** by default. The collector keeps the `ERROR` status and the error class (plus the HTTP status of a Gemini API error) and removes the text; see [Privacy](#privacy). -- **Cost.** Gemini CLI reports tokens, not cost. Future AGI computes cost from the model name only when it has pricing for that model. -- **Cached, thinking and tool-use token counts.** Only input and output tokens are on the spans. -- **Gemini CLI logs and metrics** (`gemini_cli.user_prompt`, `gemini_cli.api_request`, token and latency metrics). The collector drops them; Future AGI ingests traces only. -- **Telemetry in sandbox mode** was not tested. +- Gemini CLI has no whole-session span. Use **Sessions**, rather than expecting one trace for the entire session. +- A tool that **returns an error instead of throwing** can end with status `OK` in Gemini CLI 0.64.0. Only a thrown exception marks its span `ERROR`; returned error details require content capture. +- Spans include input and output tokens, not cached, thinking or tool-use token counts. Gemini CLI does not export cost; Future AGI can calculate it only when pricing is available for the model. +- Logs and metrics are not forwarded by this configuration. -## Privacy +## Content and privacy -With the settings above, only structure and metadata leave your machine: model, token counts, tool names, call ids, tool descriptions (static text from Gemini CLI or your MCP servers), durations, statuses, error classes (`ApiError HTTP 429`, `ENOENT`), and the session id. +By default, Future AGI receives model names, token counts, tool names and descriptions, call IDs, durations, statuses, error classes and session IDs. **Tool descriptions can come from your MCP servers**; remove sensitive text from tool names and descriptions before using them. -- **Not exported by default:** prompts and responses, tool arguments and results (Gemini CLI leaves them out with `logPrompts: false`); the system prompt, which includes your `GEMINI.md` memory and local paths, the tool schemas, and error messages, which carry API error bodies and, for a tool that throws, file paths and command output (the collector deletes them or replaces them with the error class); the command line, executable paths, OS user name and machine name. -- **Never forwarded:** Gemini CLI's logs and metrics, and error stack traces (they list your Gemini CLI install path, including your home directory; the collector deletes them even with the content opt-in). +The collector removes prompt and response messages, tool arguments and results, system instructions (including `GEMINI.md` memory and local paths), tool schemas and full error messages. Failed spans retain `ERROR` and an error class, plus an HTTP status for Gemini API errors, such as `ApiError HTTP 503`. Command lines, executable paths, OS user names, machine names and exception stack traces are stripped even with content capture enabled. -To include content: +The local collector receives attributes before filtering them. Keep its receiver on loopback and do not add a debug or file exporter to this pipeline if you need to avoid recording that content locally. -1. Set `"logPrompts": true` in the `telemetry` block. -2. In `otel-collector.yaml`, remove `transform/strip-content` from the traces pipeline: +To opt in to content capture: + +1. Set `"logPrompts": true` in the telemetry settings. +2. Remove `transform/strip-content` from the collector's traces pipeline, leaving: ```yaml processors: [transform/gemini-cli, batch] ``` -Model spans then carry the request and response messages, and tool spans the call request and result, as `gen_ai.input.messages` / `gen_ai.output.messages`; the collector also copies them to the span's input and output in Future AGI. Gemini CLI truncates each value to 10,000 characters. The `user_prompt` and `system_prompt` turn spans stay without input and output: Gemini CLI 0.64.0 does not record content on them. The system prompt (with your `GEMINI.md` memory and local paths such as the working directory), the tool schemas and the full error messages (`exception.message` and the status message, which can contain file paths) are exported too in this mode. The command line, executable paths, user and machine name, and stack traces stay removed. +3. Restart the collector and Gemini CLI. -## Troubleshooting +Model messages and tool arguments/results then appear as input and output in Future AGI. Gemini CLI caps each message attribute at 10,000 characters. **This also exports system instructions, `GEMINI.md` memory, tool schemas and full error messages**, which can contain file paths or other sensitive data. Turn spans (`user_prompt` and `system_prompt`) still have no input or output. Stack traces and the local identity resource fields remain removed. -**No traces appear.** -1. Check that the collector is running and listening on the `otlpEndpoint` port. With `otlpProtocol: "http"` Gemini CLI posts to `/v1/traces`; the default protocol is `grpc`, which this collector config does not accept. -2. If the collector exits with `address already in use`, another process owns port 4318 (often another OpenTelemetry Collector or agent). Start this one on a free port, for example `GEMINI_OTLP_LISTEN=127.0.0.1:24318`, and set the same port in `otlpEndpoint` (`http://127.0.0.1:24318`). -3. Check the collector's output for `Exporting failed`. `401`/`403` means wrong keys. `400` means a request the Future AGI collector could not decode: keep `compression: none` on the exporter. -4. `telemetry.enabled` must be `true` in the settings file. `useCollector` together with `useCliAuth` disables telemetry. +## Troubleshooting -**Model spans have no token counts.** `telemetry.traces` is `false` in the settings file. +**No traces appear.** Check that the collector is running, `telemetry.enabled` is `true`, and `otlpProtocol` is `"http"`. Gemini CLI posts to `/v1/traces`; this collector does not accept gRPC. Do not combine `useCollector` with `useCliAuth`, which disables telemetry in this version. -**A failed tool call shows status `OK`.** Expected in Gemini CLI 0.64.0 when the tool reports the failure in its result instead of throwing; see [What is not available](#what-is-not-available). +**Port 4318 is already in use.** Set `GEMINI_OTLP_LISTEN=127.0.0.1:24318` before starting the collector and change Gemini CLI's `otlpEndpoint` to `http://127.0.0.1:24318`. -**Error spans show only `ApiError HTTP 503` or `error`.** The collector removes error text by default. Remove `transform/strip-content` (the content opt-in) to keep full error messages. +**The collector reports an export error.** For `401` or `403`, check your Future AGI keys. For `400`, keep `compression: none` in the exporter: the Future AGI endpoint requires uncompressed OTLP requests for this setup. -**Tool spans show as `unknown` and are named `tool_call`, or `user_prompt` spans have no span kind.** The spans did not go through the `transform/gemini-cli` processor; send Gemini CLI's export to the collector, not straight to Future AGI. +**Model spans have no token counts.** Set `telemetry.traces` to `true` and restart Gemini CLI. -**No `user_prompt` traces, only separate `llm_call` and `schedule_tool_calls` traces.** Expected for `gemini -p` (non-interactive): Gemini CLI emits the turn span only in interactive mode. +**Span kinds are missing or tool spans are still named `tool_call`.** Send Gemini CLI's export through the provided collector, with `transform/gemini-cli` in its traces pipeline, rather than directly to Future AGI. -**Prompts show up although you did not opt in.** `logPrompts` defaults to `true` in Gemini CLI. Set it to `false`, and keep `transform/strip-content` in the pipeline. +**Prompts appear without an intentional opt-in.** Set `logPrompts` to `false` and restore `transform/strip-content` in the collector pipeline, then restart both processes. From 711e5d035a0665d2063e3a72f8688a346962a8c3 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Thu, 5 Nov 2026 16:33:34 -0800 Subject: [PATCH 6/7] docs(ci): pin browser checks to supported Ubuntu image The PR build completed, but Playwright 1.56.1 cannot install Chromium on the ubuntu-latest 26.04 image. Keep all browser checks enabled on Ubuntu 24.04. Co-Authored-By: Claude Code --- .github/workflows/pr-checks.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 2bb5470c3..9bc167954 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -76,7 +76,8 @@ jobs: docs-build: name: Docs build & Pagefind - runs-on: ubuntu-latest + # Playwright 1.56.1 does not support the ubuntu-latest 26.04 image. + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 From 9b3a3de24f41d8d05de7862bc8240bb994bacdee Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Thu, 5 Nov 2026 22:22:53 -0800 Subject: [PATCH 7/7] docs(gemini-cli): clear inherited OTLP endpoint overrides Gemini CLI resolves its telemetry endpoint from environment variables before the settings file. Clear both endpoint overrides in the launch terminal so inherited configuration cannot bypass the privacy-filtering collector, and preserve this prerequisite in the page contract. Co-Authored-By: Claude Code --- scripts/gemini-cli-page.test.mjs | 14 ++++++++++++++ .../docs/integrations/coding-agents/gemini-cli.mdx | 13 +++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/scripts/gemini-cli-page.test.mjs b/scripts/gemini-cli-page.test.mjs index fe1ddf8ee..8d8c6a4f6 100644 --- a/scripts/gemini-cli-page.test.mjs +++ b/scripts/gemini-cli-page.test.mjs @@ -46,6 +46,20 @@ test('Gemini CLI setup matches the shipped native settings and preserves privacy assert.match(page, /processors: \[transform\/gemini-cli, batch\]/); }); +test('Gemini CLI clears inherited endpoint overrides before launching with privacy filtering', () => { + const setup = page.split('## 2. Configure native telemetry')[1]?.split('## 3. Run a session')[0]; + assert.ok(setup, 'Keep endpoint cleanup in the setup before the first session'); + assert.match(setup, /```bash\nunset OTEL_EXPORTER_OTLP_ENDPOINT GEMINI_TELEMETRY_OTLP_ENDPOINT\n```/); + assert.match(setup, /terminal that will launch Gemini CLI/); + assert.match(setup, /remove conflicting `GEMINI_TELEMETRY_\*` environment overrides/); + assert.match(setup, /take precedence over `telemetry\.otlpEndpoint`/); + assert.match(setup, /bypass the local collector and its privacy filters/); + assert.match(setup, /even with `logPrompts: false`/); + const session = page.split('## 3. Run a session')[1]?.split('## 4. Inspect the session')[0]; + assert.ok(session, 'Include session launch instructions'); + assert.match(session, /In the terminal where you cleared the overrides/); +}); + test('Gemini CLI explains session mapping and native trace limitations', () => { assert.match(page, /copies `gen_ai\.conversation\.id` to `session\.id`/); assert.match(page, /does not emit turn spans in this mode/); diff --git a/src/pages/docs/integrations/coding-agents/gemini-cli.mdx b/src/pages/docs/integrations/coding-agents/gemini-cli.mdx index a3317fd99..c9dcd2e9a 100644 --- a/src/pages/docs/integrations/coding-agents/gemini-cli.mdx +++ b/src/pages/docs/integrations/coding-agents/gemini-cli.mdx @@ -55,13 +55,22 @@ Merge this `telemetry` block into `~/.gemini/settings.json` for all projects, or - Keep `traces: true` for input and output token counts. Turning it off is **not** a privacy control: Gemini CLI 0.64.0 still emits spans with model, tool and system-instruction attributes. - Set `logPrompts: false` explicitly; Gemini CLI defaults to `true`. - Do not set `outfile`, which replaces OTLP export, or `target: "gcp"`, which routes telemetry to Google Cloud. -- Use the settings file for this setup and remove conflicting `GEMINI_TELEMETRY_*` environment overrides. To stop exporting, set `telemetry.enabled` to `false` in the file and restart Gemini CLI. + +Use the settings file for this setup. In a separate terminal that will launch Gemini CLI, remove conflicting `GEMINI_TELEMETRY_*` environment overrides and clear both endpoint overrides before starting a session: + +```bash +unset OTEL_EXPORTER_OTLP_ENDPOINT GEMINI_TELEMETRY_OTLP_ENDPOINT +``` + +Both endpoint variables take precedence over `telemetry.otlpEndpoint` and can bypass the local collector and its privacy filters, exporting content even with `logPrompts: false`. Repeat this cleanup in each terminal used to launch Gemini CLI. + +To stop exporting, set `telemetry.enabled` to `false` in the file and restart Gemini CLI. Optionally set `OTEL_NODE_RESOURCE_DETECTORS=env` in the shell that runs Gemini CLI. This prevents Node's resource detectors from adding the command line, executable paths, OS user name and machine name. The collector strips those fields either way, including when content capture is enabled. ## 3. Run a session -In a separate terminal, open a project you trust and start an interactive session: +In the terminal where you cleared the overrides, open a project you trust and start an interactive session: ```bash gemini