diff --git a/.github/workflows/gemini-cli-docs.yml b/.github/workflows/gemini-cli-docs.yml new file mode 100644 index 000000000..be57c2661 --- /dev/null +++ b/.github/workflows/gemini-cli-docs.yml @@ -0,0 +1,43 @@ +name: Gemini CLI collector privacy + +on: + pull_request: + paths: + - 'src/pages/docs/integrations/coding-agents/gemini-cli.mdx' + - 'examples/coding-agents/gemini-cli/**' + - 'scripts/gemini-cli-page.test.mjs' + - '.github/workflows/gemini-cli-docs.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + collector-privacy: + name: Gemini CLI collector privacy + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download released Collector Contrib + shell: bash + run: | + set -euo pipefail + mkdir "$RUNNER_TEMP/gemini-collector" + cd "$RUNNER_TEMP/gemini-collector" + archive=otelcol-contrib_0.162.0_linux_amd64.tar.gz + curl -fLsS "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.162.0/$archive" -o "$archive" + echo "fcc063749f730f8c21fe29f2d340ff174f5f1c5885bd3156fb6c985a3036fcc3 $archive" | sha256sum --check + tar -xzf "$archive" otelcol-contrib + test -x otelcol-contrib + - name: Check default and opt-in exports with loopback-only networking + shell: bash + run: | + sudo unshare --net -- bash -c ' + set -euo pipefail + ip link set lo up + exec env OTELCOL_BIN="$1" "$2" -I -m unittest discover -s "$3" -v + ' bash "$RUNNER_TEMP/gemini-collector/otelcol-contrib" "$(command -v python)" "$PWD/examples/coding-agents/gemini-cli/tests" diff --git a/examples/coding-agents/gemini-cli/otel-collector.yaml b/examples/coding-agents/gemini-cli/otel-collector.yaml new file mode 100644 index 000000000..a7780f767 --- /dev/null +++ b/examples/coding-agents/gemini-cli/otel-collector.yaml @@ -0,0 +1,130 @@ +# OpenTelemetry Collector between Gemini CLI and Future AGI. +# +# Gemini CLI exports OTLP traces, logs and metrics to this collector. The +# collector keeps the traces, removes prompt-shaped attributes, labels the span +# kinds Future AGI shows, and forwards the traces with your Future AGI keys. +# Logs and metrics are dropped: the Future AGI collector accepts traces only. +# +# FI_API_KEY=... FI_SECRET_KEY=... otelcol-contrib --config otel-collector.yaml +# +# Environment: +# FI_API_KEY, FI_SECRET_KEY required +# FI_BASE_URL default https://api.futureagi.com (traces go to $FI_BASE_URL/tracer/v1/traces) +# FI_PROJECT_NAME default gemini-cli +# GEMINI_OTLP_LISTEN default 127.0.0.1:4318 (Gemini CLI's otlpEndpoint must match) +# +# The collector's own metrics endpoint (port 8888) is turned off at the end of +# this file, so it does not collide with Jupyter or another collector. + +receivers: + otlp: + protocols: + http: + endpoint: ${env:GEMINI_OTLP_LISTEN:-127.0.0.1:4318} + +processors: + transform/gemini-cli: + error_mode: ignore + trace_statements: + - context: resource + statements: + - set(attributes["project_name"], "${env:FI_PROJECT_NAME:-gemini-cli}") + # Node's default resource detectors record the full command line + # (for `gemini -p "..."` it contains the prompt), local paths, the + # OS user name and the machine name. Keep only runtime and arch. + - delete_key(attributes, "process.command_args") + - delete_key(attributes, "process.command") + - delete_key(attributes, "process.executable.path") + - delete_key(attributes, "process.executable.name") + - delete_key(attributes, "process.owner") + - delete_key(attributes, "host.name") + - delete_key(attributes, "host.id") + - context: span + statements: + # Gemini CLI names its operations llm_call, tool_call, agent_call, + # schedule_tool_calls, and in interactive mode user_prompt (one turn, + # the root of its model and tool spans) and system_prompt (the + # follow-up turn that sends tool results back to the model). Map them + # to the span kinds Future AGI uses. + - set(attributes["gen_ai.span.kind"], "LLM") where attributes["gen_ai.operation.name"] == "llm_call" + - set(attributes["gen_ai.span.kind"], "TOOL") where attributes["gen_ai.operation.name"] == "tool_call" + - set(attributes["gen_ai.span.kind"], "AGENT") where attributes["gen_ai.operation.name"] == "agent_call" + - set(attributes["gen_ai.span.kind"], "CHAIN") where attributes["gen_ai.operation.name"] == "schedule_tool_calls" + - set(attributes["gen_ai.span.kind"], "CHAIN") where attributes["gen_ai.operation.name"] == "user_prompt" + - set(attributes["gen_ai.span.kind"], "CHAIN") where attributes["gen_ai.operation.name"] == "system_prompt" + # Name tool spans after the tool instead of "tool_call". + - set(name, attributes["gen_ai.tool.name"]) where name == "tool_call" and attributes["gen_ai.tool.name"] != nil + # Every span carries the Gemini CLI session id; as session.id it groups + # the session's traces into one Future AGI session. + - set(attributes["session.id"], attributes["gen_ai.conversation.id"]) where attributes["gen_ai.conversation.id"] != nil + # Only present with the content opt-in (logPrompts: true): show the + # messages as the span's input and output in Future AGI. + - set(attributes["input.value"], attributes["gen_ai.input.messages"]) where attributes["gen_ai.input.messages"] != nil + - set(attributes["output.value"], attributes["gen_ai.output.messages"]) where attributes["gen_ai.output.messages"] != nil + - context: spanevent + statements: + # Gemini CLI records every error as an "exception" event. Its stack + # trace lists file:// paths of your Gemini CLI install (home + # directory, OS user name) and no content, so it is dropped even + # with the content opt-in. + - delete_key(attributes, "exception.stacktrace") + + # Privacy default. With logPrompts false Gemini CLI already leaves out the + # message attributes; this also drops the system prompt (it embeds your + # GEMINI.md memory), the tool schemas and error text. Remove this processor + # from the pipeline below only to opt in to content. + transform/strip-content: + error_mode: ignore + trace_statements: + - context: span + statements: + - delete_key(attributes, "gen_ai.input.messages") + - delete_key(attributes, "gen_ai.output.messages") + - delete_key(attributes, "input.value") + - delete_key(attributes, "output.value") + - delete_key(attributes, "gen_ai.system_instructions") + - delete_key(attributes, "gen_ai.tool.definitions") + # The status message of a failed span is the raw error text: an API + # error body, or for a tool that throws, file paths and command + # output. Keep the ERROR status, replace the text (refined from the + # exception event below). + - set(status.message, "error") where status.code == STATUS_CODE_ERROR + - set(status.message, "") where status.code != STATUS_CODE_ERROR + - context: spanevent + statements: + # Status message = the error class or code (ApiError, ENOENT, ...), + # plus the HTTP status for Gemini API errors ("ApiError HTTP 429"). + - set(span.status.message, attributes["exception.type"]) where name == "exception" and span.status.code == STATUS_CODE_ERROR and IsMatch(attributes["exception.type"], "^[A-Za-z0-9_.:-]{1,80}$") + - set(span.status.message, Concat([attributes["exception.type"], ExtractPatterns(attributes["exception.message"], "\"code\\\\?\":\\s*(?P[1-5][0-9][0-9])")["code"]], " HTTP ")) where name == "exception" and span.status.code == STATUS_CODE_ERROR and attributes["exception.type"] == "ApiError" and IsMatch(attributes["exception.message"], "\"code\\\\?\":\\s*[1-5][0-9][0-9]") + - delete_key(attributes, "exception.type") where not IsMatch(attributes["exception.type"], "^[A-Za-z0-9_.:-]{1,80}$") + - delete_key(attributes, "exception.message") + + batch: + timeout: 2s + +exporters: + otlphttp/futureagi: + traces_endpoint: ${env:FI_BASE_URL:-https://api.futureagi.com}/tracer/v1/traces + # The Future AGI collector reads the request body as-is; the exporter's + # default gzip encoding is answered with HTTP 400. + compression: none + headers: + X-Api-Key: ${env:FI_API_KEY} + X-Secret-Key: ${env:FI_SECRET_KEY} + nop: {} + +service: + telemetry: + metrics: + level: none + pipelines: + traces: + receivers: [otlp] + processors: [transform/gemini-cli, transform/strip-content, batch] + exporters: [otlphttp/futureagi] + logs: + receivers: [otlp] + exporters: [nop] + metrics: + receivers: [otlp] + exporters: [nop] diff --git a/examples/coding-agents/gemini-cli/run_e2e.sh b/examples/coding-agents/gemini-cli/run_e2e.sh new file mode 100755 index 000000000..6a315ee56 --- /dev/null +++ b/examples/coding-agents/gemini-cli/run_e2e.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# One real Gemini CLI session, exported through otel-collector.yaml to Future AGI. +# +# FI_API_KEY=... FI_SECRET_KEY=... GEMINI_API_KEY=... ./run_e2e.sh +# +# Optional: FI_BASE_URL (default https://api.futureagi.com), FI_PROJECT_NAME +# (default gemini-cli), GEMINI_MODEL (default gemini-3.8-flash), +# GEMINI_OTLP_PORT (port the collector listens on, default 24318), +# GEMINI_BIN / OTELCOL_BIN (default: gemini / otelcol-contrib on PATH), +# CAPTURE_CONTENT=1 to run the documented content opt-in instead. +# +# Gemini CLI runs with a throwaway HOME and project directory, so your own +# ~/.gemini settings and sessions are not read or changed. The prompt asks for +# a directory listing and a file read, so the session has tool calls. The +# collector also writes what it forwards to a local file, which the script +# checks: by default the marker string from the prompt, your user name, home +# directory and machine name must not be in it; with CAPTURE_CONTENT=1 the +# marker must be. It also fails if the port is already taken by another process +# or the collector forwarded no spans. The script exits non-zero if a check +# fails and then keeps its work directory for inspection; on success it deletes +# it. It runs Gemini CLI non-interactively (-p), so there is no user_prompt turn +# span; see the page for the interactive trace shape. +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +: "${FI_API_KEY:?set FI_API_KEY}" "${FI_SECRET_KEY:?set FI_SECRET_KEY}" +: "${GEMINI_API_KEY:?set GEMINI_API_KEY}" +GEMINI_BIN="${GEMINI_BIN:-gemini}" +OTELCOL_BIN="${OTELCOL_BIN:-otelcol-contrib}" +PORT="${GEMINI_OTLP_PORT:-24318}" +export FI_PROJECT_NAME="${FI_PROJECT_NAME:-gemini-cli}" +export GEMINI_OTLP_LISTEN="127.0.0.1:$PORT" + +# Another process on the port would receive Gemini CLI's spans instead of this +# collector, and the checks below would run on an empty capture. +if (exec 3<>"/dev/tcp/127.0.0.1/$PORT") 2>/dev/null; then + echo "FAIL: port $PORT is already in use by another process; set GEMINI_OTLP_PORT to a free port" >&2 + exit 1 +fi + +WORK="$(mktemp -d)" +fail() { + echo "FAIL: $*" >&2 + echo "work directory kept: $WORK" >&2 + exit 1 +} +COLLECTOR_ARGS=(--config "$HERE/otel-collector.yaml" + --set "exporters.file/e2e.path=$WORK/forwarded.json" + --set "service.pipelines.traces.exporters=[otlphttp/futureagi,file/e2e]") +LOG_PROMPTS=false +if [ "${CAPTURE_CONTENT:-0}" = "1" ]; then + LOG_PROMPTS=true + COLLECTOR_ARGS+=(--set "service.pipelines.traces.processors=[transform/gemini-cli,batch]") +fi + +"$OTELCOL_BIN" "${COLLECTOR_ARGS[@]}" > "$WORK/collector.log" 2>&1 & +COLLECTOR_PID=$! +trap 'kill "$COLLECTOR_PID" 2>/dev/null || true' EXIT +listening=0 +for _ in $(seq 1 50); do + if (exec 3<>"/dev/tcp/127.0.0.1/$PORT") 2>/dev/null; then listening=1; break; fi + kill -0 "$COLLECTOR_PID" 2>/dev/null || break + sleep 0.2 +done +if [ "$listening" != 1 ] || ! kill -0 "$COLLECTOR_PID" 2>/dev/null; then + tail -5 "$WORK/collector.log" >&2 + if grep -q "address already in use" "$WORK/collector.log"; then + fail "port $PORT is already in use by another process; set GEMINI_OTLP_PORT to a free port" + fi + fail "the collector is not listening on 127.0.0.1:$PORT (see $WORK/collector.log)" +fi + +mkdir -p "$WORK/home/.gemini" "$WORK/project" +python3 - "$HERE/settings.json" "$WORK/home/.gemini/settings.json" "$PORT" "$LOG_PROMPTS" <<'EOF' +import json, sys +src, dst, port, log_prompts = sys.argv[1:] +settings = json.load(open(src)) +settings["telemetry"]["otlpEndpoint"] = "http://127.0.0.1:" + port +settings["telemetry"]["logPrompts"] = log_prompts == "true" +settings["security"] = {"auth": {"selectedType": "gemini-api-key"}} +json.dump(settings, open(dst, "w"), indent=2) +EOF +printf '# Gemini CLI E2E fixture\n\nA throwaway project for the Future AGI recipe.\n' > "$WORK/project/README.md" + +SESSION_ID="$(python3 -c 'import uuid; print(uuid.uuid4())')" +MARKER="FI-E2E-MARKER-$(date +%s)" +( + cd "$WORK/project" + HOME="$WORK/home" GEMINI_CLI_TRUST_WORKSPACE=true OTEL_NODE_RESOURCE_DETECTORS=env \ + "$GEMINI_BIN" -m "${GEMINI_MODEL:-gemini-3.8-flash}" --session-id "$SESSION_ID" \ + -p "List the files in this folder and read README.md. Reply with its first heading. $MARKER" \ + < /dev/null +) || echo "gemini exited non-zero; checking what was exported anyway" >&2 + +# Gemini CLI flushes on exit; give the collector's batch processor time to send. +sleep 5 +kill "$COLLECTOR_PID" +wait "$COLLECTOR_PID" 2>/dev/null || true +trap - EXIT +if grep -q -E $'\terror\t|Exporting failed' "$WORK/collector.log"; then + echo "collector log has errors:" >&2 + grep -E $'\terror\t|Exporting failed' "$WORK/collector.log" | tail -5 >&2 +fi + +echo "session_id=$SESSION_ID" +echo "marker=$MARKER" +echo "project=$FI_PROJECT_NAME" +touch "$WORK/forwarded.json" +SPANS_FORWARDED="$(python3 - "$WORK/forwarded.json" <<'EOF' +import json, sys +spans = [s for line in open(sys.argv[1]) if line.strip() + for rs in json.loads(line)["resourceSpans"] for ss in rs["scopeSpans"] for s in ss["spans"]] +print(len(spans)) +print("span_names=%s" % ",".join(sorted({s["name"] for s in spans})), file=sys.stderr) +EOF +)" +echo "spans_forwarded=$SPANS_FORWARDED" +# Zero spans would make the privacy checks below pass on nothing. +[ "$SPANS_FORWARDED" -gt 0 ] || fail "the collector forwarded no spans (see $WORK/collector.log)" +if [ "$LOG_PROMPTS" = true ]; then + grep -q -F "$MARKER" "$WORK/forwarded.json" || fail "content opt-in: the marker was not forwarded" + echo "privacy: content opt-in, marker forwarded" +else + if grep -q -F "$MARKER" "$WORK/forwarded.json"; then fail "the prompt marker was forwarded"; fi + for needle in "$HOME" "/$(id -un)/" "$(hostname)" "$(hostname -s)"; do + if grep -q -F "$needle" "$WORK/forwarded.json"; then + fail "forwarded spans contain a local identity string" + fi + done + echo "privacy: marker, home directory, user name and machine name not forwarded" +fi +rm -rf "$WORK" diff --git a/examples/coding-agents/gemini-cli/settings.json b/examples/coding-agents/gemini-cli/settings.json new file mode 100644 index 000000000..8a399e483 --- /dev/null +++ b/examples/coding-agents/gemini-cli/settings.json @@ -0,0 +1,10 @@ +{ + "telemetry": { + "enabled": true, + "target": "local", + "otlpEndpoint": "http://127.0.0.1:4318", + "otlpProtocol": "http", + "traces": true, + "logPrompts": false + } +} diff --git a/examples/coding-agents/gemini-cli/tests/collector-test-overrides.yaml b/examples/coding-agents/gemini-cli/tests/collector-test-overrides.yaml new file mode 100644 index 000000000..5567f93fa --- /dev/null +++ b/examples/coding-agents/gemini-cli/tests/collector-test-overrides.yaml @@ -0,0 +1,13 @@ +# Test-only overlay for otel-collector.yaml: write forwarded traces to a file +# instead of sending them to Future AGI. +exporters: + file/test: + path: ${env:TEST_OUT} + flush_interval: 100ms +processors: + batch: + timeout: 100ms +service: + pipelines: + traces: + exporters: [file/test] diff --git a/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json b/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json new file mode 100644 index 000000000..ae72b1789 --- /dev/null +++ b/examples/coding-agents/gemini-cli/tests/fixtures/gemini-cli-0.64.0-traces.json @@ -0,0 +1,728 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "v26.8.1" + } + }, + { + "key": "session.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "process.command_args", + "value": { + "arrayValue": { + "values": [ + { + "stringValue": "node" + }, + { + "stringValue": "gemini" + }, + { + "stringValue": "-p" + }, + { + "stringValue": "FIXTURE-CONTENT-MARKER prompt on the command line" + } + ] + } + } + }, + { + "key": "process.executable.path", + "value": { + "stringValue": "/Users/fixture-user/.nvm/versions/node/v22.18.0/bin/node" + } + }, + { + "key": "process.executable.name", + "value": { + "stringValue": "/Users/fixture-user/.nvm/versions/node/v22.18.0/bin/node" + } + }, + { + "key": "process.command", + "value": { + "stringValue": "/Users/fixture-user/.npm-global/bin/gemini" + } + }, + { + "key": "process.owner", + "value": { + "stringValue": "fixture-user" + } + }, + { + "key": "host.name", + "value": { + "stringValue": "fixture-users-laptop.local" + } + }, + { + "key": "host.arch", + "value": { + "stringValue": "arm64" + } + }, + { + "key": "process.runtime.name", + "value": { + "stringValue": "nodejs" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "gemini-cli", + "version": "v1" + }, + "spans": [ + { + "traceId": "45dc5d4d5ecd078b2b9c925d48995b57", + "spanId": "81ea502ba18b0eff", + "flags": 257, + "name": "llm_call", + "kind": 1, + "startTimeUnixNano": "1792027795983000000", + "endTimeUnixNano": "1792027802035021583", + "attributes": [ + { + "key": "gen_ai.request.model", + "value": { + "stringValue": "gemini-3.8-flash" + } + }, + { + "key": "gen_ai.prompt.name", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.system_instructions", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.system_instructions\"}]" + } + }, + { + "key": "gen_ai.tool.definitions", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.tool.definitions\"}]" + } + }, + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.input.messages\"}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.output.messages\"}]" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "llm_call" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power " + } + }, + { + "key": "gen_ai.usage.input_tokens", + "value": { + "intValue": "8816" + } + }, + { + "key": "gen_ai.usage.output_tokens", + "value": { + "intValue": "38" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "44fef0931f271d79c16809bc24382afb", + "spanId": "577562294772ba39", + "parentSpanId": "1669e7ece2fceb8b", + "flags": 257, + "name": "tool_call", + "kind": 1, + "startTimeUnixNano": "1792027802044000000", + "endTimeUnixNano": "1792027802057322208", + "attributes": [ + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "list_directory" + } + }, + { + "key": "gen_ai.tool.call_id", + "value": { + "stringValue": "list_directory__call_131963" + } + }, + { + "key": "gen_ai.tool.description", + "value": { + "stringValue": "Lists the names of files and subdirectories directly within " + } + }, + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.input.messages\"}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.output.messages\"}]" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "tool_call" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power " + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "44fef0931f271d79c16809bc24382afb", + "spanId": "234a34b7405f5ec8", + "parentSpanId": "1669e7ece2fceb8b", + "flags": 257, + "name": "tool_call", + "kind": 1, + "startTimeUnixNano": "1792027802047000000", + "endTimeUnixNano": "1792027802064669833", + "attributes": [ + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "read_file" + } + }, + { + "key": "gen_ai.tool.call_id", + "value": { + "stringValue": "read_file__call_131969" + } + }, + { + "key": "gen_ai.tool.description", + "value": { + "stringValue": "Reads and returns the content of a specified file. If the fi" + } + }, + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.input.messages\"}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.output.messages\"}]" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "tool_call" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power " + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "44fef0931f271d79c16809bc24382afb", + "spanId": "1669e7ece2fceb8b", + "flags": 257, + "name": "schedule_tool_calls", + "kind": 1, + "startTimeUnixNano": "1792027802038000000", + "endTimeUnixNano": "1792027802070371583", + "attributes": [ + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.input.messages\"}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.output.messages\"}]" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "schedule_tool_calls" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power " + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "a4ee9efb18dd4ee74e57eba8f8d30a82", + "spanId": "f10e2357106088c0", + "flags": 257, + "name": "llm_call", + "kind": 1, + "startTimeUnixNano": "1792027802079000000", + "endTimeUnixNano": "1792027804392816500", + "attributes": [ + { + "key": "gen_ai.request.model", + "value": { + "stringValue": "gemini-3.8-flash" + } + }, + { + "key": "gen_ai.prompt.name", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.system_instructions", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.system_instructions\"}]" + } + }, + { + "key": "gen_ai.tool.definitions", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.tool.definitions\"}]" + } + }, + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.input.messages\"}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.output.messages\"}]" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "llm_call" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power " + } + }, + { + "key": "gen_ai.usage.input_tokens", + "value": { + "intValue": "9255" + } + }, + { + "key": "gen_ai.usage.output_tokens", + "value": { + "intValue": "38" + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "5a1e0f3c2b9d4e8f8a7b6c5d4e3f2a10", + "spanId": "9b8a7c6d5e4f3a21", + "flags": 257, + "name": "llm_call", + "kind": 1, + "startTimeUnixNano": "1792027810000000000", + "endTimeUnixNano": "1792027810472056458", + "attributes": [ + { + "key": "gen_ai.request.model", + "value": { + "stringValue": "gemini-2.5-flash" + } + }, + { + "key": "gen_ai.prompt.name", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.system_instructions", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.system_instructions\"}]" + } + }, + { + "key": "gen_ai.tool.definitions", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.tool.definitions\"}]" + } + }, + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.input.messages\"}]" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "llm_call" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power " + } + } + ], + "events": [ + { + "timeUnixNano": "1792027810472056458", + "name": "exception", + "attributes": [ + { + "key": "exception.type", + "value": { + "stringValue": "ApiError" + } + }, + { + "key": "exception.message", + "value": { + "stringValue": "{\"error\":{\"message\":\"{\\n \\\"error\\\": {\\n \\\"code\\\": 404,\\n \\\"message\\\": \\\"This model models/gemini-2.5-flash is no longer available to new users. Please update your code to use models/gemini-3.8-flash for the latest features and improvements. We recommend you to use the Interactions API (https://ai.google.dev/gemini-api/docs/get-started).\\\",\\n \\\"status\\\": \\\"NOT_FOUND\\\"\\n }\\n}\\n\",\"code\":404,\"status\":\"\"}}" + } + }, + { + "key": "exception.stacktrace", + "value": { + "stringValue": "ApiError: {\"error\":{\"message\":\"{\\n \\\"error\\\": {\\n \\\"code\\\": 404,\\n \\\"message\\\": \\\"This model models/gemini-2.5-flash is no longer available to new users. Please update your code to use models/gemini-3.8-flash for the latest features and improvements. We recommend you to use the Interactions API (https://ai.google.dev/gemini-api/docs/get-started).\\\",\\n \\\"status\\\": \\\"NOT_FOUND\\\"\\n }\\n}\\n\",\"code\":404,\"status\":\"\"}}\n at throwErrorIfNotOK (file:///Users/fixture-user/.npm-global/lib/node_modules/@google/gemini-cli/bundle/chunk-NR7L3FXV.js:267903:24)\n at process.processTicksAndRejections (node:internal/process/task_queues:104:5)\n at async file:///Users/fixture-user/.npm-global/lib/node_modules/@google/gemini-cli/bundle/chunk-NR7L3FXV.js:267666:7\n at async Models.generateContentStream (file:///Users/fixture-user/.npm-global/lib/node_modules/@google/gemini-cli/bundle/chunk-NR7L3FXV.js:268762:16)\n at async file:///Users/fixture-user/.npm-global/lib/node_modules/@google/gemini-cli/bundle/chunk-NR7L3FXV.js:312364:19" + } + } + ] + } + ], + "status": { + "message": "{\"error\":{\"message\":\"{\\n \\\"error\\\": {\\n \\\"code\\\": 404,\\n \\\"message\\\": \\\"This model models/gemini-2.5-flash is no longer available to new users. Please update your code to use models/gemini-3.8-flash for the latest features and improvements. We recommend you to use the Interactions API (https://ai.google.dev/gemini-api/docs/get-started).\\\",\\n \\\"status\\\": \\\"NOT_FOUND\\\"\\n }\\n}\\n\",\"code\":404,\"status\":\"\"}}", + "code": 2 + } + }, + { + "traceId": "5a1e0f3c2b9d4e8f8a7b6c5d4e3f2a11", + "spanId": "9b8a7c6d5e4f3a22", + "flags": 257, + "name": "tool_call", + "kind": 1, + "startTimeUnixNano": "1792027811000000000", + "endTimeUnixNano": "1792027811012000000", + "attributes": [ + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "read_file" + } + }, + { + "key": "gen_ai.tool.call_id", + "value": { + "stringValue": "read_file__call_131990" + } + }, + { + "key": "gen_ai.tool.description", + "value": { + "stringValue": "Reads and returns the content of a specified file. " + } + }, + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.input.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.input.messages\"}]" + } + }, + { + "key": "gen_ai.output.messages", + "value": { + "stringValue": "[{\"text\": \"FIXTURE-CONTENT-MARKER gen_ai.output.messages\"}]" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "tool_call" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power " + } + } + ], + "events": [ + { + "timeUnixNano": "1792027811012000000", + "name": "exception", + "attributes": [ + { + "key": "exception.type", + "value": { + "stringValue": "ENOENT" + } + }, + { + "key": "exception.message", + "value": { + "stringValue": "ENOENT: no such file or directory, open '/Users/fixture-user/project/FIXTURE-CONTENT-MARKER.txt'" + } + }, + { + "key": "exception.stacktrace", + "value": { + "stringValue": "Error: ENOENT: no such file or directory, open '/Users/fixture-user/project/FIXTURE-CONTENT-MARKER.txt'\n at async open (node:internal/fs/promises:639:25)\n at async file:///Users/fixture-user/.npm-global/lib/node_modules/@google/gemini-cli/bundle/chunk-J7SXW2FI.js:316160:7" + } + } + ] + } + ], + "status": { + "message": "ENOENT: no such file or directory, open '/Users/fixture-user/project/FIXTURE-CONTENT-MARKER.txt'", + "code": 2 + } + }, + { + "traceId": "7c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f", + "spanId": "6e5f4a3b2c1d0e01", + "flags": 257, + "name": "user_prompt", + "kind": 1, + "startTimeUnixNano": "1792027900000000000", + "endTimeUnixNano": "1792028067923258250", + "attributes": [ + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "user_prompt" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power of Gemini directly into your terminal. It is designed to be a terminal-first, extensible, and powerful tool for developers, engineers, SREs, and beyond." + } + } + ], + "status": { + "code": 1 + } + }, + { + "traceId": "7c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f", + "spanId": "6e5f4a3b2c1d0e02", + "parentSpanId": "6e5f4a3b2c1d0e01", + "flags": 257, + "name": "system_prompt", + "kind": 1, + "startTimeUnixNano": "1792028067923000000", + "endTimeUnixNano": "1792028181319740708", + "attributes": [ + { + "key": "gen_ai.conversation.id", + "value": { + "stringValue": "fixture-session-0001" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "system_prompt" + } + }, + { + "key": "gen_ai.agent.name", + "value": { + "stringValue": "gemini-cli" + } + }, + { + "key": "gen_ai.agent.description", + "value": { + "stringValue": "Gemini CLI is an open-source AI agent that brings the power of Gemini directly into your terminal. It is designed to be a terminal-first, extensible, and powerful tool for developers, engineers, SREs, and beyond." + } + } + ], + "status": { + "code": 1 + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/examples/coding-agents/gemini-cli/tests/test_collector.py b/examples/coding-agents/gemini-cli/tests/test_collector.py new file mode 100644 index 000000000..c09493da3 --- /dev/null +++ b/examples/coding-agents/gemini-cli/tests/test_collector.py @@ -0,0 +1,203 @@ +"""Run otel-collector.yaml in a real otelcol-contrib and check what it forwards. + +The input is spans recorded from Gemini CLI 0.64.0 with content capture on +(``logPrompts: true``), with the content values replaced by a marker +(tests/fixtures/gemini-cli-0.64.0-traces.json). Appended to that: two error +spans, an ``llm_call`` that failed with a Gemini API error (status message, +exception event and stack trace from a real 0.64.0 run, install path rewritten +to /Users/fixture-user) and a synthetic ``tool_call`` that threw an fs error +with a local path in its message; and the ``user_prompt`` turn span with its +``system_prompt`` follow-up child from a real interactive 0.64.0 session +(session id replaced). Gemini CLI puts no content on those two even with +``logPrompts: true``. The Future AGI exporter is swapped for a file +exporter (tests/collector-test-overrides.yaml); everything else is the shipped +config. + + OTELCOL_BIN=/path/to/otelcol-contrib python3 -m unittest discover -s tests + +Skipped when no otelcol-contrib binary is found. +""" +from __future__ import annotations + +import json +import os +import shutil +import socket +import subprocess +import tempfile +import time +import unittest +import urllib.request +from pathlib import Path + +HERE = Path(__file__).resolve().parent +CONFIG = HERE.parent / "otel-collector.yaml" +OVERRIDES = HERE / "collector-test-overrides.yaml" +FIXTURE = HERE / "fixtures" / "gemini-cli-0.64.0-traces.json" +MARKER = "FIXTURE-CONTENT-MARKER" +# Added by Node's process and host resource detectors when +# OTEL_NODE_RESOURCE_DETECTORS is not set to env: paths, user and machine name. +LOCAL_IDENTITY = ("process.command_args", "process.command", "process.executable.path", + "process.executable.name", "process.owner", "host.name", "host.id") +OTELCOL = os.environ.get("OTELCOL_BIN") or shutil.which("otelcol-contrib") +CONTENT_KEYS = ("gen_ai.input.messages", "gen_ai.output.messages", "input.value", + "output.value", "gen_ai.system_instructions", "gen_ai.tool.definitions") + + +def free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +class ShippedConfigTest(unittest.TestCase): + def test_collector_metrics_off_and_uncompressed(self): + text = CONFIG.read_text() + # Port 8888 (the collector's own metrics) collides with Jupyter or a + # second collector; Future AGI answers gzip with HTTP 400. + self.assertRegex(text, r"(?m)^service:\n telemetry:\n metrics:\n level: none$") + self.assertRegex(text, r"(?m)^ compression: none$") + self.assertNotIn("level: none", OVERRIDES.read_text()) + + +def attrs(items): + out = {} + for a in items or []: + value = a["value"] + out[a["key"]] = next(iter(value.values())) if value else None + return out + + +@unittest.skipUnless(OTELCOL, "otelcol-contrib not found (set OTELCOL_BIN)") +class CollectorTest(unittest.TestCase): + def run_collector(self, *extra_config: str): + tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, tmp, True) + out = Path(tmp) / "out.json" + port = free_port() + env = dict(os.environ, GEMINI_OTLP_LISTEN="127.0.0.1:%d" % port, TEST_OUT=str(out), + FI_API_KEY="test-key", FI_SECRET_KEY="test-secret", FI_PROJECT_NAME="gemini-cli-test") + args = [OTELCOL, "--config", "file:" + str(CONFIG), "--config", "file:" + str(OVERRIDES)] + for item in extra_config: + args += ["--set", item] + proc = subprocess.Popen(args, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + self.addCleanup(proc.stderr.close) + self.addCleanup(proc.wait, 10) + self.addCleanup(proc.terminate) + body = FIXTURE.read_bytes() + for _ in range(100): + try: + req = urllib.request.Request("http://127.0.0.1:%d/v1/traces" % port, data=body, + headers={"Content-Type": "application/json"}) + urllib.request.urlopen(req, timeout=2).read() + break + except OSError: + if proc.poll() is not None: + self.fail("collector exited: " + proc.stderr.read().decode()[-2000:]) + time.sleep(0.2) + for _ in range(100): + if out.exists() and out.stat().st_size: + break + time.sleep(0.2) + text = out.read_text() + self.assertTrue(text, "collector wrote nothing") + return text, [json.loads(line) for line in text.splitlines() if line.strip()] + + def spans(self, requests): + result = [] + for req in requests: + for rs in req["resourceSpans"]: + resource = attrs(rs["resource"]["attributes"]) + for ss in rs["scopeSpans"]: + for span in ss["spans"]: + result.append((resource, span["name"], attrs(span["attributes"]))) + return result + + def raw_spans(self, requests): + return {span["spanId"]: span for req in requests for rs in req["resourceSpans"] + for ss in rs["scopeSpans"] for span in ss["spans"]} + + def test_default_strips_content_and_labels_kinds(self): + text, requests = self.run_collector() + self.assertNotIn(MARKER, text) + self.assertNotIn("/Users/", text) + self.assertNotIn("fixture-user", text) + self.assertNotIn("file://", text) + self.assertNotIn("NOT_FOUND", text) # the API error body + spans = self.spans(requests) + self.assertEqual(len(spans), 9) + kinds = {} + for resource, name, a in spans: + self.assertEqual(resource["project_name"], "gemini-cli-test") + for key in LOCAL_IDENTITY: + self.assertNotIn(key, resource) + self.assertEqual(resource["host.arch"], "arm64") + self.assertEqual(resource["process.runtime.name"], "nodejs") + for key in CONTENT_KEYS: + self.assertNotIn(key, a) + self.assertEqual(a["gen_ai.conversation.id"], "fixture-session-0001") + self.assertEqual(a["session.id"], "fixture-session-0001") + kinds.setdefault(name, set()).add(a.get("gen_ai.span.kind")) + # user_prompt / system_prompt: interactive mode's turn spans. + self.assertEqual(kinds, {"llm_call": {"LLM"}, "list_directory": {"TOOL"}, + "read_file": {"TOOL"}, "schedule_tool_calls": {"CHAIN"}, + "user_prompt": {"CHAIN"}, "system_prompt": {"CHAIN"}}) + raw = self.raw_spans(requests) + turn, follow_up = raw["6e5f4a3b2c1d0e01"], raw["6e5f4a3b2c1d0e02"] + self.assertEqual((turn["name"], follow_up["name"]), ("user_prompt", "system_prompt")) + self.assertFalse(turn.get("parentSpanId")) + self.assertEqual(follow_up["parentSpanId"], turn["spanId"]) + ok_llm = [attrs(s["attributes"]) for s in raw.values() + if s["name"] == "llm_call" and s["status"].get("code") == 1] + self.assertEqual(len(ok_llm), 2) + self.assertTrue(all(a["gen_ai.request.model"] == "gemini-3.8-flash" for a in ok_llm)) + self.assertTrue(all(int(a["gen_ai.usage.input_tokens"]) > 0 for a in ok_llm)) + # Errors stay ERROR; only the error class (plus the HTTP status of a + # Gemini API error) is kept, not the message or the stack trace. + api_error, tool_error = raw["9b8a7c6d5e4f3a21"], raw["9b8a7c6d5e4f3a22"] + self.assertEqual(api_error["status"], {"code": 2, "message": "ApiError HTTP 404"}) + self.assertEqual(tool_error["status"], {"code": 2, "message": "ENOENT"}) + for span, error_type in ((api_error, "ApiError"), (tool_error, "ENOENT")): + self.assertEqual([(e["name"], attrs(e["attributes"])) for e in span["events"]], + [("exception", {"exception.type": error_type})]) + for span in raw.values(): + if span["status"].get("code") != 2: + self.assertFalse(span["status"].get("message")) + + def test_opt_in_pipeline_keeps_messages(self): + # The documented opt-in: drop transform/strip-content from the pipeline. + text, requests = self.run_collector( + "service.pipelines.traces.processors=[transform/gemini-cli,batch]") + self.assertIn(MARKER + " gen_ai.input.messages", text) + self.assertIn(MARKER + " gen_ai.output.messages", text) + for _, name, a in self.spans(requests): + if name in ("user_prompt", "system_prompt"): + # Gemini CLI 0.64.0 records no content on its turn spans. + self.assertEqual(a["gen_ai.span.kind"], "CHAIN") + self.assertNotIn("input.value", a) + continue + self.assertIn("gen_ai.input.messages", a["input.value"]) + if "gen_ai.output.messages" in a: # a failed model call has no output + self.assertIn("gen_ai.output.messages", a["output.value"]) + # Error text is content: kept with the opt-in. Stack traces are not + # (install paths only), with or without it. + raw = self.raw_spans(requests) + self.assertIn("NOT_FOUND", raw["9b8a7c6d5e4f3a21"]["status"]["message"]) + self.assertIn("FIXTURE-CONTENT-MARKER.txt", raw["9b8a7c6d5e4f3a22"]["status"]["message"]) + event = attrs(raw["9b8a7c6d5e4f3a22"]["events"][0]["attributes"]) + self.assertEqual(sorted(event), ["exception.message", "exception.type"]) + self.assertNotIn("file://", text) + # The command line, executable paths, user and machine name are removed + # either way. (The opted-in tool error message above carries a path.) + self.assertNotIn("prompt on the command line", text) + self.assertNotIn("fixture-users-laptop", text) + self.assertNotIn(".nvm", text) + for resource, _, _ in self.spans(requests): + for key in LOCAL_IDENTITY: + self.assertNotIn(key, resource) + # The opt-in exports the system prompt and tool schemas (documented). + self.assertIn(MARKER + " gen_ai.system_instructions", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/gemini-cli-page.test.mjs b/scripts/gemini-cli-page.test.mjs new file mode 100644 index 000000000..8d8c6a4f6 --- /dev/null +++ b/scripts/gemini-cli-page.test.mjs @@ -0,0 +1,74 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { integrations } from '../src/lib/integrations.mjs'; + +const read = file => fs.readFileSync(new URL(`../${file}`, import.meta.url), 'utf8'); +const page = read('src/pages/docs/integrations/coding-agents/gemini-cli.mdx'); +const collector = read('examples/coding-agents/gemini-cli/otel-collector.yaml'); +const settings = JSON.parse(read('examples/coding-agents/gemini-cli/settings.json')); + +test('Gemini CLI is discoverable as native OTLP with explicit non-sandbox scope', () => { + const entry = integrations.find(item => item.id === 'gemini-cli'); + assert.equal(entry?.href, '/docs/integrations/coding-agents/gemini-cli'); + assert.equal(entry.category, 'coding'); + assert.equal(entry.method, 'Native OTLP'); + const prerequisites = page.split('## 1. Start the collector')[0]; + assert.match(prerequisites, /Gemini CLI 0\.64\.0 on macOS/); + assert.match(prerequisites, /non-sandbox interactive \(`gemini`\) and non-interactive \(`gemini -p`\) sessions/); + assert.match(prerequisites, /does not cover sandbox runtimes/); + assert.match(prerequisites, /@google\/gemini-cli@0\.64\.0/); + assert.match(prerequisites, /releases\/tag\/v0\.162\.0/); + assert.doesNotMatch(page, /run_e2e\.sh|CAPTURE_CONTENT|test_collector\.py|in our test|not tested|read 2026-|verified live/i); +}); + +test('Gemini CLI setup matches the shipped native settings and preserves privacy controls', () => { + const snippet = page.match(/```json\n([\s\S]*?)\n```/); + assert.ok(snippet, 'Include a complete telemetry settings example'); + assert.deepEqual(JSON.parse(snippet[1]), settings); + assert.equal(settings.telemetry.logPrompts, false); + assert.equal(settings.telemetry.traces, true); + assert.equal(settings.telemetry.otlpProtocol, 'http'); + assert.match(collector, /processors: \[transform\/gemini-cli, transform\/strip-content, batch\]/); + assert.match(collector, /compression: none/); + assert.match(collector, /endpoint: \$\{env:GEMINI_OTLP_LISTEN:-127\.0\.0\.1:4318\}/); + for (const key of [ + 'gen_ai.input.messages', 'gen_ai.output.messages', 'input.value', 'output.value', + 'gen_ai.system_instructions', 'gen_ai.tool.definitions', 'exception.stacktrace', + 'exception.message', 'process.command_args', 'process.command', + 'process.executable.path', 'process.executable.name', 'process.owner', 'host.name', 'host.id', + ]) assert.ok(collector.includes(`delete_key(attributes, "${key}")`), `Preserve filtering for ${key}`); + assert.match(page, /local collector receives attributes before filtering/); + assert.match(page, /Tool descriptions can come from your MCP servers/); + assert.match(page, /remove sensitive text from tool names and descriptions/i); + assert.match(page, /This also exports system instructions, `GEMINI\.md` memory, tool schemas and full error messages/); + assert.match(page, /Stack traces and the local identity resource fields remain removed/); + assert.match(page, /processors: \[transform\/gemini-cli, batch\]/); +}); + +test('Gemini CLI clears inherited endpoint overrides before launching with privacy filtering', () => { + const setup = page.split('## 2. Configure native telemetry')[1]?.split('## 3. Run a session')[0]; + assert.ok(setup, 'Keep endpoint cleanup in the setup before the first session'); + assert.match(setup, /```bash\nunset OTEL_EXPORTER_OTLP_ENDPOINT GEMINI_TELEMETRY_OTLP_ENDPOINT\n```/); + assert.match(setup, /terminal that will launch Gemini CLI/); + assert.match(setup, /remove conflicting `GEMINI_TELEMETRY_\*` environment overrides/); + assert.match(setup, /take precedence over `telemetry\.otlpEndpoint`/); + assert.match(setup, /bypass the local collector and its privacy filters/); + assert.match(setup, /even with `logPrompts: false`/); + const session = page.split('## 3. Run a session')[1]?.split('## 4. Inspect the session')[0]; + assert.ok(session, 'Include session launch instructions'); + assert.match(session, /In the terminal where you cleared the overrides/); +}); + +test('Gemini CLI explains session mapping and native trace limitations', () => { + assert.match(page, /copies `gen_ai\.conversation\.id` to `session\.id`/); + assert.match(page, /does not emit turn spans in this mode/); + assert.match(page, /returns an error instead of throwing/); + assert.match(page, /status `OK`/); + assert.match(page, /Gemini CLI has no whole-session span/); + assert.match(page, /Logs and metrics are not forwarded/); + assert.match(collector, /set\(attributes\["session.id"\], attributes\["gen_ai.conversation.id"\]\)/); + for (const operation of ['user_prompt', 'system_prompt', 'schedule_tool_calls']) { + assert.ok(collector.includes(`set(attributes["gen_ai.span.kind"], "CHAIN") where attributes["gen_ai.operation.name"] == "${operation}"`)); + } +}); diff --git a/scripts/integrations.test.mjs b/scripts/integrations.test.mjs index 30ff909ed..e838521d8 100644 --- a/scripts/integrations.test.mjs +++ b/scripts/integrations.test.mjs @@ -1,3 +1,4 @@ +import './gemini-cli-page.test.mjs'; import './ollama-java.test.mjs'; import './azure-openai-java.test.mjs'; import './langsmith-page.test.mjs'; diff --git a/src/lib/integrations.mjs b/src/lib/integrations.mjs index df0f390a2..c4e6eeb44 100644 --- a/src/lib/integrations.mjs +++ b/src/lib/integrations.mjs @@ -57,6 +57,7 @@ export const integrations = [ { id: 'gradio', name: 'Gradio', href: '/docs/integrations/traceai/gradio', category: 'frameworks', summary: 'Trace synchronous chat callbacks through their model client.', method: 'OpenAI client', logo: 'generic-framework' }, { id: 'z-ai', name: 'Z.ai', href: '/docs/integrations/providers/z-ai', category: 'providers', summary: 'Trace Python chat completions on the general-platform endpoint.', method: 'OpenAI client', logo: 'generic-model' }, { id: 'cursor', name: 'Cursor IDE', href: '/docs/integrations/coding-agents/cursor', category: 'coding', summary: 'Trace macOS IDE agent sessions through observation hooks.', method: 'Hook-to-OTLP', logo: 'generic-framework' }, + { id: 'gemini-cli', name: 'Gemini CLI', href: '/docs/integrations/coding-agents/gemini-cli', category: 'coding', summary: 'Trace non-sandbox CLI sessions through a privacy-filtering collector.', method: 'Native OTLP', logo: 'google' }, { id: 'langsmith', name: 'LangSmith', href: '/docs/integrations/langsmith', category: 'frameworks', summary: 'Forward supported LangSmith SDK traces over OTLP.', method: 'Native OTLP', logo: 'generic-framework' }, {"id": "mlflow", "name": "MLflow", "href": "/docs/integrations/mlflow", "category": "frameworks", "summary": "Export current MLflow traces with OTLP/HTTP.", "method": "Native OTLP", "logo": "generic-framework"}, { id: 'openai', name: 'OpenAI', href: '/docs/integrations/traceai/openai', category: 'providers', summary: 'Instrument OpenAI client calls in your application.', method: 'traceAI SDK', logo: 'openai' }, diff --git a/src/pages/docs/integrations/coding-agents/gemini-cli.mdx b/src/pages/docs/integrations/coding-agents/gemini-cli.mdx new file mode 100644 index 000000000..c9dcd2e9a --- /dev/null +++ b/src/pages/docs/integrations/coding-agents/gemini-cli.mdx @@ -0,0 +1,148 @@ +--- +title: "Gemini CLI session tracing with Future AGI" +description: "Send Gemini CLI's native OpenTelemetry traces through a local collector to inspect model calls, tools, tokens and sessions in Future AGI." +--- + +Trace Gemini CLI model calls, tool calls and interactive turns in Future AGI using its [native OpenTelemetry export](https://geminicli.com/docs/cli/telemetry/) and a local OpenTelemetry Collector. + +## Prerequisites + +- **Gemini CLI 0.64.0 on macOS**, authenticated with your own Gemini account or API key, and Node.js 20 or newer. This guide covers **non-sandbox interactive (`gemini`) and non-interactive (`gemini -p`) sessions**. The loopback collector setup does not cover sandbox runtimes. +- **[OpenTelemetry Collector Contrib 0.162.0](https://github.com/open-telemetry/opentelemetry-collector-releases/releases/tag/v0.162.0)**, with `otelcol-contrib` on your PATH. Use the contrib distribution: the configuration requires its transform processor. +- **Future AGI API and secret keys** for the workspace where you want to collect traces. + +Install the version used by this guide: + +```bash +npm install -g @google/gemini-cli@0.64.0 +``` + +This integration traces the Gemini CLI application, not your own Gemini API code. For application SDK calls, use [Google GenAI](/docs/integrations/traceai/google_genai) or [Vertex AI](/docs/integrations/traceai/vertexai). + +## 1. Start the collector + +Download the collector configuration and start it with your Future AGI credentials: + +```bash +curl -fsSLO https://raw.githubusercontent.com/future-agi/docs/main/examples/coding-agents/gemini-cli/otel-collector.yaml +export FI_API_KEY="your-futureagi-api-key" +export FI_SECRET_KEY="your-futureagi-secret-key" +export FI_PROJECT_NAME="gemini-cli" +otelcol-contrib --config otel-collector.yaml +``` + +Keep this terminal open while using Gemini CLI. The collector listens on `127.0.0.1:4318` and forwards traces to `https://api.futureagi.com/tracer/v1/traces`. For self-hosted Future AGI, set `FI_BASE_URL` to your API base URL before starting it. + +The collector adds authentication, maps span kinds and session IDs, drops logs and metrics, and removes content before forwarding traces. Keep both transform processors in the default pipeline; Gemini CLI's `logPrompts: false` setting alone does not remove system instructions, tool schemas or error text. See [Content and privacy](#content-and-privacy). + +## 2. Configure native telemetry + +Merge this `telemetry` block into `~/.gemini/settings.json` for all projects, or `.gemini/settings.json` for one project. Preserve your other settings: + +```json +{ + "telemetry": { + "enabled": true, + "target": "local", + "otlpEndpoint": "http://127.0.0.1:4318", + "otlpProtocol": "http", + "traces": true, + "logPrompts": false + } +} +``` + +- Keep `traces: true` for input and output token counts. Turning it off is **not** a privacy control: Gemini CLI 0.64.0 still emits spans with model, tool and system-instruction attributes. +- Set `logPrompts: false` explicitly; Gemini CLI defaults to `true`. +- Do not set `outfile`, which replaces OTLP export, or `target: "gcp"`, which routes telemetry to Google Cloud. + +Use the settings file for this setup. In a separate terminal that will launch Gemini CLI, remove conflicting `GEMINI_TELEMETRY_*` environment overrides and clear both endpoint overrides before starting a session: + +```bash +unset OTEL_EXPORTER_OTLP_ENDPOINT GEMINI_TELEMETRY_OTLP_ENDPOINT +``` + +Both endpoint variables take precedence over `telemetry.otlpEndpoint` and can bypass the local collector and its privacy filters, exporting content even with `logPrompts: false`. Repeat this cleanup in each terminal used to launch Gemini CLI. + +To stop exporting, set `telemetry.enabled` to `false` in the file and restart Gemini CLI. + +Optionally set `OTEL_NODE_RESOURCE_DETECTORS=env` in the shell that runs Gemini CLI. This prevents Node's resource detectors from adding the command line, executable paths, OS user name and machine name. The collector strips those fields either way, including when content capture is enabled. + +## 3. Run a session + +In the terminal where you cleared the overrides, open a project you trust and start an interactive session: + +```bash +gemini +``` + +Ask **"List the files in this folder and read README.md"**, and approve the tool calls you want to run. Alternatively, run the same task non-interactively: + +```bash +gemini -p "List the files in this folder and read README.md" +``` + +Keep Gemini CLI and the collector on the same host, outside a sandbox. Exit Gemini CLI or wait a few seconds for batched spans to arrive. + +## 4. Inspect the session + +Open **Observe** in Future AGI and select the project named by `FI_PROJECT_NAME` (`gemini-cli` above). Use **Sessions** to view a whole Gemini CLI session; the collector copies `gen_ai.conversation.id` to `session.id` to group its traces. + +In the **Trace** tab, the trace shape depends on the mode: + +- **Interactive:** a `user_prompt` span groups a turn's model and tool spans. A `system_prompt` child contains follow-up model calls after tools return. Despite its name, that span does not contain the system prompt. It starts when `user_prompt` ends, so include both when inspecting a turn's duration. +- **Non-interactive (`gemini -p`):** each `llm_call` and each `schedule_tool_calls` batch is a separate trace. Gemini CLI does not emit turn spans in this mode. + +In Gemini CLI 0.64.0, a prompt queued during a running turn can nest under that turn instead of starting a new trace. Typed slash commands can create empty `user_prompt` traces. + +### Captured fields + +| Gemini CLI operation | Future AGI span kind | Fields | +| --- | --- | --- | +| `user_prompt`, `system_prompt` | `CHAIN` | Interactive turn structure, duration, status; no input or output, even with content capture | +| `llm_call` | `LLM` | Model, input/output token counts, prompt ID, duration, status | +| `schedule_tool_calls` | `CHAIN` | Parent of the tool calls requested in one model response | +| `tool_call` | `TOOL` | Tool name, call ID, description, duration, status; the collector names the span after the tool | +| `agent_call` | `AGENT` | Agent operation; Gemini CLI 0.64.0 records `gemini-cli` rather than the individual subagent name | + +### Compatibility limits + +- Gemini CLI has no whole-session span. Use **Sessions**, rather than expecting one trace for the entire session. +- A tool that **returns an error instead of throwing** can end with status `OK` in Gemini CLI 0.64.0. Only a thrown exception marks its span `ERROR`; returned error details require content capture. +- Spans include input and output tokens, not cached, thinking or tool-use token counts. Gemini CLI does not export cost; Future AGI can calculate it only when pricing is available for the model. +- Logs and metrics are not forwarded by this configuration. + +## Content and privacy + +By default, Future AGI receives model names, token counts, tool names and descriptions, call IDs, durations, statuses, error classes and session IDs. **Tool descriptions can come from your MCP servers**; remove sensitive text from tool names and descriptions before using them. + +The collector removes prompt and response messages, tool arguments and results, system instructions (including `GEMINI.md` memory and local paths), tool schemas and full error messages. Failed spans retain `ERROR` and an error class, plus an HTTP status for Gemini API errors, such as `ApiError HTTP 503`. Command lines, executable paths, OS user names, machine names and exception stack traces are stripped even with content capture enabled. + +The local collector receives attributes before filtering them. Keep its receiver on loopback and do not add a debug or file exporter to this pipeline if you need to avoid recording that content locally. + +To opt in to content capture: + +1. Set `"logPrompts": true` in the telemetry settings. +2. Remove `transform/strip-content` from the collector's traces pipeline, leaving: + + ```yaml + processors: [transform/gemini-cli, batch] + ``` + +3. Restart the collector and Gemini CLI. + +Model messages and tool arguments/results then appear as input and output in Future AGI. Gemini CLI caps each message attribute at 10,000 characters. **This also exports system instructions, `GEMINI.md` memory, tool schemas and full error messages**, which can contain file paths or other sensitive data. Turn spans (`user_prompt` and `system_prompt`) still have no input or output. Stack traces and the local identity resource fields remain removed. + +## Troubleshooting + +**No traces appear.** Check that the collector is running, `telemetry.enabled` is `true`, and `otlpProtocol` is `"http"`. Gemini CLI posts to `/v1/traces`; this collector does not accept gRPC. Do not combine `useCollector` with `useCliAuth`, which disables telemetry in this version. + +**Port 4318 is already in use.** Set `GEMINI_OTLP_LISTEN=127.0.0.1:24318` before starting the collector and change Gemini CLI's `otlpEndpoint` to `http://127.0.0.1:24318`. + +**The collector reports an export error.** For `401` or `403`, check your Future AGI keys. For `400`, keep `compression: none` in the exporter: the Future AGI endpoint requires uncompressed OTLP requests for this setup. + +**Model spans have no token counts.** Set `telemetry.traces` to `true` and restart Gemini CLI. + +**Span kinds are missing or tool spans are still named `tool_call`.** Send Gemini CLI's export through the provided collector, with `transform/gemini-cli` in its traces pipeline, rather than directly to Future AGI. + +**Prompts appear without an intentional opt-in.** Set `logPrompts` to `false` and restore `transform/strip-content` in the collector pipeline, then restart both processes.