Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions .github/workflows/prismml-page.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
name: PrismML page contracts

on:
pull_request:
paths:
- 'src/pages/docs/integrations/providers/prismml.mdx'
- 'scripts/test_prismml_page.py'
- 'scripts/prismml-page.test.mjs'
- '.github/workflows/prismml-page.yml'

permissions:
contents: read

jobs:
page:
runs-on: ubuntu-latest
timeout-minutes: 10
strategy:
fail-fast: false
matrix:
include:
- python: '3.10'
openai: '1.69.0'
- python: '3.13'
openai: '3.42.1'
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: Install published dependencies
run: python -m pip install 'openai==${{ matrix.openai }}' 'traceAI-openai==0.1.12' 'fi-instrumentation-otel==1.2.2' pytest
- name: Check dependency consistency
run: python -m pip check
- name: Execute the exact guide with loopback model and OTLP servers
run: python -m pytest scripts/test_prismml_page.py -q
1 change: 1 addition & 0 deletions scripts/integrations.test.mjs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import './prismml-page.test.mjs';
import './langsmith-page.test.mjs';
import test from 'node:test';
import assert from 'node:assert/strict';
Expand Down
29 changes: 29 additions & 0 deletions scripts/prismml-page.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import { integrations, integrationSearchText } from '../src/lib/integrations.mjs';

const page = fs.readFileSync(new URL('../src/pages/docs/integrations/providers/prismml.mdx', import.meta.url), 'utf8');

test('PrismML is discoverable as a local OpenAI-client integration', () => {
const item = integrations.find(entry => entry.id === 'prismml');
assert.equal(item.href, '/docs/integrations/providers/prismml');
assert.equal(item.method, 'OpenAI client');
assert.equal(item.category, 'providers');
for (const term of ['bonsai', 'prismml', 'llama.cpp']) {
assert.ok(integrationSearchText(item).includes(term));
}
});

test('PrismML guide uses released packages and keeps local model and trace export separate', () => {
for (const text of [
'openai==3.42.1', 'traceAI-openai==0.1.12', 'fi-instrumentation-otel==1.2.2',
'http://localhost:8080/v1', 'https://api.futureagi.com/tracer/v1/traces',
'FI_API_KEY', 'FI_SECRET_KEY', 'FI_PROJECT_NAME', 'PRISMML_BASE_URL',
'OpenAIInstrumentor().instrument(tracer_provider=provider)', 'provider.force_flush()',
'FI_HIDE_INPUTS=true', 'FI_HIDE_OUTPUTS=true', 'FI_HIDE_LLM_INVOCATION_PARAMETERS=true',
'--alias bonsai', 'absolute path', 'does not authenticate requests',
]) assert.ok(page.includes(text), text);
assert.doesNotMatch(page, /verified live|verification|fixture|test dates|QA|span\.set_attribute|class .*Exporter/i);
assert.equal((page.match(/```python/g) || []).length, 1);
});
144 changes: 144 additions & 0 deletions scripts/test_prismml_page.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
"""Execute the published PrismML guide against a loopback model and OTLP receiver."""

import gzip
import json
import os
from pathlib import Path
import re
import socket
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer

from google.protobuf.json_format import MessageToDict
import openai
from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest
import pytest
from traceai_openai import OpenAIInstrumentor

PAGE = Path(__file__).resolve().parents[1] / "src/pages/docs/integrations/providers/prismml.mdx"


@pytest.mark.parametrize("mode", ["chat", "hidden", "error", "cancel"])
def test_exact_page_code(monkeypatch, capsys, mode):
for name in list(os.environ):
if name.startswith(("FI_", "OTEL_", "OPENAI_", "PRISMML_")) or name.lower() in (
"http_proxy", "https_proxy", "all_proxy", "no_proxy",
):
monkeypatch.delenv(name)
monkeypatch.setenv("NO_PROXY", "*")
original_connect = socket.socket.connect

def local_only(sock, address):
if sock.family in (socket.AF_INET, socket.AF_INET6):
assert address[0] in ("127.0.0.1", "::1")
return original_connect(sock, address)

monkeypatch.setattr(socket.socket, "connect", local_only)
calls, exports = [], []

class Handler(BaseHTTPRequestHandler):
def log_message(self, *args):
pass

def do_POST(self):
body = self.rfile.read(int(self.headers["Content-Length"]))
headers = {key.lower(): value for key, value in self.headers.items()}
if self.path == "/tracer/v1/traces":
if headers.get("content-encoding") == "gzip":
body = gzip.decompress(body)
exports.append((headers, MessageToDict(ExportTraceServiceRequest.FromString(body))))
self.send_response(200)
self.end_headers()
self.wfile.write(b"")
return
calls.append((self.path, headers, json.loads(body)))
if mode == "error":
payload = {"error": {"message": "Synthetic rejected role", "type": "invalid_request_error"}}
else:
payload = {
"id": "synthetic-bonsai", "object": "chat.completion", "created": 1,
"model": "bonsai-loaded",
"choices": [{"index": 0, "finish_reason": "stop", "message": {
"role": "assistant", "content": "Paris is the capital of France.",
}}],
"usage": {"prompt_tokens": 10, "completion_tokens": 7, "total_tokens": 17},
}
self.send_response(400 if mode == "error" else 200)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps(payload).encode())

server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
origin = f"http://127.0.0.1:{server.server_port}"
monkeypatch.setenv("PRISMML_BASE_URL", origin + "/v1")
monkeypatch.setenv("FI_BASE_URL", origin)
monkeypatch.setenv("FI_API_KEY", "synthetic-fi-key")
monkeypatch.setenv("FI_SECRET_KEY", "synthetic-fi-secret")
for name in ("FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_HIDE_LLM_INVOCATION_PARAMETERS"):
monkeypatch.setenv(name, str(mode == "hidden").lower())
code = re.search(r"```python\n(.*?)```", PAGE.read_text(), re.S).group(1)
namespace = {}
interruption = KeyboardInterrupt("synthetic guide cancellation")
if mode == "cancel":
# Interrupt the SDK's default HTTP client, not the instrumented method.
# This supports both httpx (floor) and httpx2 (latest SDK).
# The guide's finally/force_flush must export one ended ERROR span.
def interrupted_send(self, request, **kwargs):
calls.append((request.url.path, dict(request.headers), json.loads(request.content)))
raise interruption

monkeypatch.setattr(openai.DefaultHttpxClient, "send", interrupted_send)
try:
if mode == "cancel":
with pytest.raises(KeyboardInterrupt) as caught:
exec(compile(code, str(PAGE), "exec"), namespace)
assert caught.value is interruption
elif mode == "error":
with pytest.raises(openai.BadRequestError):
exec(compile(code, str(PAGE), "exec"), namespace)
else:
exec(compile(code, str(PAGE), "exec"), namespace)
assert capsys.readouterr().out.strip() == "Paris is the capital of France."
assert len(calls) == len(exports) == 1
path, headers, body = calls[0]
assert path == "/v1/chat/completions"
assert headers["authorization"] == "Bearer not-needed"
assert "x-api-key" not in headers and "x-secret-key" not in headers
assert "synthetic-fi-" not in json.dumps(calls)
assert body["model"] == "bonsai"
export_headers, payload = exports[0]
assert export_headers["x-api-key"] == "synthetic-fi-key"
assert export_headers["x-secret-key"] == "synthetic-fi-secret"
resource = payload["resourceSpans"][0]
def attrs(items):
return {item["key"]: next(iter(item["value"].values())) for item in items}

assert attrs(resource["resource"]["attributes"])["project_name"] == "prismml-bonsai"
assert attrs(resource["resource"]["attributes"])["project_type"] == "observe"
spans = [span for rs in payload["resourceSpans"] for scope in rs["scopeSpans"] for span in scope["spans"]]
assert len(spans) == 1
span = spans[0]
assert span["name"] == "ChatCompletion"
attributes = attrs(span["attributes"])
assert attributes["gen_ai.span.kind"] == "LLM"
assert attributes["gen_ai.provider.name"] == "openai"
assert attributes["gen_ai.request.model"] == ("bonsai" if mode in ("error", "cancel") else "bonsai-loaded")
assert "not-needed" not in json.dumps(payload)
if mode in ("error", "cancel"):
assert span["status"]["code"] == "STATUS_CODE_ERROR"
assert any(event["name"] == "exception" for event in span["events"])
else:
assert int(attributes["gen_ai.usage.total_tokens"]) == 17
assert attributes["output.value"] == ("__REDACTED__" if mode == "hidden" else "Paris is the capital of France.")
if mode == "hidden":
assert "capital of France" not in json.dumps(payload)
assert "gen_ai.request.parameters" not in attributes
finally:
OpenAIInstrumentor().uninstrument()
if "provider" in namespace:
namespace["provider"].shutdown()
server.shutdown()
server.server_close()
thread.join(timeout=5)
1 change: 1 addition & 0 deletions src/lib/integrations.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ export const integrations = [
{ id: 'groq', name: 'Groq', href: '/docs/integrations/traceai/groq', category: 'providers', summary: 'Capture model calls made with the Groq SDK.', method: 'traceAI SDK', logo: 'groq' },
{ id: 'mistralai', name: 'Mistral AI', href: '/docs/integrations/traceai/mistralai', category: 'providers', summary: 'Instrument Mistral model requests and responses.', method: 'traceAI SDK', logo: 'mistral' },
{ id: 'togetherai', name: 'Together AI', href: '/docs/integrations/traceai/togetherai', category: 'providers', summary: 'Trace Together AI through its OpenAI-compatible API.', method: 'OpenAI client', logo: 'together' },
{ id: 'prismml', name: 'PrismML Bonsai', href: '/docs/integrations/providers/prismml', category: 'providers', summary: 'Trace local Bonsai chat completions through the OpenAI client.', method: 'OpenAI client', logo: 'generic-model', aliases: ['Bonsai', 'llama.cpp', 'PrismML'] },
{ id: 'ollama', name: 'Ollama', href: '/docs/integrations/traceai/ollama', category: 'providers', summary: 'Trace local model calls through the OpenAI client.', method: 'OpenAI client', logo: 'ollama' },
{ id: 'cometapi', name: 'CometAPI', href: '/docs/integrations/providers/cometapi', category: 'providers', summary: 'Trace chat completions through the OpenAI client.', method: 'OpenAI client', logo: 'cometapi' },
{ id: 'novita-ai', name: 'Novita AI', href: '/docs/integrations/providers/novita-ai', category: 'providers', summary: 'Trace Novita chat completions from your Python app.', method: 'OpenAI client', logo: 'novita' },
Expand Down
112 changes: 112 additions & 0 deletions src/pages/docs/integrations/providers/prismml.mdx
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
---
title: "Trace PrismML Bonsai"
description: "Trace local PrismML Bonsai chat completions in Future AGI with the OpenAI Python client, including model names, token usage and privacy controls."
---

Trace calls to your local [PrismML Bonsai server](https://docs.prismml.com/run/server) using the maintained [traceAI OpenAI integration](/docs/integrations/traceai/openai). This guide covers your Python client's Chat Completions requests, not the server's built-in web chat or other products named Prism.

## Install

Use Python 3.10–3.13:

```bash
python -m pip install 'openai==3.42.1' 'traceAI-openai==0.1.12' 'fi-instrumentation-otel==1.2.2'
```

## Start the local server

Follow the [PrismML setup](https://docs.prismml.com/run/server) to download a model. In the Bonsai demo directory, start the llama.cpp server with a stable model alias:

```bash
./scripts/start_llama_server.sh --alias bonsai
```

The API base URL is `http://localhost:8080/v1`. The SDK appends `/chat/completions`; do not include that suffix in the base URL.

<Warning>
The local server does not authenticate requests. Keep it on loopback, or restrict access with a trusted network and firewall. The `not-needed` key below is only a placeholder required by the OpenAI SDK, not a Future AGI credential.
</Warning>

On Apple Silicon, PrismML also provides `./scripts/start_mlx_server.sh` at `http://localhost:8081/v1`. Follow the [MLX sampling and thinking settings](https://docs.prismml.com/run/server), which must be supplied in each request rather than inherited from the llama.cpp launcher.

## Configure tracing

Set your Future AGI credentials separately from the model server address:

```bash
export FI_API_KEY="YOUR_FUTURE_AGI_API_KEY"
export FI_SECRET_KEY="YOUR_FUTURE_AGI_SECRET_KEY"
export FI_PROJECT_NAME="prismml-bonsai"
export PRISMML_BASE_URL="http://localhost:8080/v1"
export PRISMML_MODEL="bonsai"
```

Trace export goes to `https://api.futureagi.com/tracer/v1/traces` by default. If you self-host Future AGI, set `FI_BASE_URL` to its origin, without `/tracer/v1/traces`. Do not point it at the Bonsai server.

## Run a completion

Save this as `prismml_chat.py`. Instrument the OpenAI SDK once, before creating the client; do not layer another OpenAI instrumentor over the same requests.

```python
import os

from fi_instrumentation import register
from fi_instrumentation.fi_types import ProjectType
from openai import OpenAI
from traceai_openai import OpenAIInstrumentor

provider = register(
project_name=os.environ.get("FI_PROJECT_NAME") or "prismml-bonsai",
project_type=ProjectType.OBSERVE,
set_global_tracer_provider=False,
verbose=False,
)
OpenAIInstrumentor().instrument(tracer_provider=provider)
try:
with OpenAI(
base_url=os.environ.get("PRISMML_BASE_URL") or "http://localhost:8080/v1",
api_key=os.environ.get("PRISMML_API_KEY") or "not-needed",
) as client:
response = client.chat.completions.create(
model=os.environ.get("PRISMML_MODEL") or "bonsai",
messages=[{"role": "user", "content": "What is the capital of France?"}],
)
print(response.choices[0].message.content or "")
finally:
provider.force_flush()
```

```bash
python prismml_chat.py
```

`PRISMML_API_KEY` is optional and defaults to `not-needed`. The script prints the answer and flushes the trace before exiting.

## View the result

Open **Observe** in Future AGI and select `prismml-bonsai`, or the project set in `FI_PROJECT_NAME`. Each completion produces one `ChatCompletion` LLM span with the prompt, answer, latency and token counts supplied by the server. The provider label is `openai` because this local endpoint uses the shared OpenAI instrumentor.

Non-streamed calls record the server's returned model name in `gen_ai.request.model`. For streamed and failed calls, that field contains the requested name. Local models may have no matching price entry, so cost can be unavailable even when token counts are present.

For a ready-to-run streaming client, use the [PrismML recipe](https://github.com/future-agi/traceAI/tree/dev/python/examples/prismml). If its stream ends without a finish reason, the command exits with status 1; treat any printed text as partial and retry. Streamed token counts require both `stream_options={"include_usage": True}` and a final usage chunk from the server; missing counts are not replaced with zero.

## Privacy

Local model execution does not mean local-only telemetry: prompts and answers are exported to Future AGI by default. Set these before starting the script to hide content and invocation parameters:

```bash
export FI_HIDE_INPUTS=true
export FI_HIDE_OUTPUTS=true
export FI_HIDE_LLM_INVOCATION_PARAMETERS=true
```

Hide both inputs and outputs when the answer might echo sensitive input. These flags do not change what the local server receives or logs.

The model name remains visible metadata. Without an alias, llama-server can return the model file's absolute path, including your user name or directory layout. Use `--alias bonsai` to keep that path out of traces.

## Troubleshooting

- **Connection refused:** start the server and check the port: 8080 for llama.cpp, 8081 for MLX.
- **404 from the server:** use the API base URL ending in `/v1`, not the web chat root or the full completion URL.
- **No traces:** check your Future AGI keys, project name and export origin. See [No traces appearing](/docs/observe/troubleshooting/no-traces-appearing).
- **A local path appears as the model:** restart llama-server with `--alias bonsai`.
Loading