diff --git a/.github/workflows/prismml-page.yml b/.github/workflows/prismml-page.yml new file mode 100644 index 000000000..cb9b59796 --- /dev/null +++ b/.github/workflows/prismml-page.yml @@ -0,0 +1,38 @@ +name: PrismML page contracts + +on: + pull_request: + paths: + - 'src/pages/docs/integrations/providers/prismml.mdx' + - 'scripts/test_prismml_page.py' + - 'scripts/prismml-page.test.mjs' + - '.github/workflows/prismml-page.yml' + +permissions: + contents: read + +jobs: + page: + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + include: + - python: '3.10' + openai: '1.69.0' + - python: '3.13' + openai: '3.42.1' + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install published dependencies + run: python -m pip install 'openai==${{ matrix.openai }}' 'traceAI-openai==0.1.12' 'fi-instrumentation-otel==1.2.2' pytest + - name: Check dependency consistency + run: python -m pip check + - name: Execute the exact guide with loopback model and OTLP servers + run: python -m pytest scripts/test_prismml_page.py -q diff --git a/scripts/integrations.test.mjs b/scripts/integrations.test.mjs index 848838092..4b0429c91 100644 --- a/scripts/integrations.test.mjs +++ b/scripts/integrations.test.mjs @@ -1,3 +1,4 @@ +import './prismml-page.test.mjs'; import './langsmith-page.test.mjs'; import test from 'node:test'; import assert from 'node:assert/strict'; diff --git a/scripts/prismml-page.test.mjs b/scripts/prismml-page.test.mjs new file mode 100644 index 000000000..a50b6b797 --- /dev/null +++ b/scripts/prismml-page.test.mjs @@ -0,0 +1,29 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { integrations, integrationSearchText } from '../src/lib/integrations.mjs'; + +const page = fs.readFileSync(new URL('../src/pages/docs/integrations/providers/prismml.mdx', import.meta.url), 'utf8'); + +test('PrismML is discoverable as a local OpenAI-client integration', () => { + const item = integrations.find(entry => entry.id === 'prismml'); + assert.equal(item.href, '/docs/integrations/providers/prismml'); + assert.equal(item.method, 'OpenAI client'); + assert.equal(item.category, 'providers'); + for (const term of ['bonsai', 'prismml', 'llama.cpp']) { + assert.ok(integrationSearchText(item).includes(term)); + } +}); + +test('PrismML guide uses released packages and keeps local model and trace export separate', () => { + for (const text of [ + 'openai==3.42.1', 'traceAI-openai==0.1.12', 'fi-instrumentation-otel==1.2.2', + 'http://localhost:8080/v1', 'https://api.futureagi.com/tracer/v1/traces', + 'FI_API_KEY', 'FI_SECRET_KEY', 'FI_PROJECT_NAME', 'PRISMML_BASE_URL', + 'OpenAIInstrumentor().instrument(tracer_provider=provider)', 'provider.force_flush()', + 'FI_HIDE_INPUTS=true', 'FI_HIDE_OUTPUTS=true', 'FI_HIDE_LLM_INVOCATION_PARAMETERS=true', + '--alias bonsai', 'absolute path', 'does not authenticate requests', + ]) assert.ok(page.includes(text), text); + assert.doesNotMatch(page, /verified live|verification|fixture|test dates|QA|span\.set_attribute|class .*Exporter/i); + assert.equal((page.match(/```python/g) || []).length, 1); +}); diff --git a/scripts/test_prismml_page.py b/scripts/test_prismml_page.py new file mode 100644 index 000000000..638057f19 --- /dev/null +++ b/scripts/test_prismml_page.py @@ -0,0 +1,144 @@ +"""Execute the published PrismML guide against a loopback model and OTLP receiver.""" + +import gzip +import json +import os +from pathlib import Path +import re +import socket +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from google.protobuf.json_format import MessageToDict +import openai +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +import pytest +from traceai_openai import OpenAIInstrumentor + +PAGE = Path(__file__).resolve().parents[1] / "src/pages/docs/integrations/providers/prismml.mdx" + + +@pytest.mark.parametrize("mode", ["chat", "hidden", "error", "cancel"]) +def test_exact_page_code(monkeypatch, capsys, mode): + for name in list(os.environ): + if name.startswith(("FI_", "OTEL_", "OPENAI_", "PRISMML_")) or name.lower() in ( + "http_proxy", "https_proxy", "all_proxy", "no_proxy", + ): + monkeypatch.delenv(name) + monkeypatch.setenv("NO_PROXY", "*") + original_connect = socket.socket.connect + + def local_only(sock, address): + if sock.family in (socket.AF_INET, socket.AF_INET6): + assert address[0] in ("127.0.0.1", "::1") + return original_connect(sock, address) + + monkeypatch.setattr(socket.socket, "connect", local_only) + calls, exports = [], [] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + body = self.rfile.read(int(self.headers["Content-Length"])) + headers = {key.lower(): value for key, value in self.headers.items()} + if self.path == "/tracer/v1/traces": + if headers.get("content-encoding") == "gzip": + body = gzip.decompress(body) + exports.append((headers, MessageToDict(ExportTraceServiceRequest.FromString(body)))) + self.send_response(200) + self.end_headers() + self.wfile.write(b"") + return + calls.append((self.path, headers, json.loads(body))) + if mode == "error": + payload = {"error": {"message": "Synthetic rejected role", "type": "invalid_request_error"}} + else: + payload = { + "id": "synthetic-bonsai", "object": "chat.completion", "created": 1, + "model": "bonsai-loaded", + "choices": [{"index": 0, "finish_reason": "stop", "message": { + "role": "assistant", "content": "Paris is the capital of France.", + }}], + "usage": {"prompt_tokens": 10, "completion_tokens": 7, "total_tokens": 17}, + } + self.send_response(400 if mode == "error" else 200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(payload).encode()) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + origin = f"http://127.0.0.1:{server.server_port}" + monkeypatch.setenv("PRISMML_BASE_URL", origin + "/v1") + monkeypatch.setenv("FI_BASE_URL", origin) + monkeypatch.setenv("FI_API_KEY", "synthetic-fi-key") + monkeypatch.setenv("FI_SECRET_KEY", "synthetic-fi-secret") + for name in ("FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_HIDE_LLM_INVOCATION_PARAMETERS"): + monkeypatch.setenv(name, str(mode == "hidden").lower()) + code = re.search(r"```python\n(.*?)```", PAGE.read_text(), re.S).group(1) + namespace = {} + interruption = KeyboardInterrupt("synthetic guide cancellation") + if mode == "cancel": + # Interrupt the SDK's default HTTP client, not the instrumented method. + # This supports both httpx (floor) and httpx2 (latest SDK). + # The guide's finally/force_flush must export one ended ERROR span. + def interrupted_send(self, request, **kwargs): + calls.append((request.url.path, dict(request.headers), json.loads(request.content))) + raise interruption + + monkeypatch.setattr(openai.DefaultHttpxClient, "send", interrupted_send) + try: + if mode == "cancel": + with pytest.raises(KeyboardInterrupt) as caught: + exec(compile(code, str(PAGE), "exec"), namespace) + assert caught.value is interruption + elif mode == "error": + with pytest.raises(openai.BadRequestError): + exec(compile(code, str(PAGE), "exec"), namespace) + else: + exec(compile(code, str(PAGE), "exec"), namespace) + assert capsys.readouterr().out.strip() == "Paris is the capital of France." + assert len(calls) == len(exports) == 1 + path, headers, body = calls[0] + assert path == "/v1/chat/completions" + assert headers["authorization"] == "Bearer not-needed" + assert "x-api-key" not in headers and "x-secret-key" not in headers + assert "synthetic-fi-" not in json.dumps(calls) + assert body["model"] == "bonsai" + export_headers, payload = exports[0] + assert export_headers["x-api-key"] == "synthetic-fi-key" + assert export_headers["x-secret-key"] == "synthetic-fi-secret" + resource = payload["resourceSpans"][0] + def attrs(items): + return {item["key"]: next(iter(item["value"].values())) for item in items} + + assert attrs(resource["resource"]["attributes"])["project_name"] == "prismml-bonsai" + assert attrs(resource["resource"]["attributes"])["project_type"] == "observe" + spans = [span for rs in payload["resourceSpans"] for scope in rs["scopeSpans"] for span in scope["spans"]] + assert len(spans) == 1 + span = spans[0] + assert span["name"] == "ChatCompletion" + attributes = attrs(span["attributes"]) + assert attributes["gen_ai.span.kind"] == "LLM" + assert attributes["gen_ai.provider.name"] == "openai" + assert attributes["gen_ai.request.model"] == ("bonsai" if mode in ("error", "cancel") else "bonsai-loaded") + assert "not-needed" not in json.dumps(payload) + if mode in ("error", "cancel"): + assert span["status"]["code"] == "STATUS_CODE_ERROR" + assert any(event["name"] == "exception" for event in span["events"]) + else: + assert int(attributes["gen_ai.usage.total_tokens"]) == 17 + assert attributes["output.value"] == ("__REDACTED__" if mode == "hidden" else "Paris is the capital of France.") + if mode == "hidden": + assert "capital of France" not in json.dumps(payload) + assert "gen_ai.request.parameters" not in attributes + finally: + OpenAIInstrumentor().uninstrument() + if "provider" in namespace: + namespace["provider"].shutdown() + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/src/lib/integrations.mjs b/src/lib/integrations.mjs index c3868743d..b2362066d 100644 --- a/src/lib/integrations.mjs +++ b/src/lib/integrations.mjs @@ -63,6 +63,7 @@ export const integrations = [ { id: 'groq', name: 'Groq', href: '/docs/integrations/traceai/groq', category: 'providers', summary: 'Capture model calls made with the Groq SDK.', method: 'traceAI SDK', logo: 'groq' }, { id: 'mistralai', name: 'Mistral AI', href: '/docs/integrations/traceai/mistralai', category: 'providers', summary: 'Instrument Mistral model requests and responses.', method: 'traceAI SDK', logo: 'mistral' }, { id: 'togetherai', name: 'Together AI', href: '/docs/integrations/traceai/togetherai', category: 'providers', summary: 'Trace Together AI through its OpenAI-compatible API.', method: 'OpenAI client', logo: 'together' }, + { id: 'prismml', name: 'PrismML Bonsai', href: '/docs/integrations/providers/prismml', category: 'providers', summary: 'Trace local Bonsai chat completions through the OpenAI client.', method: 'OpenAI client', logo: 'generic-model', aliases: ['Bonsai', 'llama.cpp', 'PrismML'] }, { id: 'ollama', name: 'Ollama', href: '/docs/integrations/traceai/ollama', category: 'providers', summary: 'Trace local model calls through the OpenAI client.', method: 'OpenAI client', logo: 'ollama' }, { id: 'cometapi', name: 'CometAPI', href: '/docs/integrations/providers/cometapi', category: 'providers', summary: 'Trace chat completions through the OpenAI client.', method: 'OpenAI client', logo: 'cometapi' }, { id: 'novita-ai', name: 'Novita AI', href: '/docs/integrations/providers/novita-ai', category: 'providers', summary: 'Trace Novita chat completions from your Python app.', method: 'OpenAI client', logo: 'novita' }, diff --git a/src/pages/docs/integrations/providers/prismml.mdx b/src/pages/docs/integrations/providers/prismml.mdx new file mode 100644 index 000000000..c950c933c --- /dev/null +++ b/src/pages/docs/integrations/providers/prismml.mdx @@ -0,0 +1,112 @@ +--- +title: "Trace PrismML Bonsai" +description: "Trace local PrismML Bonsai chat completions in Future AGI with the OpenAI Python client, including model names, token usage and privacy controls." +--- + +Trace calls to your local [PrismML Bonsai server](https://docs.prismml.com/run/server) using the maintained [traceAI OpenAI integration](/docs/integrations/traceai/openai). This guide covers your Python client's Chat Completions requests, not the server's built-in web chat or other products named Prism. + +## Install + +Use Python 3.10–3.13: + +```bash +python -m pip install 'openai==3.42.1' 'traceAI-openai==0.1.12' 'fi-instrumentation-otel==1.2.2' +``` + +## Start the local server + +Follow the [PrismML setup](https://docs.prismml.com/run/server) to download a model. In the Bonsai demo directory, start the llama.cpp server with a stable model alias: + +```bash +./scripts/start_llama_server.sh --alias bonsai +``` + +The API base URL is `http://localhost:8080/v1`. The SDK appends `/chat/completions`; do not include that suffix in the base URL. + + +The local server does not authenticate requests. Keep it on loopback, or restrict access with a trusted network and firewall. The `not-needed` key below is only a placeholder required by the OpenAI SDK, not a Future AGI credential. + + +On Apple Silicon, PrismML also provides `./scripts/start_mlx_server.sh` at `http://localhost:8081/v1`. Follow the [MLX sampling and thinking settings](https://docs.prismml.com/run/server), which must be supplied in each request rather than inherited from the llama.cpp launcher. + +## Configure tracing + +Set your Future AGI credentials separately from the model server address: + +```bash +export FI_API_KEY="YOUR_FUTURE_AGI_API_KEY" +export FI_SECRET_KEY="YOUR_FUTURE_AGI_SECRET_KEY" +export FI_PROJECT_NAME="prismml-bonsai" +export PRISMML_BASE_URL="http://localhost:8080/v1" +export PRISMML_MODEL="bonsai" +``` + +Trace export goes to `https://api.futureagi.com/tracer/v1/traces` by default. If you self-host Future AGI, set `FI_BASE_URL` to its origin, without `/tracer/v1/traces`. Do not point it at the Bonsai server. + +## Run a completion + +Save this as `prismml_chat.py`. Instrument the OpenAI SDK once, before creating the client; do not layer another OpenAI instrumentor over the same requests. + +```python +import os + +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from openai import OpenAI +from traceai_openai import OpenAIInstrumentor + +provider = register( + project_name=os.environ.get("FI_PROJECT_NAME") or "prismml-bonsai", + project_type=ProjectType.OBSERVE, + set_global_tracer_provider=False, + verbose=False, +) +OpenAIInstrumentor().instrument(tracer_provider=provider) +try: + with OpenAI( + base_url=os.environ.get("PRISMML_BASE_URL") or "http://localhost:8080/v1", + api_key=os.environ.get("PRISMML_API_KEY") or "not-needed", + ) as client: + response = client.chat.completions.create( + model=os.environ.get("PRISMML_MODEL") or "bonsai", + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + print(response.choices[0].message.content or "") +finally: + provider.force_flush() +``` + +```bash +python prismml_chat.py +``` + +`PRISMML_API_KEY` is optional and defaults to `not-needed`. The script prints the answer and flushes the trace before exiting. + +## View the result + +Open **Observe** in Future AGI and select `prismml-bonsai`, or the project set in `FI_PROJECT_NAME`. Each completion produces one `ChatCompletion` LLM span with the prompt, answer, latency and token counts supplied by the server. The provider label is `openai` because this local endpoint uses the shared OpenAI instrumentor. + +Non-streamed calls record the server's returned model name in `gen_ai.request.model`. For streamed and failed calls, that field contains the requested name. Local models may have no matching price entry, so cost can be unavailable even when token counts are present. + +For a ready-to-run streaming client, use the [PrismML recipe](https://github.com/future-agi/traceAI/tree/dev/python/examples/prismml). If its stream ends without a finish reason, the command exits with status 1; treat any printed text as partial and retry. Streamed token counts require both `stream_options={"include_usage": True}` and a final usage chunk from the server; missing counts are not replaced with zero. + +## Privacy + +Local model execution does not mean local-only telemetry: prompts and answers are exported to Future AGI by default. Set these before starting the script to hide content and invocation parameters: + +```bash +export FI_HIDE_INPUTS=true +export FI_HIDE_OUTPUTS=true +export FI_HIDE_LLM_INVOCATION_PARAMETERS=true +``` + +Hide both inputs and outputs when the answer might echo sensitive input. These flags do not change what the local server receives or logs. + +The model name remains visible metadata. Without an alias, llama-server can return the model file's absolute path, including your user name or directory layout. Use `--alias bonsai` to keep that path out of traces. + +## Troubleshooting + +- **Connection refused:** start the server and check the port: 8080 for llama.cpp, 8081 for MLX. +- **404 from the server:** use the API base URL ending in `/v1`, not the web chat root or the full completion URL. +- **No traces:** check your Future AGI keys, project name and export origin. See [No traces appearing](/docs/observe/troubleshooting/no-traces-appearing). +- **A local path appears as the model:** restart llama-server with `--alias bonsai`.