Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions .github/workflows/ragas-recipe.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
name: Ragas evaluation recipe

on:
pull_request:
branches: [main]
paths:
- 'src/pages/docs/integrations/frameworks/ragas.mdx'
- 'scripts/test_ragas_recipe.py'
- '.github/workflows/ragas-recipe.yml'

permissions:
contents: read

jobs:
recipe:
name: Ragas ${{ matrix.ragas }} / Python ${{ matrix.python }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
include:
- python: '3.10'
ragas: '0.4.0'
- python: '3.13'
ragas: '0.4.3'
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python }}
- name: Install released recipe dependencies
run: >-
python -m pip install
'ragas==${{ matrix.ragas }}'
'langchain-community==0.4.1' 'futureagi==0.6.13'
- name: Execute the exact page snippet
# The test denies non-loopback sockets and uses only synthetic credentials.
run: python -I scripts/test_ragas_recipe.py -v
140 changes: 140 additions & 0 deletions scripts/test_ragas_recipe.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
"""Execute the Ragas recipe with real published packages and a loopback Scores API.

Run in an environment containing the page's pinned requirements:
python -m unittest discover -s scripts -p test_ragas_recipe.py -v
No Future AGI or model-provider credentials are needed.
"""

import asyncio
import contextlib
import io
import json
import os
from pathlib import Path
import re
import threading
import sys
from types import SimpleNamespace
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from unittest.mock import AsyncMock, patch


def deny_external_network(event, args):
if event == "socket.getaddrinfo":
host = args[0]
elif event in ("socket.connect", "socket.sendto"):
address = args[-1]
host = address[0] if isinstance(address, tuple) else address
else:
return
if host not in ("localhost", "127.0.0.1", "::1"):
raise RuntimeError("Only loopback network access is allowed in this test")


sys.addaudithook(deny_external_network)
os.environ["RAGAS_DO_NOT_TRACK"] = "true"
os.environ["HF_HUB_OFFLINE"] = "1"

PAGE = Path(__file__).resolve().parents[1] / "src/pages/docs/integrations/frameworks/ragas.mdx"
LABEL_ID = "550e8400-e29b-41d4-a716-446655440002"
SPAN_ID = "1234567890abcdef"


class RagasRecipeTests(unittest.TestCase):
def test_modern_metric_runs_without_a_model(self):
from ragas.metrics.collections import ExactMatch

metric = ExactMatch()
for response, expected in [("30 days", 1.0), ("60 days", 0.0)]:
result = asyncio.run(metric.ascore(response=response, reference="30 days"))
self.assertEqual(result.value, expected)

def test_documented_recipe_posts_scores_to_the_original_span(self):
# A missing page is the audited regression, not a missing instrumentor.
self.assertTrue(PAGE.is_file(), "Ragas evaluation recipe is missing")
code_blocks = re.findall(r"```python\n(.*?)\n```", PAGE.read_text(), re.S)
self.assertEqual(len(code_blocks), 1)
requests = []

class Handler(BaseHTTPRequestHandler):
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
requests.append((self.path, dict(self.headers), body))
payload = json.dumps({"status": True, "result": {"id": "score-id", **body}}).encode()
self.send_response(server.response_status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)

def log_message(self, *args):
pass

server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
server.response_status = 200
worker = threading.Thread(target=server.serve_forever, daemon=True)
worker.start()
env = {
"FI_API_KEY": "synthetic-api-key",
"FI_SECRET_KEY": "synthetic-secret-key",
"FI_BASE_URL": f"http://127.0.0.1:{server.server_port}",
"FI_RAGAS_SPAN_ID": SPAN_ID,
"FI_RAGAS_LABEL_ID": LABEL_ID,
"RAGAS_DO_NOT_TRACK": "true",
}
try:
with patch.dict(os.environ, env), contextlib.redirect_stdout(io.StringIO()):
namespace = {"__name__": "__main__"}
exec(compile(code_blocks[0], str(PAGE), "exec"), namespace)
self.assertEqual(len(requests), 1)
score = asyncio.run(namespace["score_span"](
span_id=SPAN_ID, response="60 days", reference="30 days", label_id=LABEL_ID,
))
self.assertEqual(score.value, {"value": 0.0})
for invalid_score in (float("nan"), float("inf"), float("-inf")):
with patch.object(namespace["ExactMatch"], "ascore", new=AsyncMock(
return_value=SimpleNamespace(value=invalid_score)
)), self.assertRaises(ValueError):
asyncio.run(namespace["score_span"](
span_id=SPAN_ID, response="30 days", reference="30 days", label_id=LABEL_ID,
))
# Empty or malformed target IDs must not send a score.
for invalid_id in ("", "not-a-span", "0" * 16):
with self.assertRaises(ValueError):
asyncio.run(namespace["score_span"](
span_id=invalid_id, response="30 days", reference="30 days", label_id=LABEL_ID,
))
self.assertEqual(len(requests), 2)
from fi.utils.errors import InvalidAuthError

server.response_status = 403
with self.assertRaises(InvalidAuthError):
asyncio.run(namespace["score_span"](
span_id=SPAN_ID, response="30 days", reference="30 days", label_id=LABEL_ID,
))
# The SDK can retry a rejected request, but must propagate the error.
self.assertGreaterEqual(len(requests), 3)
for request in requests[2:]:
self.assertEqual(request[2], requests[0][2])
for (path, headers, body), value in zip(requests[:2], (1.0, 0.0)):
self.assertEqual(path, "/model-hub/scores/")
normalized_headers = {key.lower(): val for key, val in headers.items()}
self.assertEqual(normalized_headers["x-api-key"], "synthetic-api-key")
self.assertEqual(normalized_headers["x-secret-key"], "synthetic-secret-key")
self.assertEqual(body["source_type"], "observation_span")
self.assertEqual(body["source_id"], SPAN_ID)
self.assertEqual(body["label_id"], LABEL_ID)
self.assertEqual(body["score_source"], "auto")
# Numeric annotation values use an object, as consumed by the UI.
self.assertEqual(body["value"], {"value": value})
self.assertIn("ragas", body["notes"])
self.assertNotIn("30 days", json.dumps(body))
finally:
server.shutdown()
server.server_close()
worker.join()


if __name__ == "__main__":
unittest.main()
1 change: 1 addition & 0 deletions src/lib/integrations.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ export const integrations = [
{ id: 'koog', name: 'Koog', href: '/docs/integrations/frameworks/koog', category: 'frameworks', summary: 'Export Kotlin JVM agent spans with Koog’s OTel feature.', method: 'OpenTelemetry', logo: 'koog', aliases: ['Kotlin', 'JVM'] },
{ id: 'mirascope', name: 'Mirascope', href: '/docs/integrations/frameworks/mirascope', category: 'frameworks', summary: 'Send Mirascope 2.x model spans with your own exporter.', method: 'OpenTelemetry', logo: 'mirascope' },
{ id: 'open-agent-spec', name: 'Open Agent Spec', href: '/docs/integrations/frameworks/open-agent-spec', category: 'frameworks', summary: 'Trace Agent Spec agents through the LangGraph adapter.', method: 'traceAI SDK', logo: 'open-agent-spec', aliases: ['Oracle', 'pyagentspec'] },
{ id: 'ragas', name: 'Ragas', href: '/docs/integrations/frameworks/ragas', category: 'frameworks', summary: 'Attach Ragas evaluation scores to original application spans.', method: 'Scores SDK', logo: 'generic-framework', aliases: ['RAG evaluation'] },
{ id: 'restate', name: 'Restate', href: '/docs/integrations/frameworks/restate', category: 'frameworks', summary: 'Connect durable invocation, attempt and model spans.', method: 'OTel + traceAI', logo: 'restate' },
{ id: 'ruby-llm', name: 'RubyLLM', href: '/docs/integrations/frameworks/ruby-llm', category: 'frameworks', summary: 'Trace RubyLLM calls with the third-party OTel gem.', method: 'OpenTelemetry', logo: 'ruby-llm', aliases: ['Ruby LLM', 'ruby_llm'] },
{ id: 'swiftide', name: 'Swiftide', href: '/docs/integrations/frameworks/swiftide', category: 'frameworks', summary: 'Send typed Rust agent, model and tool spans through an OpenTelemetry Collector.', method: 'OpenTelemetry', logo: 'swiftide', aliases: ['Rust'] },
Expand Down
120 changes: 120 additions & 0 deletions src/pages/docs/integrations/frameworks/ragas.mdx
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
---
title: "Ragas"
description: "Run Ragas evaluations in Python and attach their numeric scores to the original application spans in Future AGI with the published Scores SDK."
---

## About

Run a Ragas metric on an application response and save the result as an automated annotation on its original Future AGI span. Keep your application's existing tracing setup; scoring does not require a separate Ragas instrumentor.

## Install

Use Python 3.10–3.13 in a virtual environment:

```bash
python -m venv .venv
source .venv/bin/activate
python -m pip install "ragas==0.4.3" "langchain-community==0.4.1" "futureagi==0.6.13"
```

<Note>
Keep the `langchain-community` pin with Ragas 0.4.3. Version 0.4.2 removes a module that Ragas imports, causing `ModuleNotFoundError` before evaluation starts.
</Note>

## Choose a span and label

1. Trace your application using its existing [LangChain](/docs/integrations/traceai/langchain), [OpenAI](/docs/integrations/traceai/openai), or other [supported integration](/docs/integrations). Wait until the response's span is visible in **Tracing**.
2. Copy that span's 16-character hexadecimal **span ID**. Use the application span that produced the response, not a trace ID or an evaluator run ID.
3. Create a [numeric annotation label](/docs/annotations/guides/create-label) named `ragas_exact_match`, with a range of 0 to 1, in the same workspace. Find its ID with the [List labels API](/docs/api/annotations/labels/list-labels). Use a separate label for each metric.
4. Set your [Future AGI API credentials](/docs/admin-settings/api-keys) and both IDs:

```bash
export FI_API_KEY="YOUR_API_KEY"
export FI_SECRET_KEY="YOUR_SECRET_KEY"
export FI_RAGAS_SPAN_ID="YOUR_APPLICATION_SPAN_ID"
export FI_RAGAS_LABEL_ID="YOUR_NUMERIC_LABEL_ID"
export RAGAS_DO_NOT_TRACK="true"
```

The example uses Ragas's local `ExactMatch` metric, so it needs no model-provider key. It returns 1 for identical strings and 0 otherwise, including differences in case or whitespace. Use it for short, fixed-format answers, not semantic equivalence or faithfulness.

## Evaluate and save the score

Save this as `score_ragas.py`. Replace `response` with the output from the selected span and `reference` with the expected answer. The example checks a return-policy answer of `30 days`.

```python
import asyncio
import math
import os
import re

from fi.queues import AnnotationQueue
from ragas.metrics.collections import ExactMatch


async def score_span(*, span_id, response, reference, label_id):
if not re.fullmatch(r"[0-9a-f]{16}", span_id) or int(span_id, 16) == 0:
raise ValueError("Use the original application's 16-character span ID")

result = await ExactMatch().ascore(response=response, reference=reference)
value = float(result.value)
if not math.isfinite(value):
raise ValueError("The metric returned no finite score; do not upload it")

return AnnotationQueue().create_score(
source_type="observation_span",
source_id=span_id,
label_id=label_id,
value={"value": value},
score_source="auto",
notes="ragas:exact_match",
timeout=30,
)


async def main():
score = await score_span(
span_id=os.environ["FI_RAGAS_SPAN_ID"],
label_id=os.environ["FI_RAGAS_LABEL_ID"],
response="30 days",
reference="30 days",
)
print(f"Saved Ragas score {score.value['value']} (score ID: {score.id})")


if __name__ == "__main__":
asyncio.run(main())
```

```bash
python score_ragas.py
```

The SDK sends the numeric value through `/model-hub/scores/`, not the OTLP traces endpoint. The request identifies the original span and marks the score as `auto`; it does not upload the response or reference text. Repeating the same span, label, API user, and queue context updates that score rather than creating a separate evaluation history. Use a new metric label when you need to preserve results from a different metric definition.

## View the result

Open the original trace in **Tracing**, select the response span, and view its **Annotations**. The `ragas_exact_match` label holds `1.0` for the example above. Change the response to `60 days` and rerun to record `0.0`.

These are imported annotation scores, not Future AGI evaluation-template runs. For dataset-level results, the [Scores SDK](/docs/sdk/annotation-queues/scores) also supports `source_type="dataset_row"` with the existing dataset row's ID. Keep each row's source ID paired with its response when evaluating a batch; do not attach an aggregate score to every row.

## Tracing and other metrics

Application tracing and evaluation scoring are separate. Keep one instrumentor for each application framework or model client; do not add another exporter just to upload scores.

For existing code using Ragas `evaluate()` or `aevaluate()`, the [LangChain instrumentor](/docs/integrations/traceai/langchain) captures their official callback lifecycle. The modern `ragas.metrics.collections` API used above does not emit that callback tree. For LLM-based metrics, configure your own evaluator-provider credentials and instrument the provider client if you need its model-call traces. Those evaluator spans are distinct from the application span being scored.

Choose the metric and required inputs from the [Ragas metrics documentation](https://docs.ragas.io/en/stable/concepts/metrics/available_metrics/). Keep the original source ID, a metric-specific label, and the finite-score check when replacing `ExactMatch`. LLM-based metrics may send your response, reference, and retrieved context to the chosen model provider.

## Troubleshooting

- **Ragas fails to import:** install the complete pinned command above in a clean environment. Updating only Ragas can leave an incompatible `langchain-community` installed.
- **Source not found:** wait for application-span ingestion and use the span ID from the same workspace as your API key. Use `observation_span`, not `span`, as the source type.
- **Label not found or the wrong label appears:** verify the numeric label's ID and workspace. Do not use the label name where the code expects an ID.
- **The metric fails or returns NaN:** do not convert the failure to zero. Fix the missing inputs or provider error before uploading a score; zero is a legitimate evaluation result.

## Next steps

- [Manage annotation labels](/docs/annotations/guides/create-label)
- [Use the Scores SDK](/docs/sdk/annotation-queues/scores)
- [Manage evaluation datasets](/docs/sdk/datasets)
Loading