From 82f9d08ff14ac17652e215a465ff41c30df4d8c5 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 20:41:42 -0700 Subject: [PATCH] docs(ragas): link evaluation scores to application spans Co-Authored-By: Claude Code --- .github/workflows/ragas-recipe.yml | 38 +++++ scripts/test_ragas_recipe.py | 140 ++++++++++++++++++ src/lib/integrations.mjs | 1 + .../docs/integrations/frameworks/ragas.mdx | 120 +++++++++++++++ 4 files changed, 299 insertions(+) create mode 100644 .github/workflows/ragas-recipe.yml create mode 100644 scripts/test_ragas_recipe.py create mode 100644 src/pages/docs/integrations/frameworks/ragas.mdx diff --git a/.github/workflows/ragas-recipe.yml b/.github/workflows/ragas-recipe.yml new file mode 100644 index 000000000..b47441b15 --- /dev/null +++ b/.github/workflows/ragas-recipe.yml @@ -0,0 +1,38 @@ +name: Ragas evaluation recipe + +on: + pull_request: + branches: [main] + paths: + - 'src/pages/docs/integrations/frameworks/ragas.mdx' + - 'scripts/test_ragas_recipe.py' + - '.github/workflows/ragas-recipe.yml' + +permissions: + contents: read + +jobs: + recipe: + name: Ragas ${{ matrix.ragas }} / Python ${{ matrix.python }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - python: '3.10' + ragas: '0.4.0' + - python: '3.13' + ragas: '0.4.3' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install released recipe dependencies + run: >- + python -m pip install + 'ragas==${{ matrix.ragas }}' + 'langchain-community==0.4.1' 'futureagi==0.6.13' + - name: Execute the exact page snippet + # The test denies non-loopback sockets and uses only synthetic credentials. + run: python -I scripts/test_ragas_recipe.py -v diff --git a/scripts/test_ragas_recipe.py b/scripts/test_ragas_recipe.py new file mode 100644 index 000000000..556d187b4 --- /dev/null +++ b/scripts/test_ragas_recipe.py @@ -0,0 +1,140 @@ +"""Execute the Ragas recipe with real published packages and a loopback Scores API. + +Run in an environment containing the page's pinned requirements: + python -m unittest discover -s scripts -p test_ragas_recipe.py -v +No Future AGI or model-provider credentials are needed. +""" + +import asyncio +import contextlib +import io +import json +import os +from pathlib import Path +import re +import threading +import sys +from types import SimpleNamespace +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from unittest.mock import AsyncMock, patch + + +def deny_external_network(event, args): + if event == "socket.getaddrinfo": + host = args[0] + elif event in ("socket.connect", "socket.sendto"): + address = args[-1] + host = address[0] if isinstance(address, tuple) else address + else: + return + if host not in ("localhost", "127.0.0.1", "::1"): + raise RuntimeError("Only loopback network access is allowed in this test") + + +sys.addaudithook(deny_external_network) +os.environ["RAGAS_DO_NOT_TRACK"] = "true" +os.environ["HF_HUB_OFFLINE"] = "1" + +PAGE = Path(__file__).resolve().parents[1] / "src/pages/docs/integrations/frameworks/ragas.mdx" +LABEL_ID = "550e8400-e29b-41d4-a716-446655440002" +SPAN_ID = "1234567890abcdef" + + +class RagasRecipeTests(unittest.TestCase): + def test_modern_metric_runs_without_a_model(self): + from ragas.metrics.collections import ExactMatch + + metric = ExactMatch() + for response, expected in [("30 days", 1.0), ("60 days", 0.0)]: + result = asyncio.run(metric.ascore(response=response, reference="30 days")) + self.assertEqual(result.value, expected) + + def test_documented_recipe_posts_scores_to_the_original_span(self): + # A missing page is the audited regression, not a missing instrumentor. + self.assertTrue(PAGE.is_file(), "Ragas evaluation recipe is missing") + code_blocks = re.findall(r"```python\n(.*?)\n```", PAGE.read_text(), re.S) + self.assertEqual(len(code_blocks), 1) + requests = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + requests.append((self.path, dict(self.headers), body)) + payload = json.dumps({"status": True, "result": {"id": "score-id", **body}}).encode() + self.send_response(server.response_status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.response_status = 200 + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + env = { + "FI_API_KEY": "synthetic-api-key", + "FI_SECRET_KEY": "synthetic-secret-key", + "FI_BASE_URL": f"http://127.0.0.1:{server.server_port}", + "FI_RAGAS_SPAN_ID": SPAN_ID, + "FI_RAGAS_LABEL_ID": LABEL_ID, + "RAGAS_DO_NOT_TRACK": "true", + } + try: + with patch.dict(os.environ, env), contextlib.redirect_stdout(io.StringIO()): + namespace = {"__name__": "__main__"} + exec(compile(code_blocks[0], str(PAGE), "exec"), namespace) + self.assertEqual(len(requests), 1) + score = asyncio.run(namespace["score_span"]( + span_id=SPAN_ID, response="60 days", reference="30 days", label_id=LABEL_ID, + )) + self.assertEqual(score.value, {"value": 0.0}) + for invalid_score in (float("nan"), float("inf"), float("-inf")): + with patch.object(namespace["ExactMatch"], "ascore", new=AsyncMock( + return_value=SimpleNamespace(value=invalid_score) + )), self.assertRaises(ValueError): + asyncio.run(namespace["score_span"]( + span_id=SPAN_ID, response="30 days", reference="30 days", label_id=LABEL_ID, + )) + # Empty or malformed target IDs must not send a score. + for invalid_id in ("", "not-a-span", "0" * 16): + with self.assertRaises(ValueError): + asyncio.run(namespace["score_span"]( + span_id=invalid_id, response="30 days", reference="30 days", label_id=LABEL_ID, + )) + self.assertEqual(len(requests), 2) + from fi.utils.errors import InvalidAuthError + + server.response_status = 403 + with self.assertRaises(InvalidAuthError): + asyncio.run(namespace["score_span"]( + span_id=SPAN_ID, response="30 days", reference="30 days", label_id=LABEL_ID, + )) + # The SDK can retry a rejected request, but must propagate the error. + self.assertGreaterEqual(len(requests), 3) + for request in requests[2:]: + self.assertEqual(request[2], requests[0][2]) + for (path, headers, body), value in zip(requests[:2], (1.0, 0.0)): + self.assertEqual(path, "/model-hub/scores/") + normalized_headers = {key.lower(): val for key, val in headers.items()} + self.assertEqual(normalized_headers["x-api-key"], "synthetic-api-key") + self.assertEqual(normalized_headers["x-secret-key"], "synthetic-secret-key") + self.assertEqual(body["source_type"], "observation_span") + self.assertEqual(body["source_id"], SPAN_ID) + self.assertEqual(body["label_id"], LABEL_ID) + self.assertEqual(body["score_source"], "auto") + # Numeric annotation values use an object, as consumed by the UI. + self.assertEqual(body["value"], {"value": value}) + self.assertIn("ragas", body["notes"]) + self.assertNotIn("30 days", json.dumps(body)) + finally: + server.shutdown() + server.server_close() + worker.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/src/lib/integrations.mjs b/src/lib/integrations.mjs index 2c723d3cd..61039dd6f 100644 --- a/src/lib/integrations.mjs +++ b/src/lib/integrations.mjs @@ -68,6 +68,7 @@ export const integrations = [ { id: 'novita-ai', name: 'Novita AI', href: '/docs/integrations/providers/novita-ai', category: 'providers', summary: 'Trace Novita chat completions from your Python app.', method: 'OpenAI client', logo: 'novita' }, { id: 'litellm-sdk', name: 'LiteLLM SDK', href: '/docs/integrations/traceai/litellm', category: 'frameworks', summary: 'Instrument LiteLLM calls in your Python application.', method: 'traceAI SDK', logo: 'litellm', aliases: ['traceAI-litellm', 'client'] }, + { id: 'ragas', name: 'Ragas', href: '/docs/integrations/frameworks/ragas', category: 'frameworks', summary: 'Attach Ragas evaluation scores to original application spans.', method: 'Scores SDK', logo: 'generic-framework', aliases: ['RAG evaluation'] }, { id: 'langchain', name: 'LangChain', href: '/docs/integrations/traceai/langchain', category: 'frameworks', summary: 'Trace chains, retrievers, tools and model calls.', method: 'traceAI SDK', logo: 'langchain' }, { id: 'langgraph', name: 'LangGraph', href: '/docs/integrations/traceai/langgraph', category: 'frameworks', summary: 'Follow agent graph execution and tool calls.', method: 'traceAI SDK', logo: 'langgraph' }, { id: 'llamaindex', name: 'LlamaIndex', href: '/docs/integrations/traceai/llamaindex', category: 'frameworks', summary: 'Trace retrieval and generation in RAG applications.', method: 'traceAI SDK', logo: 'llamaindex', aliases: ['RAG'] }, diff --git a/src/pages/docs/integrations/frameworks/ragas.mdx b/src/pages/docs/integrations/frameworks/ragas.mdx new file mode 100644 index 000000000..c76efc277 --- /dev/null +++ b/src/pages/docs/integrations/frameworks/ragas.mdx @@ -0,0 +1,120 @@ +--- +title: "Ragas" +description: "Run Ragas evaluations in Python and attach their numeric scores to the original application spans in Future AGI with the published Scores SDK." +--- + +## About + +Run a Ragas metric on an application response and save the result as an automated annotation on its original Future AGI span. Keep your application's existing tracing setup; scoring does not require a separate Ragas instrumentor. + +## Install + +Use Python 3.10–3.13 in a virtual environment: + +```bash +python -m venv .venv +source .venv/bin/activate +python -m pip install "ragas==0.4.3" "langchain-community==0.4.1" "futureagi==0.6.13" +``` + + + Keep the `langchain-community` pin with Ragas 0.4.3. Version 0.4.2 removes a module that Ragas imports, causing `ModuleNotFoundError` before evaluation starts. + + +## Choose a span and label + +1. Trace your application using its existing [LangChain](/docs/integrations/traceai/langchain), [OpenAI](/docs/integrations/traceai/openai), or other [supported integration](/docs/integrations). Wait until the response's span is visible in **Tracing**. +2. Copy that span's 16-character hexadecimal **span ID**. Use the application span that produced the response, not a trace ID or an evaluator run ID. +3. Create a [numeric annotation label](/docs/annotations/guides/create-label) named `ragas_exact_match`, with a range of 0 to 1, in the same workspace. Find its ID with the [List labels API](/docs/api/annotations/labels/list-labels). Use a separate label for each metric. +4. Set your [Future AGI API credentials](/docs/admin-settings/api-keys) and both IDs: + +```bash +export FI_API_KEY="YOUR_API_KEY" +export FI_SECRET_KEY="YOUR_SECRET_KEY" +export FI_RAGAS_SPAN_ID="YOUR_APPLICATION_SPAN_ID" +export FI_RAGAS_LABEL_ID="YOUR_NUMERIC_LABEL_ID" +export RAGAS_DO_NOT_TRACK="true" +``` + +The example uses Ragas's local `ExactMatch` metric, so it needs no model-provider key. It returns 1 for identical strings and 0 otherwise, including differences in case or whitespace. Use it for short, fixed-format answers, not semantic equivalence or faithfulness. + +## Evaluate and save the score + +Save this as `score_ragas.py`. Replace `response` with the output from the selected span and `reference` with the expected answer. The example checks a return-policy answer of `30 days`. + +```python +import asyncio +import math +import os +import re + +from fi.queues import AnnotationQueue +from ragas.metrics.collections import ExactMatch + + +async def score_span(*, span_id, response, reference, label_id): + if not re.fullmatch(r"[0-9a-f]{16}", span_id) or int(span_id, 16) == 0: + raise ValueError("Use the original application's 16-character span ID") + + result = await ExactMatch().ascore(response=response, reference=reference) + value = float(result.value) + if not math.isfinite(value): + raise ValueError("The metric returned no finite score; do not upload it") + + return AnnotationQueue().create_score( + source_type="observation_span", + source_id=span_id, + label_id=label_id, + value={"value": value}, + score_source="auto", + notes="ragas:exact_match", + timeout=30, + ) + + +async def main(): + score = await score_span( + span_id=os.environ["FI_RAGAS_SPAN_ID"], + label_id=os.environ["FI_RAGAS_LABEL_ID"], + response="30 days", + reference="30 days", + ) + print(f"Saved Ragas score {score.value['value']} (score ID: {score.id})") + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +```bash +python score_ragas.py +``` + +The SDK sends the numeric value through `/model-hub/scores/`, not the OTLP traces endpoint. The request identifies the original span and marks the score as `auto`; it does not upload the response or reference text. Repeating the same span, label, API user, and queue context updates that score rather than creating a separate evaluation history. Use a new metric label when you need to preserve results from a different metric definition. + +## View the result + +Open the original trace in **Tracing**, select the response span, and view its **Annotations**. The `ragas_exact_match` label holds `1.0` for the example above. Change the response to `60 days` and rerun to record `0.0`. + +These are imported annotation scores, not Future AGI evaluation-template runs. For dataset-level results, the [Scores SDK](/docs/sdk/annotation-queues/scores) also supports `source_type="dataset_row"` with the existing dataset row's ID. Keep each row's source ID paired with its response when evaluating a batch; do not attach an aggregate score to every row. + +## Tracing and other metrics + +Application tracing and evaluation scoring are separate. Keep one instrumentor for each application framework or model client; do not add another exporter just to upload scores. + +For existing code using Ragas `evaluate()` or `aevaluate()`, the [LangChain instrumentor](/docs/integrations/traceai/langchain) captures their official callback lifecycle. The modern `ragas.metrics.collections` API used above does not emit that callback tree. For LLM-based metrics, configure your own evaluator-provider credentials and instrument the provider client if you need its model-call traces. Those evaluator spans are distinct from the application span being scored. + +Choose the metric and required inputs from the [Ragas metrics documentation](https://docs.ragas.io/en/stable/concepts/metrics/available_metrics/). Keep the original source ID, a metric-specific label, and the finite-score check when replacing `ExactMatch`. LLM-based metrics may send your response, reference, and retrieved context to the chosen model provider. + +## Troubleshooting + +- **Ragas fails to import:** install the complete pinned command above in a clean environment. Updating only Ragas can leave an incompatible `langchain-community` installed. +- **Source not found:** wait for application-span ingestion and use the span ID from the same workspace as your API key. Use `observation_span`, not `span`, as the source type. +- **Label not found or the wrong label appears:** verify the numeric label's ID and workspace. Do not use the label name where the code expects an ID. +- **The metric fails or returns NaN:** do not convert the failure to zero. Fix the missing inputs or provider error before uploading a score; zero is a legitimate evaluation result. + +## Next steps + +- [Manage annotation labels](/docs/annotations/guides/create-label) +- [Use the Scores SDK](/docs/sdk/annotation-queues/scores) +- [Manage evaluation datasets](/docs/sdk/datasets)