From 5936e0960f01a23e842d3df90b7c101874ed8149 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 18:58:31 -0700 Subject: [PATCH 01/22] test(harness): add shared OTLP harness for TH-8103 contract tests --- python/tests/harness/__init__.py | 184 +++++++++++++++++++++++++++ python/tests/harness/test_harness.py | 101 +++++++++++++++ 2 files changed, 285 insertions(+) create mode 100644 python/tests/harness/__init__.py create mode 100644 python/tests/harness/test_harness.py diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..dfd9198e4 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,184 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + if urlsplit(self.path).path != "/v1/traces": + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def clear(self) -> None: + """Remove every decoded span received so far.""" + with self._lock: + self._spans.clear() + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..4e76ceabe --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,101 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out From 32f5c9d2b3b80f97a70e00b3f3ca1b408b14dc5c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 23:21:29 -0700 Subject: [PATCH 02/22] test(harness): accept the collector /tracer/v1/traces path --- python/tests/harness/__init__.py | 3 ++- python/tests/harness/test_harness.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index dfd9198e4..cb510ad10 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -28,7 +28,7 @@ def __init__(self) -> None: class Handler(BaseHTTPRequestHandler): def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path != "/v1/traces": + if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return @@ -76,6 +76,7 @@ def log_message(self, _format: str, *args: Any) -> None: self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) self._thread.start() diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 4e76ceabe..6a32c2af8 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -38,6 +38,18 @@ def test_receiver_stores_posted_span_and_can_clear() -> None: assert receiver.spans() == [] +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From 7586f645c39fdc16f37d4f917ba139f4ab3224f3 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 23:27:59 -0700 Subject: [PATCH 03/22] feat(firecrawl): add traceAI-firecrawl instrumentor for TH-8325 --- python/frameworks/firecrawl/README.md | 29 +++ .../frameworks/firecrawl/examples/scrape.py | 30 ++++ python/frameworks/firecrawl/pyproject.toml | 35 ++++ .../firecrawl/tests/test_instrumentor.py | 99 +++++++++++ .../firecrawl/traceai_firecrawl/__init__.py | 88 +++++++++ .../firecrawl/traceai_firecrawl/_wrappers.py | 167 ++++++++++++++++++ .../firecrawl/traceai_firecrawl/package.py | 1 + .../firecrawl/traceai_firecrawl/py.typed | 0 .../firecrawl/traceai_firecrawl/version.py | 1 + 9 files changed, 450 insertions(+) create mode 100644 python/frameworks/firecrawl/README.md create mode 100644 python/frameworks/firecrawl/examples/scrape.py create mode 100644 python/frameworks/firecrawl/pyproject.toml create mode 100644 python/frameworks/firecrawl/tests/test_instrumentor.py create mode 100644 python/frameworks/firecrawl/traceai_firecrawl/__init__.py create mode 100644 python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py create mode 100644 python/frameworks/firecrawl/traceai_firecrawl/package.py create mode 100644 python/frameworks/firecrawl/traceai_firecrawl/py.typed create mode 100644 python/frameworks/firecrawl/traceai_firecrawl/version.py diff --git a/python/frameworks/firecrawl/README.md b/python/frameworks/firecrawl/README.md new file mode 100644 index 000000000..ab48ee713 --- /dev/null +++ b/python/frameworks/firecrawl/README.md @@ -0,0 +1,29 @@ +# traceAI-firecrawl + +OpenTelemetry instrumentation for the Firecrawl v2 client. + +## v1 and v2 are different clients + +`firecrawl-py` 4.46.2 ships two clients. The v2 client is `from firecrawl import Firecrawl`, and it delegates `scrape`, `search`, `map`, and `crawl` to `firecrawl.v2.client.FirecrawlClient`. The v1 client is `FirecrawlApp`, and it is feature-frozen. + +This package wraps the v2 client only. It does not wrap `FirecrawlApp`, and it does not import or disable OpenLIT. OpenLIT's `FireCrawlInstrumentor` wraps `FirecrawlApp.scrape_url` on the v1 client. A user who constructs both clients and enables both instrumentors gets two spans. Pick one. + +## What it records + +Each span records `fi.span.kind=TOOL`. `search` also records the query, capped at 1024 characters, and the result count. `scrape` and `map` record the URL host only, never the path or the body. `crawl` emits one span for the whole call, not one per page, and records the job id, the limit, and the page count. + +Content capture is off by default. The API key is never copied into an attribute. `credits_used` is recorded only when the response carries it, and it is not converted to dollars. + +## Usage + +```python +from firecrawl import Firecrawl +from fi_instrumentation import register +from traceai_firecrawl import FirecrawlInstrumentor + +tracer_provider = register(project_name="firecrawl") +FirecrawlInstrumentor().instrument(tracer_provider=tracer_provider) + +client = Firecrawl(api_key="your-firecrawl-api-key") +client.scrape("https://example.com") +``` diff --git a/python/frameworks/firecrawl/examples/scrape.py b/python/frameworks/firecrawl/examples/scrape.py new file mode 100644 index 000000000..973ea9c04 --- /dev/null +++ b/python/frameworks/firecrawl/examples/scrape.py @@ -0,0 +1,30 @@ +"""Run one traced Firecrawl scrape against the configured endpoint.""" + +import os + +from firecrawl import Firecrawl +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_firecrawl import FirecrawlInstrumentor + + +def main() -> None: + tracer_provider = register( + project_name="firecrawl-example", + project_type=ProjectType.OBSERVE, + batch=False, + verbose=False, + ) + FirecrawlInstrumentor().instrument(tracer_provider=tracer_provider) + + # firecrawl-py 4.46.2 takes api_url, confirmed from Firecrawl.__init__. + client = Firecrawl( + api_key="firecrawl-dummy-key", + api_url=os.getenv("FIRECRAWL_BASE_URL", "https://api.firecrawl.dev"), + ) + client.scrape("https://example.com") + tracer_provider.force_flush() + + +if __name__ == "__main__": + main() diff --git a/python/frameworks/firecrawl/pyproject.toml b/python/frameworks/firecrawl/pyproject.toml new file mode 100644 index 000000000..6d120ccdb --- /dev/null +++ b/python/frameworks/firecrawl/pyproject.toml @@ -0,0 +1,35 @@ +[tool.poetry] +name = "traceAI-firecrawl" +version = "0.1.0" +description = "OpenTelemetry instrumentation for the Firecrawl v2 client" +authors = ["Future AGI "] +readme = "README.md" +packages = [ + { include = "traceai_firecrawl" } +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +[tool.poetry.dependencies] +python = ">=3.9,<3.14" +fi-instrumentation-otel = ">=0.1.11" +firecrawl-py = "==4.46.2" +wrapt = ">=1.0.0" + +[tool.poetry.group.dev.dependencies] +pytest = ">=7.0.0" +pytest-asyncio = ">=0.21.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/python/frameworks/firecrawl/tests/test_instrumentor.py b/python/frameworks/firecrawl/tests/test_instrumentor.py new file mode 100644 index 000000000..1c648f37f --- /dev/null +++ b/python/frameworks/firecrawl/tests/test_instrumentor.py @@ -0,0 +1,99 @@ +"""Unit tests for Firecrawl instrumentation.""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import pytest +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +pytest.importorskip("firecrawl", reason="firecrawl-py must be installed to test its instrumentor") +from firecrawl.v2.client import FirecrawlClient # noqa: E402 +from traceai_firecrawl import FirecrawlInstrumentor # noqa: E402 + +_API_KEY = "fc-api-key-should-not-appear-in-a-span" + + +def _tracing() -> tuple[TracerProvider, InMemorySpanExporter]: + exporter = InMemorySpanExporter() + provider = TracerProvider(resource=Resource.create({"service.name": "test-firecrawl"})) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return provider, exporter + + +def _instrument(provider: TracerProvider) -> FirecrawlInstrumentor: + instrumentor = FirecrawlInstrumentor() + instrumentor.instrument(tracer_provider=provider) + return instrumentor + + +def test_crawl_emits_one_span_for_three_pages(monkeypatch: pytest.MonkeyPatch) -> None: + pages = [SimpleNamespace(), SimpleNamespace(), SimpleNamespace()] + + def crawl(_self: FirecrawlClient, _url: str, **_kwargs: Any) -> Any: + return SimpleNamespace(id="job-1", data=pages, credits_used=3) + + monkeypatch.setattr(FirecrawlClient, "crawl", crawl) + provider, exporter = _tracing() + instrumentor = _instrument(provider) + try: + FirecrawlClient(api_key=_API_KEY).crawl("https://example.com", limit=10) + finally: + instrumentor.uninstrument() + + spans = exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + attrs = dict(span.attributes or {}) + assert span.name == "firecrawl.crawl" + assert attrs["fi.span.kind"] == "TOOL" + assert attrs["firecrawl.page_count"] == 3 + assert attrs["firecrawl.limit"] == 10 + assert attrs["firecrawl.job_id"] == "job-1" + assert attrs["server.address"] == "example.com" + assert all(_API_KEY not in str(value) for value in attrs.values()) + assert all(not key.startswith("fc-") for key in attrs) + + +def test_scrape_exception_is_recorded_and_reraised(monkeypatch: pytest.MonkeyPatch) -> None: + def scrape(_self: FirecrawlClient, _url: str, **_kwargs: Any) -> Any: + raise RuntimeError("Firecrawl is unavailable") + + monkeypatch.setattr(FirecrawlClient, "scrape", scrape) + provider, exporter = _tracing() + instrumentor = _instrument(provider) + try: + with pytest.raises(RuntimeError, match="Firecrawl is unavailable"): + FirecrawlClient(api_key=_API_KEY).scrape("https://example.com") + finally: + instrumentor.uninstrument() + + span = exporter.get_finished_spans()[0] + assert span.status.status_code is StatusCode.ERROR + assert any(event.name == "exception" for event in span.events) + + +def test_search_records_query_and_count(monkeypatch: pytest.MonkeyPatch) -> None: + def search(_self: FirecrawlClient, _query: str, **_kwargs: Any) -> Any: + return SimpleNamespace(web=[SimpleNamespace(), SimpleNamespace()]) + + monkeypatch.setattr(FirecrawlClient, "search", search) + provider, exporter = _tracing() + instrumentor = _instrument(provider) + try: + FirecrawlClient(api_key=_API_KEY).search("retrieval query") + finally: + instrumentor.uninstrument() + + spans = exporter.get_finished_spans() + assert len(spans) == 1 + attrs = dict(spans[0].attributes or {}) + assert spans[0].name == "firecrawl.search" + assert attrs["fi.retrieval.query"] == "retrieval query" + assert attrs["fi.retrieval.document_count"] == 2 + assert all(_API_KEY not in str(value) for value in attrs.values()) diff --git a/python/frameworks/firecrawl/traceai_firecrawl/__init__.py b/python/frameworks/firecrawl/traceai_firecrawl/__init__.py new file mode 100644 index 000000000..09846debd --- /dev/null +++ b/python/frameworks/firecrawl/traceai_firecrawl/__init__.py @@ -0,0 +1,88 @@ +"""OpenTelemetry instrumentation for the Firecrawl v2 client.""" + +import logging +from typing import Any, Collection, Dict, Tuple + +from opentelemetry import trace as trace_api +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from wrapt import wrap_function_wrapper + +from traceai_firecrawl._wrappers import AsyncOperationWrapper, OperationWrapper +from traceai_firecrawl.package import _instruments +from traceai_firecrawl.version import __version__ + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + +# firecrawl-py 4.46.2 binds scrape/search/map/crawl onto the v2 client, and the +# top-level Firecrawl delegates to it (firecrawl/client.py assigns +# self.scrape = self._v2_client.scrape). The methods are not on the Firecrawl +# class, so the wrapper attaches to the v2 client. Wrapping that class covers +# both `from firecrawl import Firecrawl` and direct FirecrawlClient use. +_SYNC_MODULE = "firecrawl.v2.client" +_SYNC_CLASS = "FirecrawlClient" +_ASYNC_MODULE = "firecrawl.v2.client_async" +_ASYNC_CLASS = "AsyncFirecrawlClient" + +_OPERATIONS = { + "scrape": "firecrawl.scrape", + "search": "firecrawl.search", + "map": "firecrawl.map", + "crawl": "firecrawl.crawl", + "start_crawl": "firecrawl.start_crawl", + "get_crawl_status": "firecrawl.get_crawl_status", + "cancel_crawl": "firecrawl.cancel_crawl", +} + + +class FirecrawlInstrumentor(BaseInstrumentor): + """Instrument the Firecrawl v2 client's scrape, search, map, and crawl APIs.""" + + __slots__ = ("_original_methods",) + + def instrumentation_dependencies(self) -> Collection[str]: + return _instruments + + def _instrument(self, **kwargs: Any) -> None: + tracer_provider = kwargs.get("tracer_provider") or trace_api.get_tracer_provider() + tracer = trace_api.get_tracer(__name__, __version__, tracer_provider) + self._original_methods: Dict[Tuple[str, str, str], Any] = {} + + for module_name, class_name, is_async in ( + (_SYNC_MODULE, _SYNC_CLASS, False), + (_ASYNC_MODULE, _ASYNC_CLASS, True), + ): + for method_name, span_name in _OPERATIONS.items(): + wrapper = ( + AsyncOperationWrapper(tracer, span_name, method_name) + if is_async + else OperationWrapper(tracer, span_name, method_name) + ) + self._wrap(module_name, class_name, method_name, wrapper) + + logger.debug("Firecrawl instrumentation enabled") + + def _wrap(self, module_name: str, class_name: str, method_name: str, wrapper: Any) -> None: + from importlib import import_module + + module = import_module(module_name) + client = getattr(module, class_name) + if not hasattr(client, method_name): + return + + self._original_methods[(module_name, class_name, method_name)] = getattr( + client, method_name + ) + # wrapt 2.5.0 rejects keyword arguments, so these are positional. + wrap_function_wrapper(module, "{0}.{1}".format(class_name, method_name), wrapper) + + def _uninstrument(self, **kwargs: Any) -> None: + from importlib import import_module + + for (module_name, class_name, method_name), original in self._original_methods.items(): + setattr(getattr(import_module(module_name), class_name), method_name, original) + self._original_methods.clear() + logger.debug("Firecrawl instrumentation disabled") + + +__all__ = ["FirecrawlInstrumentor", "__version__"] diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py new file mode 100644 index 000000000..5d659b02e --- /dev/null +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -0,0 +1,167 @@ +"""wrapt wrappers for the supported Firecrawl v2 client methods.""" + +from __future__ import annotations + +from typing import Any, Callable, Mapping, Sequence +from urllib.parse import urlsplit + +from opentelemetry.trace import Span, Status, StatusCode, Tracer + +_FI_SPAN_KIND = "fi.span.kind" +_TOOL = "TOOL" +_RETRIEVAL_QUERY = "fi.retrieval.query" +_RETRIEVAL_DOCUMENT_COUNT = "fi.retrieval.document_count" +_JOB_ID = "firecrawl.job_id" +_LIMIT = "firecrawl.limit" +_PAGE_COUNT = "firecrawl.page_count" +_CREDITS_USED = "firecrawl.credits_used" +_MAX_QUERY_LENGTH = 1024 + +# One span per call. crawl polls internally, so it must not emit one span per page. +_CRAWL_METHODS = {"crawl", "start_crawl", "get_crawl_status", "cancel_crawl"} + + +def _redact(value: str, instance: Any) -> str: + api_key = getattr(instance, "api_key", None) + if isinstance(api_key, str) and api_key: + value = value.replace(api_key, "[redacted]") + return value[:_MAX_QUERY_LENGTH] + + +def _url_host(value: Any, instance: Any) -> str: + """Record only the host. The path and body stay off the span.""" + if not isinstance(value, str) or not value: + return "" + host = urlsplit(value).hostname or "" + return _redact(host, instance) + + +def _document_count(result: Any) -> int: + for attribute in ("data", "results", "web"): + value = getattr(result, attribute, None) + if value is not None: + try: + return len(value) + except TypeError: + return 0 + return 0 + + +def _page_count(result: Any) -> int: + count = getattr(result, "completed", None) + if isinstance(count, int): + return count + data = getattr(result, "data", None) + if data is not None: + try: + return len(data) + except TypeError: + return 0 + return 0 + + +def _job_id(result: Any, kwargs: Mapping[str, Any]) -> str: + for attribute in ("id", "job_id"): + value = getattr(result, attribute, None) + if isinstance(value, str) and value: + return value + for key in ("job_id", "crawl_id"): + value = kwargs.get(key) + if isinstance(value, str) and value: + return value + return "" + + +class _BaseWrapper: + def __init__(self, tracer: Tracer, span_name: str, method_name: str) -> None: + self._tracer = tracer + self._span_name = span_name + self._method_name = method_name + + def _attributes( + self, instance: Any, args: Sequence[Any], kwargs: Mapping[str, Any] + ) -> dict[str, Any]: + attributes = {_FI_SPAN_KIND: _TOOL} + if self._method_name == "search": + query = kwargs.get("query", args[0] if args else "") + attributes[_RETRIEVAL_QUERY] = _redact(str(query or ""), instance) + elif self._method_name in ("scrape", "map", "crawl", "start_crawl"): + url = kwargs.get("url", args[0] if args else "") + host = _url_host(url, instance) + if host: + attributes["server.address"] = host + if self._method_name in _CRAWL_METHODS: + limit = kwargs.get("limit") + if isinstance(limit, int): + attributes[_LIMIT] = limit + job_id = _job_id(None, kwargs) + if job_id: + attributes[_JOB_ID] = _redact(job_id, instance) + return attributes + + def _finish_ok(self, span: Span, result: Any, kwargs: Mapping[str, Any]) -> None: + if self._method_name == "search": + span.set_attribute(_RETRIEVAL_DOCUMENT_COUNT, _document_count(result)) + if self._method_name in _CRAWL_METHODS: + job_id = _job_id(result, kwargs) + if job_id: + span.set_attribute(_JOB_ID, _redact(job_id, span)) + if self._method_name == "crawl": + span.set_attribute(_PAGE_COUNT, _page_count(result)) + credits = getattr(result, "credits_used", None) + if isinstance(credits, int): + span.set_attribute(_CREDITS_USED, credits) + span.set_status(Status(StatusCode.OK)) + span.end() + + @staticmethod + def _finish_error(span: Span, error: BaseException) -> None: + span.record_exception(error) + span.set_status( + Status(StatusCode.ERROR, "{0}: {1}".format(type(error).__name__, error)) + ) + span.end() + + +class OperationWrapper(_BaseWrapper): + """Trace a synchronous Firecrawl v2 operation.""" + + def __call__( + self, + wrapped: Callable[..., Any], + instance: Any, + args: tuple, + kwargs: Mapping[str, Any], + ) -> Any: + span = self._tracer.start_span( + self._span_name, attributes=self._attributes(instance, args, kwargs) + ) + try: + result = wrapped(*args, **kwargs) + except BaseException as error: + self._finish_error(span, error) + raise + self._finish_ok(span, result, kwargs) + return result + + +class AsyncOperationWrapper(_BaseWrapper): + """Trace an asynchronous Firecrawl v2 operation.""" + + async def __call__( + self, + wrapped: Callable[..., Any], + instance: Any, + args: tuple, + kwargs: Mapping[str, Any], + ) -> Any: + span = self._tracer.start_span( + self._span_name, attributes=self._attributes(instance, args, kwargs) + ) + try: + result = await wrapped(*args, **kwargs) + except BaseException as error: + self._finish_error(span, error) + raise + self._finish_ok(span, result, kwargs) + return result diff --git a/python/frameworks/firecrawl/traceai_firecrawl/package.py b/python/frameworks/firecrawl/traceai_firecrawl/package.py new file mode 100644 index 000000000..c2274f157 --- /dev/null +++ b/python/frameworks/firecrawl/traceai_firecrawl/package.py @@ -0,0 +1 @@ +_instruments = ("firecrawl-py == 4.46.2",) diff --git a/python/frameworks/firecrawl/traceai_firecrawl/py.typed b/python/frameworks/firecrawl/traceai_firecrawl/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/python/frameworks/firecrawl/traceai_firecrawl/version.py b/python/frameworks/firecrawl/traceai_firecrawl/version.py new file mode 100644 index 000000000..3dc1f76bc --- /dev/null +++ b/python/frameworks/firecrawl/traceai_firecrawl/version.py @@ -0,0 +1 @@ +__version__ = "0.1.0" From 3eaadc8bdbf428a472db6779091bbce3d6d89015 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 15:59:06 -0700 Subject: [PATCH 04/22] fix(harness): decode chunked OTLP bodies and record request metadata The Node OTLP exporter streams with Transfer-Encoding: chunked and sends no Content-Length. The Receiver read Content-Length only, so every Node export came back 400 with zero spans; two TH-8103 children had to put a de-chunking relay in front of it. Receiver now accepts both framings. It also records one entry per accepted export (path, lower-cased headers, flattened resource attributes) via requests(), so a contract test can assert X-Api-Key/X-Secret-Key and project_name through the shared harness instead of a private recorder. Verified: 6 harness tests pass, and the TanStack example's real Node exporter delivered 4 chunked exports (4 spans, collector path, both auth headers, project_name/project_type) with no relay. Refs: TH-8339, TH-8103 --- python/tests/harness/__init__.py | 61 ++++++++++++++++++++-- python/tests/harness/test_harness.py | 76 ++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index cb510ad10..cf79047f1 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -23,18 +23,36 @@ class Receiver: def __init__(self) -> None: self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] self._lock = threading.Lock() owner = self class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return try: - length = int(self.headers.get("Content-Length", "0")) - body = self.rfile.read(length) + body = self._read_body() content_type = self.headers.get("Content-Type", "").lower() if content_type.startswith("application/json"): @@ -53,9 +71,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler return spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] for resource_spans in request.get( "resourceSpans", request.get("resource_spans", []) ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) for scope_spans in resource_spans.get( "scopeSpans", resource_spans.get("scope_spans", []) ): @@ -66,6 +90,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler with owner._lock: owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) self.send_response(HTTPStatus.OK) self.send_header("Content-Length", "0") self.end_headers() @@ -93,10 +126,30 @@ def spans(self) -> list[dict[str, Any]]: with self._lock: return copy.deepcopy(self._spans) + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + def clear(self) -> None: - """Remove every decoded span received so far.""" + """Remove every decoded span and request record received so far.""" with self._lock: self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat def run( diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 6a32c2af8..bd61d5adc 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -50,6 +50,82 @@ def test_receiver_accepts_the_collector_path() -> None: assert receiver.spans() == [{"name": "collector.path"}] +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From 7b89f6aa9d3672b8d596f1a8698e404b0db78213 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 16:49:25 -0700 Subject: [PATCH 05/22] test(firecrawl): collector contract through the shared harness The real firecrawl-py 4.46.2 v2 client calls a loopback fake of the Firecrawl API (Firecrawl(api_url=...)): scrape, search, crawl including its internal status poll, and a scrape that gets HTTP 500. Spans leave through the real fi_instrumentation register() exporter into harness.Receiver (3eaadc8). Asserts the collector path, both auth headers, project_name/project_type=observe, one TOOL span per call (the crawl poll adds none), page count, limit, credits and host on the crawl, query and count on search, ERROR status plus exception event on the 500, and that the key, URL path, page body and result titles never leave. Mutation check: recording the full URL instead of the host fails both contract tests. 5 passed (3 unit + 2 contract). approved: Nikhil 2026-10-03 blanket Refs: TH-8325 --- .../firecrawl/tests/test_contract_harness.py | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 python/frameworks/firecrawl/tests/test_contract_harness.py diff --git a/python/frameworks/firecrawl/tests/test_contract_harness.py b/python/frameworks/firecrawl/tests/test_contract_harness.py new file mode 100644 index 000000000..742f669bb --- /dev/null +++ b/python/frameworks/firecrawl/tests/test_contract_harness.py @@ -0,0 +1,205 @@ +"""Shared-harness contract test for traceAI-firecrawl (TH-8325, TH-8339 Receiver). + +The real ``firecrawl-py`` v2 client makes real HTTP calls to a loopback fake +of the Firecrawl API (``Firecrawl(api_url=...)``), including the crawl status +poll. Spans leave through the real ``fi_instrumentation.register()`` OTLP +exporter into ``harness.Receiver``. Nothing calls Firecrawl; every key is a +placeholder. +""" + +from __future__ import annotations + +import json +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Dict, List, Tuple + +import pytest + +pytest.importorskip("firecrawl", reason="firecrawl-py must be installed to test its instrumentor") +pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) + +from harness import Receiver, _flatten_attributes # noqa: E402 + +FI_API_KEY = "placeholder-fi-api-key" +FI_SECRET_KEY = "placeholder-fi-secret-key" +FIRECRAWL_KEY = "fc-placeholder-key-must-not-be-exported" +PROJECT = "firecrawl-contract" +# Content the fake API returns or the caller sends. None of it may reach a span. +PAGE_BODY = "PAGE-BODY-MUST-NOT-BE-EXPORTED" +RESULT_TITLE = "RESULT-TITLE-MUST-NOT-BE-EXPORTED" +SECRET_PATH = "/private/path-must-not-be-exported" + +_PAGE = {"markdown": PAGE_BODY, "metadata": {"sourceURL": "https://example.com/1", "statusCode": 200}} +_ROUTES: Dict[Tuple[str, str], Tuple[int, Dict[str, Any]]] = { + ("POST", "/v2/scrape"): (200, {"success": True, "data": _PAGE}), + ("POST", "/v2/search"): ( + 200, + { + "success": True, + "data": { + "web": [ + {"url": "https://example.com/a", "title": RESULT_TITLE, "description": PAGE_BODY}, + {"url": "https://example.com/b", "title": RESULT_TITLE, "description": PAGE_BODY}, + ] + }, + }, + ), + ("POST", "/v2/crawl"): (200, {"success": True, "id": "crawl-job-1", "url": "unused"}), + ("GET", "/v2/crawl/crawl-job-1"): ( + 200, + { + "success": True, + "status": "completed", + "total": 3, + "completed": 3, + "creditsUsed": 3, + "expiresAt": "2030-01-01T00:00:00Z", + "data": [_PAGE, _PAGE, _PAGE], + }, + ), +} + + +class FakeFirecrawl: + """Loopback stand-in for api.firecrawl.dev. Scraping a ``/fail-500`` URL gets HTTP 500.""" + + def __init__(self) -> None: + self.calls: List[Tuple[str, str]] = [] + owner = self + + class Handler(BaseHTTPRequestHandler): + def _respond(self, method: str) -> None: + length = int(self.headers.get("Content-Length", "0") or 0) + body = json.loads(self.rfile.read(length) or b"{}") if length else {} + path = self.path.split("?")[0] + owner.calls.append((method, path)) + status, payload = _ROUTES.get((method, path), (404, {"success": False, "error": "not found"})) + if str(body.get("url", "")).endswith("/fail-500"): + status, payload = 500, {"success": False, "error": "internal error"} + data = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_POST(self) -> None: # noqa: N802 + self._respond("POST") + + def do_GET(self) -> None: # noqa: N802 + self._respond("GET") + + def log_message(self, *_: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = f"http://127.0.0.1:{self._server.server_port}" + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def close(self) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + +@pytest.fixture() +def fake_firecrawl(): + server = FakeFirecrawl() + yield server + server.close() + + +def _journey(receiver: Receiver, fake: FakeFirecrawl, monkeypatch: pytest.MonkeyPatch) -> List[dict]: + from firecrawl import Firecrawl + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + from traceai_firecrawl import FirecrawlInstrumentor + + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv("FI_API_KEY", FI_API_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET_KEY) + provider = register(project_type=ProjectType.OBSERVE, project_name=PROJECT, verbose=False) + instrumentor = FirecrawlInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + client = Firecrawl(api_key=FIRECRAWL_KEY, api_url=fake.origin, max_retries=1) + assert client.scrape("https://example.com" + SECRET_PATH).markdown == PAGE_BODY + assert len(client.search("open telemetry", limit=2).web) == 2 + job = client.crawl("https://example.com", limit=3, poll_interval=0) + assert job.status == "completed" and len(job.data) == 3 + with pytest.raises(Exception): + client.scrape("https://example.com/fail-500") + assert provider.force_flush(timeout_millis=10_000) + finally: + instrumentor.uninstrument() + provider.shutdown() + return receiver.spans() + + +def test_real_client_calls_reach_the_collector_contract(fake_firecrawl, monkeypatch): + with Receiver() as receiver: + spans = _journey(receiver, fake_firecrawl, monkeypatch) + exports = receiver.requests() + + # The real SDK made every HTTP call, including the crawl status poll. + assert fake_firecrawl.calls == [ + ("POST", "/v2/scrape"), + ("POST", "/v2/search"), + ("POST", "/v2/crawl"), + ("GET", "/v2/crawl/crawl-job-1"), + ("POST", "/v2/scrape"), + ] + + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_API_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET_KEY + assert "authorization" not in export["headers"] + for resource in export["resource_attributes"]: + assert resource["project_name"] == PROJECT + assert resource["project_type"] == "observe" + + names = [span["name"] for span in spans] + # One span per call: the crawl's internal status poll does not add a span. + assert sorted(names) == sorted( + ["firecrawl.scrape", "firecrawl.search", "firecrawl.crawl", "firecrawl.scrape"] + ) + by_name: Dict[str, List[dict]] = {} + for span in spans: + by_name.setdefault(span["name"], []).append(span) + + for span in spans: + assert _flatten_attributes(span["attributes"])["fi.span.kind"] == "TOOL" + + crawl = _flatten_attributes(by_name["firecrawl.crawl"][0]["attributes"]) + assert int(crawl["firecrawl.page_count"]) == 3 + assert int(crawl["firecrawl.limit"]) == 3 + assert int(crawl["firecrawl.credits_used"]) == 3 + assert crawl["server.address"] == "example.com" + + search = _flatten_attributes(by_name["firecrawl.search"][0]["attributes"]) + assert search["fi.retrieval.query"] == "open telemetry" + assert int(search["fi.retrieval.document_count"]) == 2 + + ok_scrape, failed_scrape = by_name["firecrawl.scrape"] + assert _flatten_attributes(ok_scrape["attributes"])["server.address"] == "example.com" + assert failed_scrape["status"]["code"] == "STATUS_CODE_ERROR" + assert any(event["name"] == "exception" for event in failed_scrape.get("events", [])) + + +def test_no_key_path_or_content_is_exported(fake_firecrawl, monkeypatch): + with Receiver() as receiver: + spans = _journey(receiver, fake_firecrawl, monkeypatch) + exports = receiver.requests() + + wire = json.dumps(spans) + json.dumps(exports) + for secret in (FIRECRAWL_KEY, PAGE_BODY, RESULT_TITLE, SECRET_PATH): + assert secret not in wire, secret From c5c1f5323e6e5bcea22561d65e9bad92b9803d82 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 23:05:13 -0700 Subject: [PATCH 06/22] fix(firecrawl): emit one span per call when the async crawl nests start_crawl firecrawl-py 4.46.2 AsyncFirecrawlClient.crawl awaits self.start_crawl, which is wrapped too, so an async blocking crawl exported a sibling firecrawl.start_crawl span next to firecrawl.crawl. A context variable now marks a traced Firecrawl call as active; nested wrapped calls in the same context run untraced. Adds a loopback fake of the v2 API, real-client tests for sync and async crawl, and an async contract test through the shared harness. R1 (TH-8325 installed review). approved: Nikhil 2026-10-03 blanket Refs: TH-8325 --- .../firecrawl/tests/_firecrawl_fake.py | 125 ++++++++++++++++++ .../firecrawl/tests/test_contract_harness.py | 41 ++++++ .../frameworks/firecrawl/tests/test_spans.py | 105 +++++++++++++++ .../firecrawl/traceai_firecrawl/_wrappers.py | 17 +++ 4 files changed, 288 insertions(+) create mode 100644 python/frameworks/firecrawl/tests/_firecrawl_fake.py create mode 100644 python/frameworks/firecrawl/tests/test_spans.py diff --git a/python/frameworks/firecrawl/tests/_firecrawl_fake.py b/python/frameworks/firecrawl/tests/_firecrawl_fake.py new file mode 100644 index 000000000..ef4861602 --- /dev/null +++ b/python/frameworks/firecrawl/tests/_firecrawl_fake.py @@ -0,0 +1,125 @@ +"""Loopback fake of the Firecrawl v2 HTTP API for traceAI-firecrawl tests. + +The real ``firecrawl-py`` clients (sync ``requests`` and async ``httpx``) talk to +this server through ``api_url``. Nothing reaches api.firecrawl.dev, and every +key used against it is a placeholder. +""" + +from __future__ import annotations + +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Callable, Dict, List, Tuple, Union + +JOB_ID = "crawl-job-1" +PAGE_BODY = "PAGE-BODY-MUST-NOT-BE-EXPORTED" +RESULT_TITLE = "RESULT-TITLE-MUST-NOT-BE-EXPORTED" + +PAGE = {"markdown": PAGE_BODY, "metadata": {"sourceURL": "https://example.com/1", "statusCode": 200}} + +Payload = Union[Dict[str, Any], Callable[[Dict[str, Any]], Dict[str, Any]]] +Route = Tuple[int, Payload] + + +def crawl_status(status: str, pages: int = 3) -> Dict[str, Any]: + """Body of ``GET /v2/crawl/{id}`` for a job in ``status``.""" + return { + "success": True, + "status": status, + "total": pages, + "completed": pages, + "creditsUsed": pages, + "expiresAt": "2030-01-01T00:00:00Z", + "data": [PAGE] * pages, + } + + +def default_routes() -> Dict[Tuple[str, str], Route]: + return { + ("POST", "/v2/scrape"): (200, {"success": True, "data": PAGE}), + ("POST", "/v2/search"): ( + 200, + { + "success": True, + "data": { + "web": [ + {"url": "https://example.com/a", "title": RESULT_TITLE, "description": PAGE_BODY}, + {"url": "https://example.com/b", "title": RESULT_TITLE, "description": PAGE_BODY}, + ] + }, + }, + ), + ("POST", "/v2/map"): ( + 200, + { + "success": True, + "links": [ + {"url": "https://example.com/private/a", "title": RESULT_TITLE}, + {"url": "https://example.com/private/b", "title": RESULT_TITLE}, + ], + }, + ), + ("POST", "/v2/crawl"): (200, {"success": True, "id": JOB_ID, "url": "unused"}), + ("GET", "/v2/crawl/" + JOB_ID): (200, crawl_status("completed")), + ("DELETE", "/v2/crawl/" + JOB_ID): (200, {"success": True, "status": "cancelled"}), + } + + +class FakeFirecrawl: + """Threaded loopback HTTP server that answers like the Firecrawl v2 API. + + ``routes`` maps ``(method, path)`` to ``(status, payload)``. A payload may be + a callable that receives the decoded request body. Unknown routes get 404. + """ + + def __init__(self) -> None: + self.routes: Dict[Tuple[str, str], Route] = default_routes() + self.calls: List[Tuple[str, str]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def _respond(self, method: str) -> None: + length = int(self.headers.get("Content-Length", "0") or 0) + body = json.loads(self.rfile.read(length) or b"{}") if length else {} + path = self.path.split("?")[0] + with owner._lock: + owner.calls.append((method, path)) + status, payload = owner.routes.get( + (method, path), (404, {"success": False, "error": "not found"}) + ) + if callable(payload): + payload = payload(body) + data = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_POST(self) -> None: # noqa: N802 + self._respond("POST") + + def do_GET(self) -> None: # noqa: N802 + self._respond("GET") + + def do_DELETE(self) -> None: # noqa: N802 + self._respond("DELETE") + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def calls_to(self, method: str, path: str) -> int: + with self._lock: + return self.calls.count((method, path)) + + def close(self) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() diff --git a/python/frameworks/firecrawl/tests/test_contract_harness.py b/python/frameworks/firecrawl/tests/test_contract_harness.py index 742f669bb..8db973355 100644 --- a/python/frameworks/firecrawl/tests/test_contract_harness.py +++ b/python/frameworks/firecrawl/tests/test_contract_harness.py @@ -195,6 +195,47 @@ def test_real_client_calls_reach_the_collector_contract(fake_firecrawl, monkeypa assert any(event["name"] == "exception" for event in failed_scrape.get("events", [])) +def test_async_client_emits_one_span_per_call(fake_firecrawl, monkeypatch): + """AsyncFirecrawl.crawl awaits the wrapped start_crawl; that must not add a span.""" + import asyncio + + from firecrawl import AsyncFirecrawl + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + from traceai_firecrawl import FirecrawlInstrumentor + + async def journey() -> None: + client = AsyncFirecrawl(api_key=FIRECRAWL_KEY, api_url=fake_firecrawl.origin, max_retries=1) + await client.scrape("https://example.com" + SECRET_PATH) + job = await client.crawl(url="https://example.com", limit=3, poll_interval=0) + assert job.status == "completed" and len(job.data) == 3 + + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv("FI_API_KEY", FI_API_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET_KEY) + provider = register(project_type=ProjectType.OBSERVE, project_name=PROJECT, verbose=False) + instrumentor = FirecrawlInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + asyncio.run(journey()) + assert provider.force_flush(timeout_millis=10_000) + finally: + instrumentor.uninstrument() + provider.shutdown() + spans = receiver.spans() + + assert fake_firecrawl.calls == [ + ("POST", "/v2/scrape"), + ("POST", "/v2/crawl"), + ("GET", "/v2/crawl/crawl-job-1"), + ] + assert sorted(span["name"] for span in spans) == ["firecrawl.crawl", "firecrawl.scrape"] + crawl = next(span for span in spans if span["name"] == "firecrawl.crawl") + assert _flatten_attributes(crawl["attributes"])["firecrawl.job_id"] == "crawl-job-1" + + def test_no_key_path_or_content_is_exported(fake_firecrawl, monkeypatch): with Receiver() as receiver: spans = _journey(receiver, fake_firecrawl, monkeypatch) diff --git a/python/frameworks/firecrawl/tests/test_spans.py b/python/frameworks/firecrawl/tests/test_spans.py new file mode 100644 index 000000000..5cec5bc59 --- /dev/null +++ b/python/frameworks/firecrawl/tests/test_spans.py @@ -0,0 +1,105 @@ +"""Span contract for the real firecrawl-py v2 clients against a loopback fake. + +Every test drives the installed SDK (sync ``requests`` and async ``httpx``) at +``_firecrawl_fake.FakeFirecrawl`` and reads the spans from an in-memory +exporter. No call leaves 127.0.0.1 and every key is a placeholder. +""" + +from __future__ import annotations + +import asyncio +from typing import Any, Dict, Iterator, List, Tuple + +import pytest +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +pytest.importorskip("firecrawl", reason="firecrawl-py must be installed to test its instrumentor") + +from _firecrawl_fake import JOB_ID, FakeFirecrawl # noqa: E402 +from traceai_firecrawl import FirecrawlInstrumentor # noqa: E402 + +API_KEY = "fc-placeholder-key-must-not-be-exported" + +Tracing = Tuple[TracerProvider, InMemorySpanExporter, FirecrawlInstrumentor] + + +@pytest.fixture() +def fake() -> Iterator[FakeFirecrawl]: + server = FakeFirecrawl() + yield server + server.close() + + +@pytest.fixture() +def tracing() -> Iterator[Tracing]: + exporter = InMemorySpanExporter() + provider = TracerProvider(resource=Resource.create({"service.name": "test-firecrawl"})) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = FirecrawlInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + yield provider, exporter, instrumentor + finally: + instrumentor.uninstrument() + + +def sync_client(fake: FakeFirecrawl) -> Any: + from firecrawl import Firecrawl + + return Firecrawl(api_key=API_KEY, api_url=fake.origin, max_retries=1) + + +def async_client(fake: FakeFirecrawl) -> Any: + # Build inside the running loop: the client owns an httpx.AsyncClient. + from firecrawl import AsyncFirecrawl + + return AsyncFirecrawl(api_key=API_KEY, api_url=fake.origin, max_retries=1) + + +def spans(exporter: InMemorySpanExporter) -> List[ReadableSpan]: + return list(exporter.get_finished_spans()) + + +def attrs(span: ReadableSpan) -> Dict[str, Any]: + return dict(span.attributes or {}) + + +# R1: one span per user call. AsyncFirecrawlClient.crawl awaits self.start_crawl +# (firecrawl/v2/client_async.py), which is wrapped too; it must not add a span. + + +def test_async_crawl_emits_exactly_one_crawl_span(fake: FakeFirecrawl, tracing: Tracing) -> None: + _, exporter, _ = tracing + + async def journey() -> Any: + client = async_client(fake) + job = await client.crawl(url="https://example.com", limit=3, poll_interval=0) + await client.scrape("https://example.com/next") + return job + + job = asyncio.run(journey()) + + assert job.status == "completed" and len(job.data) == 3 + assert fake.calls_to("POST", "/v2/crawl") == 1 + names = [span.name for span in spans(exporter)] + # The nested start_crawl adds nothing, and the guard is released afterwards. + assert names == ["firecrawl.crawl", "firecrawl.scrape"] + crawl = attrs(spans(exporter)[0]) + assert crawl["firecrawl.job_id"] == JOB_ID + assert crawl["firecrawl.page_count"] == 3 + assert crawl["firecrawl.limit"] == 3 + + +def test_sync_crawl_emits_exactly_one_crawl_span(fake: FakeFirecrawl, tracing: Tracing) -> None: + _, exporter, _ = tracing + + client = sync_client(fake) + job = client.crawl("https://example.com", limit=3, poll_interval=0) + client.scrape("https://example.com/next") + + assert job.status == "completed" + assert [span.name for span in spans(exporter)] == ["firecrawl.crawl", "firecrawl.scrape"] + assert attrs(spans(exporter)[0])["firecrawl.job_id"] == JOB_ID diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index 5d659b02e..28106b1f9 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -2,6 +2,7 @@ from __future__ import annotations +from contextvars import ContextVar from typing import Any, Callable, Mapping, Sequence from urllib.parse import urlsplit @@ -20,6 +21,12 @@ # One span per call. crawl polls internally, so it must not emit one span per page. _CRAWL_METHODS = {"crawl", "start_crawl", "get_crawl_status", "cancel_crawl"} +# True while a traced Firecrawl call runs in this context. firecrawl-py 4.46.2 +# AsyncFirecrawlClient.crawl awaits self.start_crawl, which is wrapped as well; +# the nested call runs untraced so one user call yields one span. A context +# variable follows asyncio tasks, so concurrent calls do not suppress each other. +_ACTIVE: ContextVar[bool] = ContextVar("traceai_firecrawl_active", default=False) + def _redact(value: str, instance: Any) -> str: api_key = getattr(instance, "api_key", None) @@ -133,14 +140,19 @@ def __call__( args: tuple, kwargs: Mapping[str, Any], ) -> Any: + if _ACTIVE.get(): + return wrapped(*args, **kwargs) span = self._tracer.start_span( self._span_name, attributes=self._attributes(instance, args, kwargs) ) + token = _ACTIVE.set(True) try: result = wrapped(*args, **kwargs) except BaseException as error: self._finish_error(span, error) raise + finally: + _ACTIVE.reset(token) self._finish_ok(span, result, kwargs) return result @@ -155,13 +167,18 @@ async def __call__( args: tuple, kwargs: Mapping[str, Any], ) -> Any: + if _ACTIVE.get(): + return await wrapped(*args, **kwargs) span = self._tracer.start_span( self._span_name, attributes=self._attributes(instance, args, kwargs) ) + token = _ACTIVE.set(True) try: result = await wrapped(*args, **kwargs) except BaseException as error: self._finish_error(span, error) raise + finally: + _ACTIVE.reset(token) self._finish_ok(span, result, kwargs) return result From 4c2d095758292bc20f4ddaf01634081d99971fa7 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 23:07:21 -0700 Subject: [PATCH 07/22] fix(firecrawl): read the crawl job id from positional arguments get_crawl_status(job_id) and cancel_crawl(crawl_id) take the id as the first positional argument. cancel_crawl returns a bool, so its span never carried firecrawl.job_id, and a failed status call had none either. The start attributes now read args[0] for those two methods, so start, status and cancel share the job id (AC-03), sync and async, including on errors. Also shortens the fake server's shutdown poll. R2 (TH-8325 installed review). approved: Nikhil 2026-10-03 blanket Refs: TH-8325 --- .../firecrawl/tests/_firecrawl_fake.py | 2 +- .../frameworks/firecrawl/tests/test_spans.py | 65 +++++++++++++++++++ .../firecrawl/traceai_firecrawl/_wrappers.py | 13 +++- 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/python/frameworks/firecrawl/tests/_firecrawl_fake.py b/python/frameworks/firecrawl/tests/_firecrawl_fake.py index ef4861602..546c6350f 100644 --- a/python/frameworks/firecrawl/tests/_firecrawl_fake.py +++ b/python/frameworks/firecrawl/tests/_firecrawl_fake.py @@ -112,7 +112,7 @@ def log_message(self, format: str, *args: Any) -> None: # noqa: A002 self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) - self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread = threading.Thread(target=self._server.serve_forever, args=(0.05,), daemon=True) self._thread.start() def calls_to(self, method: str, path: str) -> int: diff --git a/python/frameworks/firecrawl/tests/test_spans.py b/python/frameworks/firecrawl/tests/test_spans.py index 5cec5bc59..878670aed 100644 --- a/python/frameworks/firecrawl/tests/test_spans.py +++ b/python/frameworks/firecrawl/tests/test_spans.py @@ -15,6 +15,7 @@ from opentelemetry.sdk.trace import ReadableSpan, TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode pytest.importorskip("firecrawl", reason="firecrawl-py must be installed to test its instrumentor") @@ -103,3 +104,67 @@ def test_sync_crawl_emits_exactly_one_crawl_span(fake: FakeFirecrawl, tracing: T assert job.status == "completed" assert [span.name for span in spans(exporter)] == ["firecrawl.crawl", "firecrawl.scrape"] assert attrs(spans(exporter)[0])["firecrawl.job_id"] == JOB_ID + + +# R2 / AC-03: start, status and cancel share firecrawl.job_id, including when the +# id is passed positionally (cancel_crawl returns a bool, so the id must come +# from the call) and when the status call fails. + +TRIO = ["firecrawl.start_crawl", "firecrawl.get_crawl_status", "firecrawl.cancel_crawl"] + + +def test_sync_trio_with_positional_ids_shares_job_id(fake: FakeFirecrawl, tracing: Tracing) -> None: + _, exporter, _ = tracing + + client = sync_client(fake) + started = client.start_crawl("https://example.com", limit=3) + client.get_crawl_status(started.id) + assert client.cancel_crawl(started.id) is True + + assert [span.name for span in spans(exporter)] == TRIO + assert [attrs(span).get("firecrawl.job_id") for span in spans(exporter)] == [JOB_ID] * 3 + + +def test_async_trio_with_positional_ids_shares_job_id(fake: FakeFirecrawl, tracing: Tracing) -> None: + _, exporter, _ = tracing + + async def journey() -> None: + client = async_client(fake) + started = await client.start_crawl("https://example.com", limit=3) + await client.get_crawl_status(started.id) + assert await client.cancel_crawl(started.id) is True + + asyncio.run(journey()) + + assert [span.name for span in spans(exporter)] == TRIO + assert [attrs(span).get("firecrawl.job_id") for span in spans(exporter)] == [JOB_ID] * 3 + + +def test_failed_status_and_cancel_calls_keep_positional_job_id( + fake: FakeFirecrawl, tracing: Tracing +) -> None: + _, exporter, _ = tracing + client = sync_client(fake) + + with pytest.raises(Exception): + client.get_crawl_status("missing-job") + with pytest.raises(Exception): + client.cancel_crawl("missing-job") + + async def journey() -> None: + aclient = async_client(fake) + with pytest.raises(Exception): + await aclient.get_crawl_status("missing-job") + with pytest.raises(Exception): + await aclient.cancel_crawl("missing-job") + + asyncio.run(journey()) + + finished = spans(exporter) + assert [span.name for span in finished] == [ + "firecrawl.get_crawl_status", + "firecrawl.cancel_crawl", + ] * 2 + for span in finished: + assert span.status.status_code is StatusCode.ERROR + assert attrs(span)["firecrawl.job_id"] == "missing-job" diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index 28106b1f9..fe4ef4dd2 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -20,6 +20,7 @@ # One span per call. crawl polls internally, so it must not emit one span per page. _CRAWL_METHODS = {"crawl", "start_crawl", "get_crawl_status", "cancel_crawl"} +_JOB_ID_FIRST_ARG_METHODS = {"get_crawl_status", "cancel_crawl"} # True while a traced Firecrawl call runs in this context. firecrawl-py 4.46.2 # AsyncFirecrawlClient.crawl awaits self.start_crawl, which is wrapped as well; @@ -79,6 +80,16 @@ def _job_id(result: Any, kwargs: Mapping[str, Any]) -> str: return "" +def _call_job_id(method_name: str, args: Sequence[Any], kwargs: Mapping[str, Any]) -> str: + """Job id the caller passed. get_crawl_status(job_id) and cancel_crawl(crawl_id + sync / job_id async) take it as the first positional argument.""" + if method_name in _JOB_ID_FIRST_ARG_METHODS and args: + value = args[0] + if isinstance(value, str) and value: + return value + return _job_id(None, kwargs) + + class _BaseWrapper: def __init__(self, tracer: Tracer, span_name: str, method_name: str) -> None: self._tracer = tracer @@ -101,7 +112,7 @@ def _attributes( limit = kwargs.get("limit") if isinstance(limit, int): attributes[_LIMIT] = limit - job_id = _job_id(None, kwargs) + job_id = _call_job_id(self._method_name, args, kwargs) if job_id: attributes[_JOB_ID] = _redact(job_id, instance) return attributes From 9515703ce3703d0fed27d46dc2638a37a550bb70 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 23:08:03 -0700 Subject: [PATCH 08/22] fix(firecrawl): set firecrawl.status from the returned crawl job firecrawl-py returns failed and cancelled crawl jobs without raising, so a failed blocking crawl was exported as an OK span. crawl and get_crawl_status now record firecrawl.status from the job. A failed job sets the span to ERROR "failed"; a cancelled job sets ERROR "cancelled" and firecrawl.cancelled=true. Tested sync and async for completed, failed and cancelled jobs. R3 (TH-8325 installed review). approved: Nikhil 2026-10-03 blanket Refs: TH-8325 --- .../frameworks/firecrawl/tests/test_spans.py | 52 +++++++++++++++++++ .../firecrawl/traceai_firecrawl/_wrappers.py | 19 ++++++- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/python/frameworks/firecrawl/tests/test_spans.py b/python/frameworks/firecrawl/tests/test_spans.py index 878670aed..0c55aebdc 100644 --- a/python/frameworks/firecrawl/tests/test_spans.py +++ b/python/frameworks/firecrawl/tests/test_spans.py @@ -168,3 +168,55 @@ async def journey() -> None: for span in finished: assert span.status.status_code is StatusCode.ERROR assert attrs(span)["firecrawl.job_id"] == "missing-job" + + +# R3 / J3: the SDK returns a failed or cancelled job without raising, so the span +# takes its status from the job. failed -> ERROR; cancelled -> ERROR "cancelled" +# plus firecrawl.cancelled=true; completed -> OK. + +EXPECTED_JOB_STATUS = { + "completed": (StatusCode.OK, None), + "failed": (StatusCode.ERROR, "failed"), + "cancelled": (StatusCode.ERROR, "cancelled"), +} + + +def _crawl_and_status(fake: FakeFirecrawl, use_async: bool) -> None: + if not use_async: + client = sync_client(fake) + client.crawl("https://example.com", limit=3, poll_interval=0) + client.get_crawl_status(JOB_ID) + return + + async def journey() -> None: + client = async_client(fake) + await client.crawl(url="https://example.com", limit=3, poll_interval=0) + await client.get_crawl_status(JOB_ID) + + asyncio.run(journey()) + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("job_status", sorted(EXPECTED_JOB_STATUS)) +def test_job_status_sets_attribute_and_span_status( + fake: FakeFirecrawl, tracing: Tracing, job_status: str, use_async: bool +) -> None: + from _firecrawl_fake import crawl_status + + _, exporter, _ = tracing + fake.routes[("GET", "/v2/crawl/" + JOB_ID)] = (200, crawl_status(job_status)) + + _crawl_and_status(fake, use_async) + + finished = spans(exporter) + assert [span.name for span in finished] == ["firecrawl.crawl", "firecrawl.get_crawl_status"] + code, description = EXPECTED_JOB_STATUS[job_status] + for span in finished: + assert attrs(span)["firecrawl.status"] == job_status + assert span.status.status_code is code + if description is not None: + assert span.status.description == description + if job_status == "cancelled": + assert attrs(span)["firecrawl.cancelled"] is True + else: + assert "firecrawl.cancelled" not in attrs(span) diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index fe4ef4dd2..8fbb498a8 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -16,11 +16,18 @@ _LIMIT = "firecrawl.limit" _PAGE_COUNT = "firecrawl.page_count" _CREDITS_USED = "firecrawl.credits_used" +_STATUS = "firecrawl.status" +_CANCELLED = "firecrawl.cancelled" _MAX_QUERY_LENGTH = 1024 # One span per call. crawl polls internally, so it must not emit one span per page. _CRAWL_METHODS = {"crawl", "start_crawl", "get_crawl_status", "cancel_crawl"} _JOB_ID_FIRST_ARG_METHODS = {"get_crawl_status", "cancel_crawl"} +# Methods that return a CrawlJob. The SDK returns failed and cancelled jobs +# without raising (methods/crawl.py wait_for_crawl_completion), so the span +# status comes from the job. +_JOB_STATUS_METHODS = {"crawl", "get_crawl_status"} +_ERROR_JOB_STATUSES = {"failed", "cancelled"} # True while a traced Firecrawl call runs in this context. firecrawl-py 4.46.2 # AsyncFirecrawlClient.crawl awaits self.start_crawl, which is wrapped as well; @@ -129,7 +136,17 @@ def _finish_ok(self, span: Span, result: Any, kwargs: Mapping[str, Any]) -> None credits = getattr(result, "credits_used", None) if isinstance(credits, int): span.set_attribute(_CREDITS_USED, credits) - span.set_status(Status(StatusCode.OK)) + job_status = ( + getattr(result, "status", None) if self._method_name in _JOB_STATUS_METHODS else None + ) + if isinstance(job_status, str) and job_status: + span.set_attribute(_STATUS, job_status) + if job_status == "cancelled": + span.set_attribute(_CANCELLED, True) + if job_status in _ERROR_JOB_STATUSES: + span.set_status(Status(StatusCode.ERROR, job_status)) + else: + span.set_status(Status(StatusCode.OK)) span.end() @staticmethod From cb016d92b3d93c2a9575b5c2d635383cc9614425 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 23:12:44 -0700 Subject: [PATCH 09/22] fix(firecrawl): isolate instrumentation errors from the caller Attribute extraction ran before the vendor call and outside any guard, so scrape("http://[::1/page") raised urlsplit's ValueError to the caller before the SDK ran. Reading the result and recording an exception were not guarded either: a result attribute that raised replaced the vendor's return value, and an exception whose str() failed replaced the vendor's exception. Each step is now isolated and logged at debug level, the span always ends exactly once, and a tracer that cannot start a span falls back to the plain vendor call. R5 (TH-8325 installed review). approved: Nikhil 2026-10-03 blanket Refs: TH-8325 --- .../frameworks/firecrawl/tests/test_spans.py | 97 +++++++++++++++++++ .../firecrawl/traceai_firecrawl/_wrappers.py | 78 ++++++++++++--- 2 files changed, 160 insertions(+), 15 deletions(-) diff --git a/python/frameworks/firecrawl/tests/test_spans.py b/python/frameworks/firecrawl/tests/test_spans.py index 0c55aebdc..d5523f1a7 100644 --- a/python/frameworks/firecrawl/tests/test_spans.py +++ b/python/frameworks/firecrawl/tests/test_spans.py @@ -220,3 +220,100 @@ def test_job_status_sets_attribute_and_span_status( assert attrs(span)["firecrawl.cancelled"] is True else: assert "firecrawl.cancelled" not in attrs(span) + + +# R5: instrumentation errors never reach the caller, and the span always ends. + + +def test_unparseable_url_does_not_break_the_call(fake: FakeFirecrawl, tracing: Tracing) -> None: + # urlsplit raises ValueError("Invalid IPv6 URL") for this string; the SDK + # itself sends it to the API unchanged. + _, exporter, _ = tracing + bad_url = "http://[::1/page" + + document = sync_client(fake).scrape(bad_url) + + async def journey() -> Any: + return await async_client(fake).scrape(bad_url) + + async_document = asyncio.run(journey()) + + assert document.markdown and async_document.markdown + finished = spans(exporter) + assert [span.name for span in finished] == ["firecrawl.scrape"] * 2 + assert all(attrs(span)["fi.span.kind"] == "TOOL" for span in finished) + + +class _ExplodingResult: + """A result whose attributes raise something other than AttributeError.""" + + @property + def data(self) -> Any: + raise RuntimeError("result attribute exploded") + + web = data + credits_used = data + + +def test_result_attribute_errors_do_not_reach_the_caller( + monkeypatch: pytest.MonkeyPatch, fake: FakeFirecrawl +) -> None: + from firecrawl.v2.client import FirecrawlClient + from firecrawl.v2.client_async import AsyncFirecrawlClient + + result = _ExplodingResult() + + def search(_self: Any, _query: str, **_kwargs: Any) -> Any: + return result + + async def asearch(_self: Any, _query: str, **_kwargs: Any) -> Any: + return result + + monkeypatch.setattr(FirecrawlClient, "search", search) + monkeypatch.setattr(AsyncFirecrawlClient, "search", asearch) + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = FirecrawlInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + assert sync_client(fake).search("query") is result + + async def journey() -> Any: + return await async_client(fake).search("query") + + assert asyncio.run(journey()) is result + finally: + instrumentor.uninstrument() + + assert [span.name for span in spans(exporter)] == ["firecrawl.search"] * 2 + + +class _UnprintableError(Exception): + def __str__(self) -> str: + raise RuntimeError("str() exploded") + + +def test_error_recording_failure_still_reraises_the_vendor_error( + monkeypatch: pytest.MonkeyPatch, fake: FakeFirecrawl +) -> None: + from firecrawl.v2.client import FirecrawlClient + + def scrape(_self: Any, _url: str, **_kwargs: Any) -> Any: + raise _UnprintableError() + + monkeypatch.setattr(FirecrawlClient, "scrape", scrape) + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = FirecrawlInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + with pytest.raises(_UnprintableError): + sync_client(fake).scrape("https://example.com") + finally: + instrumentor.uninstrument() + + finished = spans(exporter) + assert [span.name for span in finished] == ["firecrawl.scrape"] + assert finished[0].status.status_code is StatusCode.ERROR diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index 8fbb498a8..e3f536a7f 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -2,12 +2,15 @@ from __future__ import annotations +import logging from contextvars import ContextVar -from typing import Any, Callable, Mapping, Sequence +from typing import Any, Callable, Mapping, Optional, Sequence from urllib.parse import urlsplit from opentelemetry.trace import Span, Status, StatusCode, Tracer +logger = logging.getLogger(__name__) + _FI_SPAN_KIND = "fi.span.kind" _TOOL = "TOOL" _RETRIEVAL_QUERY = "fi.retrieval.query" @@ -98,6 +101,14 @@ def _call_job_id(method_name: str, args: Sequence[Any], kwargs: Mapping[str, Any class _BaseWrapper: + """Span bookkeeping shared by the sync and async wrappers. + + Everything here is isolated from the caller: an error while reading + arguments, reading the result or recording an exception is logged at debug + level and never replaces the vendor's own result or exception. Every span + that starts is ended exactly once. + """ + def __init__(self, tracer: Tracer, span_name: str, method_name: str) -> None: self._tracer = tracer self._span_name = span_name @@ -124,7 +135,19 @@ def _attributes( attributes[_JOB_ID] = _redact(job_id, instance) return attributes - def _finish_ok(self, span: Span, result: Any, kwargs: Mapping[str, Any]) -> None: + def _start(self, instance: Any, args: Sequence[Any], kwargs: Mapping[str, Any]) -> Optional[Span]: + try: + attributes = self._attributes(instance, args, kwargs) + except Exception: + logger.debug("Could not read %s arguments", self._span_name, exc_info=True) + attributes = {_FI_SPAN_KIND: _TOOL} + try: + return self._tracer.start_span(self._span_name, attributes=attributes) + except Exception: + logger.debug("Could not start %s span", self._span_name, exc_info=True) + return None + + def _record_result(self, span: Span, result: Any, kwargs: Mapping[str, Any]) -> None: if self._method_name == "search": span.set_attribute(_RETRIEVAL_DOCUMENT_COUNT, _document_count(result)) if self._method_name in _CRAWL_METHODS: @@ -147,15 +170,40 @@ def _finish_ok(self, span: Span, result: Any, kwargs: Mapping[str, Any]) -> None span.set_status(Status(StatusCode.ERROR, job_status)) else: span.set_status(Status(StatusCode.OK)) - span.end() - @staticmethod - def _finish_error(span: Span, error: BaseException) -> None: - span.record_exception(error) - span.set_status( - Status(StatusCode.ERROR, "{0}: {1}".format(type(error).__name__, error)) - ) + def _finish_ok(self, span: Span, result: Any, kwargs: Mapping[str, Any]) -> None: + try: + self._record_result(span, result, kwargs) + except Exception: + logger.debug("Could not read %s result", self._span_name, exc_info=True) + finally: + _end(span) + + def _finish_error(self, span: Span, error: BaseException) -> None: + try: + span.record_exception(error) + except Exception: + logger.debug("Could not record %s exception", self._span_name, exc_info=True) + try: + span.set_status(Status(StatusCode.ERROR, _describe(error))) + except Exception: + logger.debug("Could not set %s status", self._span_name, exc_info=True) + finally: + _end(span) + + +def _describe(error: BaseException) -> str: + try: + return "{0}: {1}".format(type(error).__name__, error) + except Exception: + return type(error).__name__ + + +def _end(span: Span) -> None: + try: span.end() + except Exception: + logger.debug("Could not end span", exc_info=True) class OperationWrapper(_BaseWrapper): @@ -170,9 +218,9 @@ def __call__( ) -> Any: if _ACTIVE.get(): return wrapped(*args, **kwargs) - span = self._tracer.start_span( - self._span_name, attributes=self._attributes(instance, args, kwargs) - ) + span = self._start(instance, args, kwargs) + if span is None: + return wrapped(*args, **kwargs) token = _ACTIVE.set(True) try: result = wrapped(*args, **kwargs) @@ -197,9 +245,9 @@ async def __call__( ) -> Any: if _ACTIVE.get(): return await wrapped(*args, **kwargs) - span = self._tracer.start_span( - self._span_name, attributes=self._attributes(instance, args, kwargs) - ) + span = self._start(instance, args, kwargs) + if span is None: + return await wrapped(*args, **kwargs) token = _ACTIVE.set(True) try: result = await wrapped(*args, **kwargs) From f78c281a4c316d300ac87ae3f1d03613d06fc7d5 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 23:13:56 -0700 Subject: [PATCH 10/22] fix(firecrawl): make the TOOL span current during the vendor call The wrappers started the span but never made it current, so HTTP client spans and anything else the SDK traces became siblings of the TOOL span instead of its children. The vendor call now runs inside trace.use_span(span, end_on_exit=False) with exception recording left to the wrapper, so an error is recorded once. Tests pin HTTP span parentage for sync and async, that the context is restored afterwards, and that a vendor error carries exactly one exception event. R6 (TH-8325 installed review). approved: Nikhil 2026-10-03 blanket Refs: TH-8325 --- .../frameworks/firecrawl/tests/test_spans.py | 57 +++++++++++++++++++ .../firecrawl/traceai_firecrawl/_wrappers.py | 15 ++++- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/python/frameworks/firecrawl/tests/test_spans.py b/python/frameworks/firecrawl/tests/test_spans.py index d5523f1a7..a13611a94 100644 --- a/python/frameworks/firecrawl/tests/test_spans.py +++ b/python/frameworks/firecrawl/tests/test_spans.py @@ -317,3 +317,60 @@ def scrape(_self: Any, _url: str, **_kwargs: Any) -> Any: finished = spans(exporter) assert [span.name for span in finished] == ["firecrawl.scrape"] assert finished[0].status.status_code is StatusCode.ERROR + + +# R6: the vendor call runs with the TOOL span current, so HTTP client spans +# (and anything else the SDK traces) nest under it. + + +def test_http_spans_nest_under_the_tool_span( + monkeypatch: pytest.MonkeyPatch, fake: FakeFirecrawl, tracing: Tracing +) -> None: + from firecrawl.v2.utils.http_client import HttpClient + from firecrawl.v2.utils.http_client_async import AsyncHttpClient + from opentelemetry import trace + + provider, exporter, _ = tracing + http_tracer = provider.get_tracer("fake-http-instrumentation") + sync_post, async_post = HttpClient.post, AsyncHttpClient.post + + def traced_post(self: Any, *args: Any, **kwargs: Any) -> Any: + with http_tracer.start_as_current_span("HTTP POST"): + return sync_post(self, *args, **kwargs) + + async def traced_async_post(self: Any, *args: Any, **kwargs: Any) -> Any: + with http_tracer.start_as_current_span("HTTP POST"): + return await async_post(self, *args, **kwargs) + + monkeypatch.setattr(HttpClient, "post", traced_post) + monkeypatch.setattr(AsyncHttpClient, "post", traced_async_post) + + sync_client(fake).scrape("https://example.com") + + async def journey() -> None: + await async_client(fake).scrape("https://example.com") + + asyncio.run(journey()) + # The TOOL span is not left current after the call returns. + assert not trace.get_current_span().get_span_context().is_valid + + finished = spans(exporter) + assert [span.name for span in finished] == ["HTTP POST", "firecrawl.scrape"] * 2 + for http_span, tool_span in (finished[0:2], finished[2:4]): + assert http_span.parent is not None + assert http_span.parent.span_id == tool_span.context.span_id + assert http_span.context.trace_id == tool_span.context.trace_id + + +def test_vendor_error_is_recorded_once_and_reraised(fake: FakeFirecrawl, tracing: Tracing) -> None: + from firecrawl.v2.utils.error_handler import FirecrawlError + + _, exporter, _ = tracing + fake.routes[("POST", "/v2/scrape")] = (500, {"success": False, "error": "internal error"}) + + with pytest.raises(FirecrawlError): + sync_client(fake).scrape("https://example.com") + + (span,) = spans(exporter) + assert span.status.status_code is StatusCode.ERROR + assert [event.name for event in span.events] == ["exception"] diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index e3f536a7f..892c85d85 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -7,6 +7,7 @@ from typing import Any, Callable, Mapping, Optional, Sequence from urllib.parse import urlsplit +from opentelemetry import trace as trace_api from opentelemetry.trace import Span, Status, StatusCode, Tracer logger = logging.getLogger(__name__) @@ -199,6 +200,14 @@ def _describe(error: BaseException) -> str: return type(error).__name__ +def _current(span: Span) -> Any: + """Make the span current for the vendor call so HTTP client spans nest under + it. The wrapper records the exception and ends the span itself.""" + return trace_api.use_span( + span, end_on_exit=False, record_exception=False, set_status_on_exception=False + ) + + def _end(span: Span) -> None: try: span.end() @@ -223,7 +232,8 @@ def __call__( return wrapped(*args, **kwargs) token = _ACTIVE.set(True) try: - result = wrapped(*args, **kwargs) + with _current(span): + result = wrapped(*args, **kwargs) except BaseException as error: self._finish_error(span, error) raise @@ -250,7 +260,8 @@ async def __call__( return await wrapped(*args, **kwargs) token = _ACTIVE.set(True) try: - result = await wrapped(*args, **kwargs) + with _current(span): + result = await wrapped(*args, **kwargs) except BaseException as error: self._finish_error(span, error) raise From 5495f982641ca44d7d35928656aa4076b5bd595c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 23:15:07 -0700 Subject: [PATCH 11/22] feat(firecrawl): record cancellation, requested formats and vendor error codes - AC-07: a cancelled asyncio task running a blocking crawl ends the span ERROR "cancelled" with firecrawl.cancelled=true. cancel_crawl records firecrawl.cancelled from the API's answer. - J1: scrape, search, crawl and start_crawl record firecrawl.formats, the requested format names from formats= or scrape_options.formats. A JSON format's prompt and schema stay off the span. - J5: a firecrawl-py FirecrawlError records firecrawl.error.status_code and firecrawl.error.code (for example 429 and the vendor's rate-limit code). R7 (TH-8325 installed review). approved: Nikhil 2026-10-03 blanket Refs: TH-8325 --- .../frameworks/firecrawl/tests/test_spans.py | 119 ++++++++++++++++++ .../firecrawl/traceai_firecrawl/_wrappers.py | 75 ++++++++++- 2 files changed, 190 insertions(+), 4 deletions(-) diff --git a/python/frameworks/firecrawl/tests/test_spans.py b/python/frameworks/firecrawl/tests/test_spans.py index a13611a94..1e1b2829b 100644 --- a/python/frameworks/firecrawl/tests/test_spans.py +++ b/python/frameworks/firecrawl/tests/test_spans.py @@ -374,3 +374,122 @@ def test_vendor_error_is_recorded_once_and_reraised(fake: FakeFirecrawl, tracing (span,) = spans(exporter) assert span.status.status_code is StatusCode.ERROR assert [event.name for event in span.events] == ["exception"] + + +# R7 / AC-07: cancellation sets firecrawl.cancelled=true. A cancelled asyncio task +# running a blocking crawl ends the span ERROR "cancelled"; cancel_crawl records +# whether the API cancelled the job. + + +def test_cancelling_an_async_crawl_task_marks_the_span_cancelled( + fake: FakeFirecrawl, tracing: Tracing +) -> None: + from _firecrawl_fake import crawl_status + + _, exporter, _ = tracing + fake.routes[("GET", "/v2/crawl/" + JOB_ID)] = (200, crawl_status("scraping")) + + async def journey() -> None: + client = async_client(fake) + task = asyncio.ensure_future( + client.crawl(url="https://example.com", limit=3, poll_interval=0.01) + ) + for _ in range(500): + if fake.calls_to("GET", "/v2/crawl/" + JOB_ID): + break + await asyncio.sleep(0.01) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + asyncio.run(journey()) + + (span,) = spans(exporter) + assert span.name == "firecrawl.crawl" + assert span.status.status_code is StatusCode.ERROR + assert span.status.description == "cancelled" + assert attrs(span)["firecrawl.cancelled"] is True + + +@pytest.mark.parametrize("api_status,cancelled", [("cancelled", True), ("scraping", False)]) +def test_cancel_crawl_records_whether_the_job_was_cancelled( + fake: FakeFirecrawl, tracing: Tracing, api_status: str, cancelled: bool +) -> None: + _, exporter, _ = tracing + fake.routes[("DELETE", "/v2/crawl/" + JOB_ID)] = (200, {"success": True, "status": api_status}) + + assert sync_client(fake).cancel_crawl(JOB_ID) is cancelled + + async def journey() -> Any: + return await async_client(fake).cancel_crawl(JOB_ID) + + assert asyncio.run(journey()) is cancelled + + finished = spans(exporter) + assert [span.name for span in finished] == ["firecrawl.cancel_crawl"] * 2 + for span in finished: + assert attrs(span)["firecrawl.cancelled"] is cancelled + assert span.status.status_code is StatusCode.OK + + +# R7 / J1: requested formats are recorded as a list of format names. A JSON +# format's prompt or schema is content and stays off the span. + + +def test_requested_formats_are_recorded_by_name(fake: FakeFirecrawl, tracing: Tracing) -> None: + from firecrawl.v2.types import JsonFormat, ScrapeOptions + + _, exporter, _ = tracing + prompt = "FORMAT-PROMPT-MUST-NOT-BE-EXPORTED" + client = sync_client(fake) + client.scrape("https://example.com", formats=["markdown", JsonFormat(prompt=prompt)]) + client.scrape("https://example.com") + client.start_crawl("https://example.com", scrape_options=ScrapeOptions(formats=["html"])) + + async def journey() -> None: + await async_client(fake).scrape( + "https://example.com", formats=["markdown", {"type": "json", "prompt": prompt}] + ) + + asyncio.run(journey()) + + finished = spans(exporter) + assert [attrs(span).get("firecrawl.formats") for span in finished] == [ + ("markdown", "json"), + None, + ("html",), + ("markdown", "json"), + ] + assert all(prompt not in str(attrs(span)) for span in finished) + + +# R7 / J5: a vendor error records its HTTP status and machine-readable code. + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +def test_vendor_error_records_status_code_and_code( + fake: FakeFirecrawl, tracing: Tracing, use_async: bool +) -> None: + from firecrawl.v2.utils.error_handler import RateLimitError + + _, exporter, _ = tracing + fake.routes[("POST", "/v2/scrape")] = ( + 429, + {"success": False, "error": "rate limited", "code": "RATE_LIMIT_EXCEEDED"}, + ) + + with pytest.raises(RateLimitError): + if use_async: + + async def journey() -> None: + await async_client(fake).scrape("https://example.com") + + asyncio.run(journey()) + else: + sync_client(fake).scrape("https://example.com") + + (span,) = spans(exporter) + assert span.status.status_code is StatusCode.ERROR + assert attrs(span)["firecrawl.error.status_code"] == 429 + assert attrs(span)["firecrawl.error.code"] == "RATE_LIMIT_EXCEEDED" + assert "firecrawl.cancelled" not in attrs(span) diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index 892c85d85..55afbb411 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -2,6 +2,8 @@ from __future__ import annotations +import asyncio +import concurrent.futures import logging from contextvars import ContextVar from typing import Any, Callable, Mapping, Optional, Sequence @@ -22,6 +24,9 @@ _CREDITS_USED = "firecrawl.credits_used" _STATUS = "firecrawl.status" _CANCELLED = "firecrawl.cancelled" +_FORMATS = "firecrawl.formats" +_ERROR_STATUS_CODE = "firecrawl.error.status_code" +_ERROR_CODE = "firecrawl.error.code" _MAX_QUERY_LENGTH = 1024 # One span per call. crawl polls internally, so it must not emit one span per page. @@ -32,6 +37,10 @@ # status comes from the job. _JOB_STATUS_METHODS = {"crawl", "get_crawl_status"} _ERROR_JOB_STATUSES = {"failed", "cancelled"} +# Methods whose arguments carry the requested output formats. +_FORMAT_METHODS = {"scrape", "search", "crawl", "start_crawl"} +_MAX_FORMAT_NAME_LENGTH = 64 +_CANCELLATION_ERRORS = (asyncio.CancelledError, concurrent.futures.CancelledError) # True while a traced Firecrawl call runs in this context. firecrawl-py 4.46.2 # AsyncFirecrawlClient.crawl awaits self.start_crawl, which is wrapped as well; @@ -101,6 +110,47 @@ def _call_job_id(method_name: str, args: Sequence[Any], kwargs: Mapping[str, Any return _job_id(None, kwargs) +def _formats(kwargs: Mapping[str, Any]) -> list[str]: + """Names of the requested formats. A format object's prompt or schema is + content, so only its ``type`` is kept.""" + formats = kwargs.get("formats") + if formats is None: + scrape_options = kwargs.get("scrape_options") + if isinstance(scrape_options, Mapping): + formats = scrape_options.get("formats") + else: + formats = getattr(scrape_options, "formats", None) + if not isinstance(formats, (list, tuple)): + return [] + names = [] + for item in formats: + if isinstance(item, str): + name = item + elif isinstance(item, Mapping): + name = item.get("type") + else: + name = getattr(item, "type", None) + if isinstance(name, str) and name: + names.append(name[:_MAX_FORMAT_NAME_LENGTH]) + return names + + +def _vendor_error_fields(error: BaseException) -> tuple[Optional[int], Optional[str]]: + """HTTP status and machine-readable code of a firecrawl-py FirecrawlError.""" + try: + from firecrawl.v2.utils.error_handler import FirecrawlError + except Exception: + return None, None + if not isinstance(error, FirecrawlError): + return None, None + status_code = getattr(error, "status_code", None) + code = getattr(error, "code", None) + return ( + status_code if isinstance(status_code, int) and not isinstance(status_code, bool) else None, + code if isinstance(code, str) and code else None, + ) + + class _BaseWrapper: """Span bookkeeping shared by the sync and async wrappers. @@ -127,6 +177,10 @@ def _attributes( host = _url_host(url, instance) if host: attributes["server.address"] = host + if self._method_name in _FORMAT_METHODS: + formats = _formats(kwargs) + if formats: + attributes[_FORMATS] = formats if self._method_name in _CRAWL_METHODS: limit = kwargs.get("limit") if isinstance(limit, int): @@ -157,6 +211,8 @@ def _record_result(self, span: Span, result: Any, kwargs: Mapping[str, Any]) -> span.set_attribute(_JOB_ID, _redact(job_id, span)) if self._method_name == "crawl": span.set_attribute(_PAGE_COUNT, _page_count(result)) + if self._method_name == "cancel_crawl" and isinstance(result, bool): + span.set_attribute(_CANCELLED, result) credits = getattr(result, "credits_used", None) if isinstance(credits, int): span.set_attribute(_CREDITS_USED, credits) @@ -180,13 +236,24 @@ def _finish_ok(self, span: Span, result: Any, kwargs: Mapping[str, Any]) -> None finally: _end(span) - def _finish_error(self, span: Span, error: BaseException) -> None: + def _finish_error(self, span: Span, error: BaseException, instance: Any = None) -> None: + cancelled = isinstance(error, _CANCELLATION_ERRORS) + try: + if cancelled: + span.set_attribute(_CANCELLED, True) + status_code, code = _vendor_error_fields(error) + if status_code is not None: + span.set_attribute(_ERROR_STATUS_CODE, status_code) + if code is not None: + span.set_attribute(_ERROR_CODE, _redact(code, instance)) + except Exception: + logger.debug("Could not read %s error", self._span_name, exc_info=True) try: span.record_exception(error) except Exception: logger.debug("Could not record %s exception", self._span_name, exc_info=True) try: - span.set_status(Status(StatusCode.ERROR, _describe(error))) + span.set_status(Status(StatusCode.ERROR, "cancelled" if cancelled else _describe(error))) except Exception: logger.debug("Could not set %s status", self._span_name, exc_info=True) finally: @@ -235,7 +302,7 @@ def __call__( with _current(span): result = wrapped(*args, **kwargs) except BaseException as error: - self._finish_error(span, error) + self._finish_error(span, error, instance) raise finally: _ACTIVE.reset(token) @@ -263,7 +330,7 @@ async def __call__( with _current(span): result = await wrapped(*args, **kwargs) except BaseException as error: - self._finish_error(span, error) + self._finish_error(span, error, instance) raise finally: _ACTIVE.reset(token) From 6031bcdde077afffebb75545a75c2f4aec243561 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 23:15:49 -0700 Subject: [PATCH 12/22] fix(firecrawl): redact the API key from where firecrawl-py stores it _redact read instance.api_key, which firecrawl-py 4.46.2 clients do not have: the key lives on http_client, async_http_client and config. A search query that contained the key was exported verbatim. The job id path also passed the span instead of the client. Redaction now reads every key location (including a key resolved from FIRECRAWL_API_KEY) and the result path passes the client instance. R8 (TH-8325 installed review). approved: Nikhil 2026-10-03 blanket Refs: TH-8325 --- .../frameworks/firecrawl/tests/test_spans.py | 42 +++++++++++++++++++ .../firecrawl/traceai_firecrawl/_wrappers.py | 39 +++++++++++++---- 2 files changed, 73 insertions(+), 8 deletions(-) diff --git a/python/frameworks/firecrawl/tests/test_spans.py b/python/frameworks/firecrawl/tests/test_spans.py index 1e1b2829b..08ad17b13 100644 --- a/python/frameworks/firecrawl/tests/test_spans.py +++ b/python/frameworks/firecrawl/tests/test_spans.py @@ -493,3 +493,45 @@ async def journey() -> None: assert attrs(span)["firecrawl.error.status_code"] == 429 assert attrs(span)["firecrawl.error.code"] == "RATE_LIMIT_EXCEEDED" assert "firecrawl.cancelled" not in attrs(span) + + +# R8 / AC-06: the API key is redacted from every recorded string. firecrawl-py +# keeps it on the client's HTTP clients, not on the client itself. + + +def test_api_key_is_redacted_from_query_and_job_id( + monkeypatch: pytest.MonkeyPatch, fake: FakeFirecrawl, tracing: Tracing +) -> None: + _, exporter, _ = tracing + leaky_id = "job-" + API_KEY + fake.routes[("POST", "/v2/crawl")] = (200, {"success": True, "id": leaky_id, "url": "unused"}) + + client = sync_client(fake) + client.search("find " + API_KEY) + client.start_crawl("https://example.com") + + async def journey() -> None: + aclient = async_client(fake) + await aclient.search("find " + API_KEY) + await aclient.start_crawl("https://example.com") + + asyncio.run(journey()) + + # A key taken from FIRECRAWL_API_KEY is redacted too. + monkeypatch.setenv("FIRECRAWL_API_KEY", API_KEY) + from firecrawl import Firecrawl + + Firecrawl(api_url=fake.origin, max_retries=1).search("find " + API_KEY) + + finished = spans(exporter) + assert [span.name for span in finished] == [ + "firecrawl.search", + "firecrawl.start_crawl", + "firecrawl.search", + "firecrawl.start_crawl", + "firecrawl.search", + ] + for span in finished: + assert all(API_KEY not in str(value) for value in attrs(span).values()), span.name + assert attrs(finished[0])["fi.retrieval.query"] == "find [redacted]" + assert attrs(finished[1])["firecrawl.job_id"] == "job-[redacted]" diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index 55afbb411..b35fd9187 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -49,9 +49,28 @@ _ACTIVE: ContextVar[bool] = ContextVar("traceai_firecrawl_active", default=False) +def _api_keys(instance: Any) -> list[str]: + """API keys the client holds. firecrawl-py 4.46.2 keeps the key on + ``http_client`` (sync and async), ``async_http_client`` (async) and + ``config`` (sync), not on the client; it also resolves FIRECRAWL_API_KEY + into those objects.""" + keys: list[str] = [] + for holder in ( + instance, + getattr(instance, "http_client", None), + getattr(instance, "async_http_client", None), + getattr(instance, "config", None), + ): + if holder is None: + continue + api_key = getattr(holder, "api_key", None) + if isinstance(api_key, str) and api_key and api_key not in keys: + keys.append(api_key) + return keys + + def _redact(value: str, instance: Any) -> str: - api_key = getattr(instance, "api_key", None) - if isinstance(api_key, str) and api_key: + for api_key in _api_keys(instance): value = value.replace(api_key, "[redacted]") return value[:_MAX_QUERY_LENGTH] @@ -202,13 +221,15 @@ def _start(self, instance: Any, args: Sequence[Any], kwargs: Mapping[str, Any]) logger.debug("Could not start %s span", self._span_name, exc_info=True) return None - def _record_result(self, span: Span, result: Any, kwargs: Mapping[str, Any]) -> None: + def _record_result( + self, span: Span, instance: Any, result: Any, kwargs: Mapping[str, Any] + ) -> None: if self._method_name == "search": span.set_attribute(_RETRIEVAL_DOCUMENT_COUNT, _document_count(result)) if self._method_name in _CRAWL_METHODS: job_id = _job_id(result, kwargs) if job_id: - span.set_attribute(_JOB_ID, _redact(job_id, span)) + span.set_attribute(_JOB_ID, _redact(job_id, instance)) if self._method_name == "crawl": span.set_attribute(_PAGE_COUNT, _page_count(result)) if self._method_name == "cancel_crawl" and isinstance(result, bool): @@ -228,9 +249,11 @@ def _record_result(self, span: Span, result: Any, kwargs: Mapping[str, Any]) -> else: span.set_status(Status(StatusCode.OK)) - def _finish_ok(self, span: Span, result: Any, kwargs: Mapping[str, Any]) -> None: + def _finish_ok( + self, span: Span, instance: Any, result: Any, kwargs: Mapping[str, Any] + ) -> None: try: - self._record_result(span, result, kwargs) + self._record_result(span, instance, result, kwargs) except Exception: logger.debug("Could not read %s result", self._span_name, exc_info=True) finally: @@ -306,7 +329,7 @@ def __call__( raise finally: _ACTIVE.reset(token) - self._finish_ok(span, result, kwargs) + self._finish_ok(span, instance, result, kwargs) return result @@ -334,5 +357,5 @@ async def __call__( raise finally: _ACTIVE.reset(token) - self._finish_ok(span, result, kwargs) + self._finish_ok(span, instance, result, kwargs) return result From d46646272d60b913327fa63e363f96b4978cc3f9 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 23:16:59 -0700 Subject: [PATCH 13/22] build(firecrawl): support firecrawl-py >=4.46.2,<5 and declare tested Pythons The exact firecrawl-py==4.46.2 runtime pin made BaseInstrumentor.instrument() log an error and wrap nothing on any other release, and forced that exact version on users. pyproject and _instruments now declare >=4.46.2,<5; the tests stay pinned to 4.46.2. fi-instrumentation-otel's floor moves to >=1.1.0, the release the tests import, and the Python range and classifiers list only 3.10, 3.11 and 3.13, the versions the suite runs on. Tests pin the range, that instrument() wraps on an in-range release and skips an out-of-range one, and the declared metadata. R11, N1 (TH-8325 installed review). approved: Nikhil 2026-10-03 blanket Refs: TH-8325 --- python/frameworks/firecrawl/pyproject.toml | 8 +- .../firecrawl/tests/test_packaging.py | 90 +++++++++++++++++++ .../firecrawl/traceai_firecrawl/package.py | 5 +- 3 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 python/frameworks/firecrawl/tests/test_packaging.py diff --git a/python/frameworks/firecrawl/pyproject.toml b/python/frameworks/firecrawl/pyproject.toml index 6d120ccdb..496435756 100644 --- a/python/frameworks/firecrawl/pyproject.toml +++ b/python/frameworks/firecrawl/pyproject.toml @@ -12,18 +12,16 @@ classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Topic :: Software Development :: Libraries :: Python Modules", ] [tool.poetry.dependencies] -python = ">=3.9,<3.14" -fi-instrumentation-otel = ">=0.1.11" -firecrawl-py = "==4.46.2" +python = ">=3.10,<3.14" +fi-instrumentation-otel = ">=1.1.0" +firecrawl-py = ">=4.46.2,<5" wrapt = ">=1.0.0" [tool.poetry.group.dev.dependencies] diff --git a/python/frameworks/firecrawl/tests/test_packaging.py b/python/frameworks/firecrawl/tests/test_packaging.py new file mode 100644 index 000000000..e56a50432 --- /dev/null +++ b/python/frameworks/firecrawl/tests/test_packaging.py @@ -0,0 +1,90 @@ +"""Packaging contract: the supported firecrawl-py range and the declared metadata. + +BaseInstrumentor.instrument() logs an error and instruments nothing when the +installed firecrawl-py falls outside instrumentation_dependencies(), so an +exact pin would silently turn tracing off for every other release. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Iterator + +import pytest + +pytest.importorskip("firecrawl", reason="firecrawl-py must be installed to test its instrumentor") + +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from packaging.requirements import Requirement # noqa: E402 +from traceai_firecrawl import FirecrawlInstrumentor # noqa: E402 +from traceai_firecrawl.package import _instruments # noqa: E402 + +PYPROJECT = Path(__file__).resolve().parents[1] / "pyproject.toml" + + +def _requirement() -> Requirement: + (requirement,) = [Requirement(spec) for spec in _instruments] + assert requirement.name == "firecrawl-py" + return requirement + + +@pytest.mark.parametrize( + "candidate,supported", + [("4.46.2", True), ("4.47.0", True), ("4.99.9", True), ("4.46.1", False), ("5.0.0", False)], +) +def test_supported_firecrawl_range(candidate: str, supported: bool) -> None: + assert _requirement().specifier.contains(candidate) is supported + + +@pytest.fixture() +def installed_firecrawl(monkeypatch: pytest.MonkeyPatch) -> Iterator[list]: + """Pretend a different firecrawl-py release is installed.""" + import opentelemetry.instrumentation.dependencies as dependencies + + reported: list = [] + real_version = dependencies.version + + def fake_version(name: str) -> str: + if name == "firecrawl-py" and reported: + return reported[0] + return real_version(name) + + monkeypatch.setattr(dependencies, "version", fake_version) + yield reported + + +@pytest.mark.parametrize("candidate,instrumented", [("4.47.0", True), ("5.0.0", False)]) +def test_instrument_follows_the_supported_range( + installed_firecrawl: list, candidate: str, instrumented: bool +) -> None: + from firecrawl.v2.client import FirecrawlClient + + original = FirecrawlClient.__dict__["scrape"] + installed_firecrawl.append(candidate) + instrumentor = FirecrawlInstrumentor() + instrumentor.instrument(tracer_provider=TracerProvider()) + try: + wrapped = FirecrawlClient.__dict__["scrape"] is not original + finally: + instrumentor.uninstrument() + assert wrapped is instrumented + assert FirecrawlClient.__dict__["scrape"] is original + + +def _poetry_value(name: str) -> str: + match = re.search(r'^{0}\s*=\s*"([^"]+)"'.format(re.escape(name)), PYPROJECT.read_text(), re.M) + assert match, name + return match.group(1) + + +def test_pyproject_matches_the_supported_range() -> None: + declared = Requirement("firecrawl-py" + _poetry_value("firecrawl-py")) + assert declared.specifier == _requirement().specifier + assert _poetry_value("fi-instrumentation-otel") == ">=1.1.0" + + +def test_classifiers_list_only_tested_pythons() -> None: + classifiers = re.findall(r'"Programming Language :: Python :: (3\.\d+)"', PYPROJECT.read_text()) + assert classifiers == ["3.10", "3.11", "3.13"] + assert _poetry_value("python") == ">=3.10,<3.14" diff --git a/python/frameworks/firecrawl/traceai_firecrawl/package.py b/python/frameworks/firecrawl/traceai_firecrawl/package.py index c2274f157..5af1c676a 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/package.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/package.py @@ -1 +1,4 @@ -_instruments = ("firecrawl-py == 4.46.2",) +# Supported firecrawl-py range. BaseInstrumentor.instrument() refuses to run +# (and logs an error) outside it, so this is a range, not the CI pin: the tests +# run on firecrawl-py==4.46.2, the first release this package was verified on. +_instruments = ("firecrawl-py >= 4.46.2, < 5",) From febe6e58801562e2a28ecab30b3b253d41558b43 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 23:18:36 -0700 Subject: [PATCH 14/22] test(firecrawl): cover map, async parity, uninstrument, facade binding and exit flush Adds the tests the installed review listed as missing, all against the real firecrawl-py 4.46.2 clients and a loopback fake: - map emits one TOOL span with the host only, sync and async; - AC-08: the async twins of all seven methods export the same span names, attributes and status as the sync client; - uninstrument() restores every wrapped method on both clients and new clients emit no spans afterwards; - R4: the Firecrawl facade binds methods at construction, so a client built before instrument() is untraced and one built while instrumented keeps tracing after uninstrument() (pins the documented behaviour); - AC-09: a script that exits without force_flush() still exports its scrape span through register()'s batch processor; an os._exit control exports nothing. The uninstrument and facade tests fail when _uninstrument skips the restore (mutation run locally, then reverted). approved: Nikhil 2026-10-03 blanket Refs: TH-8325 --- .../firecrawl/tests/test_contract_harness.py | 54 ++++++++ .../frameworks/firecrawl/tests/test_spans.py | 121 ++++++++++++++++++ 2 files changed, 175 insertions(+) diff --git a/python/frameworks/firecrawl/tests/test_contract_harness.py b/python/frameworks/firecrawl/tests/test_contract_harness.py index 8db973355..c8b793d28 100644 --- a/python/frameworks/firecrawl/tests/test_contract_harness.py +++ b/python/frameworks/firecrawl/tests/test_contract_harness.py @@ -244,3 +244,57 @@ def test_no_key_path_or_content_is_exported(fake_firecrawl, monkeypatch): wire = json.dumps(spans) + json.dumps(exports) for secret in (FIRECRAWL_KEY, PAGE_BODY, RESULT_TITLE, SECRET_PATH): assert secret not in wire, secret + + +_EXIT_SCRIPT = """ +import os +import sys + +from firecrawl import Firecrawl +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_firecrawl import FirecrawlInstrumentor + +provider = register(project_type=ProjectType.OBSERVE, project_name=os.environ["PROJECT"], verbose=False) +FirecrawlInstrumentor().instrument(tracer_provider=provider) +Firecrawl(api_key=os.environ["FIRECRAWL_KEY"], api_url=os.environ["FIRECRAWL_BASE_URL"], max_retries=1).scrape( + "https://example.com" +) +# No force_flush(): the batch processor must export at interpreter exit. +if sys.argv[1] == "hard-exit": + os._exit(0) +""" + + +@pytest.mark.parametrize("exit_mode,exported", [("normal", True), ("hard-exit", False)]) +def test_scrape_span_is_exported_at_process_exit(fake_firecrawl, exit_mode, exported): + """AC-09: a script that exits normally exports its span without an explicit + flush (register() uses a batch processor). The hard-exit control skips the + exit hooks and must export nothing, which shows the receiver is not fed any + other way.""" + import os + + from harness import run + + with Receiver() as receiver: + env = dict(os.environ) + here = Path(__file__).resolve() + # The package and the in-repo fi_instrumentation, whatever the cwd. + env["PYTHONPATH"] = os.pathsep.join( + [str(here.parents[1]), str(here.parents[3])] + [p for p in [env.get("PYTHONPATH")] if p] + ) + env.update( + FI_BASE_URL=receiver.origin, + FI_API_KEY=FI_API_KEY, + FI_SECRET_KEY=FI_SECRET_KEY, + PROJECT=PROJECT, + FIRECRAWL_KEY=FIRECRAWL_KEY, + FIRECRAWL_BASE_URL=fake_firecrawl.origin, + ) + result = run([sys.executable, "-c", _EXIT_SCRIPT, exit_mode], env=env, stdin=None, timeout=60) + spans = receiver.spans() + + assert not result.timed_out, result.stderr.decode(errors="replace") + assert result.returncode == 0, result.stderr.decode(errors="replace") + assert fake_firecrawl.calls == [("POST", "/v2/scrape")] + assert [span["name"] for span in spans] == (["firecrawl.scrape"] if exported else []) diff --git a/python/frameworks/firecrawl/tests/test_spans.py b/python/frameworks/firecrawl/tests/test_spans.py index 08ad17b13..c763f0400 100644 --- a/python/frameworks/firecrawl/tests/test_spans.py +++ b/python/frameworks/firecrawl/tests/test_spans.py @@ -535,3 +535,124 @@ async def journey() -> None: assert all(API_KEY not in str(value) for value in attrs(span).values()), span.name assert attrs(finished[0])["fi.retrieval.query"] == "find [redacted]" assert attrs(finished[1])["firecrawl.job_id"] == "job-[redacted]" + + +# Coverage the installed review listed as missing. + +OPERATIONS = ( + "scrape", + "search", + "map", + "crawl", + "start_crawl", + "get_crawl_status", + "cancel_crawl", +) + + +def test_map_emits_one_tool_span_without_links(fake: FakeFirecrawl, tracing: Tracing) -> None: + from _firecrawl_fake import RESULT_TITLE + + _, exporter, _ = tracing + mapped = sync_client(fake).map("https://example.com/private/start", limit=2) + + async def journey() -> Any: + return await async_client(fake).map("https://example.com/private/start", limit=2) + + async_mapped = asyncio.run(journey()) + + assert len(mapped.links) == 2 and len(async_mapped.links) == 2 + finished = spans(exporter) + assert [span.name for span in finished] == ["firecrawl.map"] * 2 + for span in finished: + assert span.status.status_code is StatusCode.OK + assert attrs(span) == {"fi.span.kind": "TOOL", "server.address": "example.com"} + assert "/private" not in str(attrs(span)) and RESULT_TITLE not in str(attrs(span)) + + +def _sync_journey(fake: FakeFirecrawl) -> None: + client = sync_client(fake) + client.scrape("https://example.com/a", formats=["markdown"]) + client.search("open telemetry", limit=2) + client.map("https://example.com") + client.crawl("https://example.com", limit=3, poll_interval=0) + client.start_crawl("https://example.com", limit=3) + client.get_crawl_status(JOB_ID) + client.cancel_crawl(JOB_ID) + + +async def _async_journey(fake: FakeFirecrawl) -> None: + client = async_client(fake) + await client.scrape("https://example.com/a", formats=["markdown"]) + await client.search("open telemetry", limit=2) + await client.map("https://example.com") + await client.crawl(url="https://example.com", limit=3, poll_interval=0) + await client.start_crawl("https://example.com", limit=3) + await client.get_crawl_status(JOB_ID) + await client.cancel_crawl(JOB_ID) + + +def test_async_twins_record_the_same_spans_as_sync(fake: FakeFirecrawl, tracing: Tracing) -> None: + """AC-08 / J6: AsyncFirecrawl emits the same span names, attributes and status.""" + _, exporter, _ = tracing + + _sync_journey(fake) + sync_spans = spans(exporter) + exporter.clear() + asyncio.run(_async_journey(fake)) + async_spans = spans(exporter) + + def summary(finished: List[ReadableSpan]) -> List[Tuple[str, Dict[str, Any], StatusCode]]: + return [(span.name, attrs(span), span.status.status_code) for span in finished] + + assert [span.name for span in sync_spans] == ["firecrawl." + name for name in OPERATIONS] + assert summary(async_spans) == summary(sync_spans) + + +def test_uninstrument_restores_methods_and_stops_new_spans(fake: FakeFirecrawl) -> None: + from firecrawl.v2.client import FirecrawlClient + from firecrawl.v2.client_async import AsyncFirecrawlClient + + classes = (FirecrawlClient, AsyncFirecrawlClient) + originals = {(cls, name): cls.__dict__[name] for cls in classes for name in OPERATIONS} + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = FirecrawlInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + assert all(cls.__dict__[name] is not original for (cls, name), original in originals.items()) + finally: + instrumentor.uninstrument() + + assert all(cls.__dict__[name] is original for (cls, name), original in originals.items()) + _sync_journey(fake) + asyncio.run(_async_journey(fake)) + assert spans(exporter) == [] + + +def test_facade_methods_bind_at_construction(fake: FakeFirecrawl) -> None: + """R4, documented in the README: Firecrawl/AsyncFirecrawl copy the v2 client's + bound methods in __init__. Build the client after instrument(); a client built + before is not traced, and one built while instrumented keeps tracing after + uninstrument().""" + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = FirecrawlInstrumentor() + + built_before = sync_client(fake) + instrumentor.instrument(tracer_provider=provider) + try: + built_during = sync_client(fake) + built_before.scrape("https://before.example.com") + built_during.scrape("https://during.example.com") + finally: + instrumentor.uninstrument() + built_during.scrape("https://during-after-uninstrument.example.com") + sync_client(fake).scrape("https://after.example.com") + + assert [attrs(span)["server.address"] for span in spans(exporter)] == [ + "during.example.com", + "during-after-uninstrument.example.com", + ] From 1fff1076c1cfbf018086c66d6b71883de60105ec Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 23:19:58 -0700 Subject: [PATCH 15/22] docs(firecrawl): document attributes, construction order, watchers and limits - R10: drop "content capture is off by default"; there is no such switch. State what is never recorded and that error messages are kept as raised. - R9: firecrawl-py defaults a crawl job's credits_used to 0 when the API omits it, so a recorded 0 is ambiguous. Documented, behaviour unchanged. - R4: Firecrawl/AsyncFirecrawl bind methods at construction; build them after instrument(), and a client built while instrumented keeps tracing after uninstrument(). - N2: the sync Watcher polls from its own thread, so each poll is a root get_crawl_status span; AsyncWatcher polls are parented. - Lists every attribute with the methods that set it, the job-status and cancellation rules (R3, R7), the supported firecrawl-py range and the tested Pythons (R11, N1). approved: Nikhil 2026-10-03 blanket Refs: TH-8325 --- python/frameworks/firecrawl/README.md | 55 ++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/python/frameworks/firecrawl/README.md b/python/frameworks/firecrawl/README.md index ab48ee713..d51fec727 100644 --- a/python/frameworks/firecrawl/README.md +++ b/python/frameworks/firecrawl/README.md @@ -4,15 +4,13 @@ OpenTelemetry instrumentation for the Firecrawl v2 client. ## v1 and v2 are different clients -`firecrawl-py` 4.46.2 ships two clients. The v2 client is `from firecrawl import Firecrawl`, and it delegates `scrape`, `search`, `map`, and `crawl` to `firecrawl.v2.client.FirecrawlClient`. The v1 client is `FirecrawlApp`, and it is feature-frozen. +`firecrawl-py` 4.x ships two clients. The v2 client is `from firecrawl import Firecrawl` (and `AsyncFirecrawl`), and it delegates `scrape`, `search`, `map`, and the crawl methods to `firecrawl.v2.client.FirecrawlClient` (and `firecrawl.v2.client_async.AsyncFirecrawlClient`). The v1 client is `FirecrawlApp`, and it is feature-frozen. This package wraps the v2 client only. It does not wrap `FirecrawlApp`, and it does not import or disable OpenLIT. OpenLIT's `FireCrawlInstrumentor` wraps `FirecrawlApp.scrape_url` on the v1 client. A user who constructs both clients and enables both instrumentors gets two spans. Pick one. -## What it records - -Each span records `fi.span.kind=TOOL`. `search` also records the query, capped at 1024 characters, and the result count. `scrape` and `map` record the URL host only, never the path or the body. `crawl` emits one span for the whole call, not one per page, and records the job id, the limit, and the page count. +## Supported versions -Content capture is off by default. The API key is never copied into an attribute. `credits_used` is recorded only when the response carries it, and it is not converted to dollars. +`firecrawl-py >=4.46.2,<5` on Python 3.10, 3.11 and 3.13. The tests run on `firecrawl-py==4.46.2`. Outside that range `instrument()` logs an error and wraps nothing. ## Usage @@ -24,6 +22,53 @@ from traceai_firecrawl import FirecrawlInstrumentor tracer_provider = register(project_name="firecrawl") FirecrawlInstrumentor().instrument(tracer_provider=tracer_provider) +# Construct the client after instrument(); see "Client construction order". client = Firecrawl(api_key="your-firecrawl-api-key") client.scrape("https://example.com") ``` + +## What it records + +Each call to `scrape`, `search`, `map`, `crawl`, `start_crawl`, `get_crawl_status` or `cancel_crawl`, sync or async, emits one span named `firecrawl.` with `fi.span.kind=TOOL`. A blocking `crawl()` is one span for the whole call: not one per page and not one per status poll. When the async `crawl()` calls `start_crawl()` internally, that inner call adds no span. The span is current while the SDK runs, so HTTP client spans nest under it. + +| Attribute | Methods | Value | +|---|---|---| +| `server.address` | `scrape`, `map`, `crawl`, `start_crawl` | Host of the target URL. Never the path or query. | +| `fi.retrieval.query` | `search` | The query, with the API key redacted, capped at 1024 characters. | +| `fi.retrieval.document_count` | `search` | Number of results. | +| `firecrawl.formats` | `scrape`, `search`, `crawl`, `start_crawl` | Names of the requested formats, from `formats=` or `scrape_options.formats`, when the caller passes them. A JSON format's prompt and schema are not recorded. | +| `firecrawl.job_id` | `crawl`, `start_crawl`, `get_crawl_status`, `cancel_crawl` | The crawl job id, positional or keyword. The start, status and cancel spans of one job share it. | +| `firecrawl.limit` | `crawl`, `start_crawl` | The `limit` argument, when passed. | +| `firecrawl.page_count` | `crawl` | Pages the job completed. | +| `firecrawl.status` | `crawl`, `get_crawl_status` | Job status from the response (`scraping`, `completed`, `failed`, `cancelled`). | +| `firecrawl.cancelled` | see "Status" | `true` when the job or the call was cancelled. | +| `firecrawl.credits_used` | any method whose result carries it | Credits the response reports. Not converted to dollars. | +| `firecrawl.error.status_code`, `firecrawl.error.code` | any, on a Firecrawl API error | HTTP status and the vendor's machine-readable error code, when present. | + +`firecrawl.credits_used` caveat: for crawl jobs, firecrawl-py fills `credits_used` with `0` when the API response omits `creditsUsed`, so a recorded `0` on `crawl` or `get_crawl_status` can mean either "no credits" or "not reported". + +### Status + +- The SDK returns a failed or cancelled crawl job without raising. A job with status `failed` sets the span to ERROR `failed`. A job with status `cancelled` sets ERROR `cancelled` and `firecrawl.cancelled=true`. +- `cancel_crawl` sets `firecrawl.cancelled` to the API's answer; the span itself is OK. +- Cancelling an asyncio task that is awaiting a Firecrawl call ends the span ERROR `cancelled` with `firecrawl.cancelled=true`. +- Any exception the SDK raises is recorded on the span, sets ERROR with the exception type and message, and is re-raised unchanged. + +## What it never records + +This package has no content-capture setting. It never records page content (markdown, HTML, screenshots, links, titles, descriptions), URL paths or query strings, JSON-format prompts or schemas, or the Firecrawl API key. The key is redacted from every attribute the package sets, including a key taken from `FIRECRAWL_API_KEY`. The search query is recorded by design. On an error, the span status and exception event carry the SDK's exception message as raised. + +Instrumentation errors never reach your code: if reading an argument or a result fails, the call still returns the SDK's result or raises the SDK's exception, and the span still ends. + +## Client construction order + +`Firecrawl` and `AsyncFirecrawl` copy the v2 client's methods when they are constructed. Call `instrument()` before you construct them: + +- A client constructed before `instrument()` is not traced. +- A client constructed while instrumented keeps tracing after `uninstrument()`. Construct a new client to stop. + +`FirecrawlClient` and `AsyncFirecrawlClient` used directly look the methods up on every call, so they follow `instrument()` and `uninstrument()` immediately. + +## Watchers + +`Firecrawl.watcher()` returns a `Watcher` that polls `get_crawl_status` from its own thread. Each poll is a separate root `firecrawl.get_crawl_status` span with no parent, and a poll error the watcher swallows still shows as an ERROR span. `AsyncFirecrawl.watcher()` polls in the task that iterates it, so its poll spans are children of the span current there. From 8c1e79ebd4bbca6da404ef721e90d8673bf674a0 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 23:20:49 -0700 Subject: [PATCH 16/22] style(firecrawl): wrap two long lines in the wrappers No behaviour change. approved: Nikhil 2026-10-03 blanket Refs: TH-8325 --- python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index b35fd9187..9f4f688b1 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -209,7 +209,9 @@ def _attributes( attributes[_JOB_ID] = _redact(job_id, instance) return attributes - def _start(self, instance: Any, args: Sequence[Any], kwargs: Mapping[str, Any]) -> Optional[Span]: + def _start( + self, instance: Any, args: Sequence[Any], kwargs: Mapping[str, Any] + ) -> Optional[Span]: try: attributes = self._attributes(instance, args, kwargs) except Exception: @@ -276,7 +278,8 @@ def _finish_error(self, span: Span, error: BaseException, instance: Any = None) except Exception: logger.debug("Could not record %s exception", self._span_name, exc_info=True) try: - span.set_status(Status(StatusCode.ERROR, "cancelled" if cancelled else _describe(error))) + description = "cancelled" if cancelled else _describe(error) + span.set_status(Status(StatusCode.ERROR, description)) except Exception: logger.debug("Could not set %s status", self._span_name, exc_info=True) finally: From 8a29398331407d1ec9a7cd292fb6e5c3181e641c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 23:33:31 -0700 Subject: [PATCH 17/22] fix(firecrawl): only the blocking crawl() span takes ERROR from the job's status A get_crawl_status poll that got an answer succeeded, whatever the job's state; marking it ERROR made healthy status polls (and every watcher poll of a failed job) look like failures. It now records firecrawl.status and stays OK; crawl() keeps ERROR for failed/cancelled jobs and firecrawl.cancelled. Coordinator decision on the open question from the fix round. Test first: the parametrised job-status test failed 4 cases (failed and cancelled, sync and async) on 8c1e79e. approved: Nikhil 2026-10-03 blanket Refs: TH-8325 --- python/frameworks/firecrawl/README.md | 2 +- .../frameworks/firecrawl/tests/test_spans.py | 31 ++++++++++++------- .../firecrawl/traceai_firecrawl/_wrappers.py | 9 ++++-- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/python/frameworks/firecrawl/README.md b/python/frameworks/firecrawl/README.md index d51fec727..b3823353e 100644 --- a/python/frameworks/firecrawl/README.md +++ b/python/frameworks/firecrawl/README.md @@ -49,7 +49,7 @@ Each call to `scrape`, `search`, `map`, `crawl`, `start_crawl`, `get_crawl_statu ### Status -- The SDK returns a failed or cancelled crawl job without raising. A job with status `failed` sets the span to ERROR `failed`. A job with status `cancelled` sets ERROR `cancelled` and `firecrawl.cancelled=true`. +- The SDK returns a failed or cancelled crawl job without raising. On a blocking `crawl()`, a job with status `failed` sets the span to ERROR `failed`, and a job with status `cancelled` sets ERROR `cancelled` and `firecrawl.cancelled=true`. A `get_crawl_status` poll that gets an answer stays OK and records the job's state in `firecrawl.status`. - `cancel_crawl` sets `firecrawl.cancelled` to the API's answer; the span itself is OK. - Cancelling an asyncio task that is awaiting a Firecrawl call ends the span ERROR `cancelled` with `firecrawl.cancelled=true`. - Any exception the SDK raises is recorded on the span, sets ERROR with the exception type and message, and is re-raised unchanged. diff --git a/python/frameworks/firecrawl/tests/test_spans.py b/python/frameworks/firecrawl/tests/test_spans.py index c763f0400..6a8335585 100644 --- a/python/frameworks/firecrawl/tests/test_spans.py +++ b/python/frameworks/firecrawl/tests/test_spans.py @@ -170,9 +170,10 @@ async def journey() -> None: assert attrs(span)["firecrawl.job_id"] == "missing-job" -# R3 / J3: the SDK returns a failed or cancelled job without raising, so the span -# takes its status from the job. failed -> ERROR; cancelled -> ERROR "cancelled" -# plus firecrawl.cancelled=true; completed -> OK. +# R3 / J3: the SDK returns a failed or cancelled job without raising, so the +# blocking crawl() span takes its status from the job. failed -> ERROR; +# cancelled -> ERROR "cancelled" plus firecrawl.cancelled=true; completed -> OK. +# get_crawl_status only records the job's state. EXPECTED_JOB_STATUS = { "completed": (StatusCode.OK, None), @@ -210,16 +211,22 @@ def test_job_status_sets_attribute_and_span_status( finished = spans(exporter) assert [span.name for span in finished] == ["firecrawl.crawl", "firecrawl.get_crawl_status"] + crawl_span, status_span = finished code, description = EXPECTED_JOB_STATUS[job_status] - for span in finished: - assert attrs(span)["firecrawl.status"] == job_status - assert span.status.status_code is code - if description is not None: - assert span.status.description == description - if job_status == "cancelled": - assert attrs(span)["firecrawl.cancelled"] is True - else: - assert "firecrawl.cancelled" not in attrs(span) + # The blocking crawl() call's outcome is the job's outcome. + assert attrs(crawl_span)["firecrawl.status"] == job_status + assert crawl_span.status.status_code is code + if description is not None: + assert crawl_span.status.description == description + if job_status == "cancelled": + assert attrs(crawl_span)["firecrawl.cancelled"] is True + else: + assert "firecrawl.cancelled" not in attrs(crawl_span) + # A status poll that got an answer succeeded, whatever the job's state: it + # records the state, and only the poll's own errors make it ERROR. + assert attrs(status_span)["firecrawl.status"] == job_status + assert status_span.status.status_code is StatusCode.OK + assert "firecrawl.cancelled" not in attrs(status_span) # R5: instrumentation errors never reach the caller, and the span always ends. diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index 9f4f688b1..1c30ec2fe 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -36,6 +36,8 @@ # without raising (methods/crawl.py wait_for_crawl_completion), so the span # status comes from the job. _JOB_STATUS_METHODS = {"crawl", "get_crawl_status"} +# Of those, the methods whose span status follows the job (crawl waits for it). +_JOB_OUTCOME_METHODS = {"crawl"} _ERROR_JOB_STATUSES = {"failed", "cancelled"} # Methods whose arguments carry the requested output formats. _FORMAT_METHODS = {"scrape", "search", "crawl", "start_crawl"} @@ -244,9 +246,12 @@ def _record_result( ) if isinstance(job_status, str) and job_status: span.set_attribute(_STATUS, job_status) - if job_status == "cancelled": + # Only the blocking crawl() call's outcome is the job's outcome. A + # get_crawl_status poll that got an answer succeeded; it records the state. + job_failed = self._method_name in _JOB_OUTCOME_METHODS and job_status in _ERROR_JOB_STATUSES + if job_failed and job_status == "cancelled": span.set_attribute(_CANCELLED, True) - if job_status in _ERROR_JOB_STATUSES: + if job_failed: span.set_status(Status(StatusCode.ERROR, job_status)) else: span.set_status(Status(StatusCode.OK)) From e983153cf327a24f1e57e5559bb52de25aee8e83 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 6 Oct 2026 15:37:12 -0700 Subject: [PATCH 18/22] fix(firecrawl): close privacy, grouped-search and setup review gaps Keep error telemetry to safe metadata and preserve original SDK exceptions. Use FITracer and explicit query privacy controls; canonicalize format labels and mirror SDK options precedence; count all supported search groups. Correct Observe registration and published-core exit provenance. Add real SDK/exporter regression coverage and admitted Python 3.12 metadata. Verified test-first bounded correction; original review and failed probes are retained in ticket evidence. No shared packages or vendor surfaces changed. --- python/frameworks/firecrawl/README.md | 21 +- python/frameworks/firecrawl/pyproject.toml | 1 + .../firecrawl/tests/_firecrawl_fake.py | 4 +- .../firecrawl/tests/test_contract_harness.py | 14 +- .../firecrawl/tests/test_packaging.py | 12 +- .../tests/test_review_regressions.py | 517 ++++++++++++++++++ .../firecrawl/traceai_firecrawl/__init__.py | 13 +- .../firecrawl/traceai_firecrawl/_wrappers.py | 133 +++-- 8 files changed, 668 insertions(+), 47 deletions(-) create mode 100644 python/frameworks/firecrawl/tests/test_review_regressions.py diff --git a/python/frameworks/firecrawl/README.md b/python/frameworks/firecrawl/README.md index b3823353e..da5682b9b 100644 --- a/python/frameworks/firecrawl/README.md +++ b/python/frameworks/firecrawl/README.md @@ -10,16 +10,17 @@ This package wraps the v2 client only. It does not wrap `FirecrawlApp`, and it d ## Supported versions -`firecrawl-py >=4.46.2,<5` on Python 3.10, 3.11 and 3.13. The tests run on `firecrawl-py==4.46.2`. Outside that range `instrument()` logs an error and wraps nothing. +`firecrawl-py >=4.46.2,<5` on Python 3.10, 3.11, 3.12 and 3.13. The tests run on `firecrawl-py==4.46.2`. Outside that range `instrument()` logs an error and wraps nothing. ## Usage ```python from firecrawl import Firecrawl from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType from traceai_firecrawl import FirecrawlInstrumentor -tracer_provider = register(project_name="firecrawl") +tracer_provider = register(project_name="firecrawl", project_type=ProjectType.OBSERVE) FirecrawlInstrumentor().instrument(tracer_provider=tracer_provider) # Construct the client after instrument(); see "Client construction order". @@ -34,9 +35,9 @@ Each call to `scrape`, `search`, `map`, `crawl`, `start_crawl`, `get_crawl_statu | Attribute | Methods | Value | |---|---|---| | `server.address` | `scrape`, `map`, `crawl`, `start_crawl` | Host of the target URL. Never the path or query. | -| `fi.retrieval.query` | `search` | The query, with the API key redacted, capped at 1024 characters. | -| `fi.retrieval.document_count` | `search` | Number of results. | -| `firecrawl.formats` | `scrape`, `search`, `crawl`, `start_crawl` | Names of the requested formats, from `formats=` or `scrape_options.formats`, when the caller passes them. A JSON format's prompt and schema are not recorded. | +| `fi.retrieval.query` | `search` | The query, with the API key redacted, capped at 1024 characters. Omitted with `hide_inputs` or `hide_input_text`; shared PII redaction applies when enabled. | +| `fi.retrieval.document_count` | `search` | Total returned records across SDK `web`, `news`, `images` and `tools` groups. Known empty groups count as zero; unknown or malformed shapes omit the attribute. | +| `firecrawl.formats` | `scrape`, `search`, `crawl`, `start_crawl` | Bounded, deduplicated canonical SDK format names from `formats=` or `scrape_options.formats`. Unknown names, prompts and schemas are omitted. Snake-case aliases use the SDK's API spelling. | | `firecrawl.job_id` | `crawl`, `start_crawl`, `get_crawl_status`, `cancel_crawl` | The crawl job id, positional or keyword. The start, status and cancel spans of one job share it. | | `firecrawl.limit` | `crawl`, `start_crawl` | The `limit` argument, when passed. | | `firecrawl.page_count` | `crawl` | Pages the job completed. | @@ -47,19 +48,25 @@ Each call to `scrape`, `search`, `map`, `crawl`, `start_crawl`, `get_crawl_statu `firecrawl.credits_used` caveat: for crawl jobs, firecrawl-py fills `credits_used` with `0` when the API response omits `creditsUsed`, so a recorded `0` on `crawl` or `get_crawl_status` can mean either "no credits" or "not reported". +Format metadata follows SDK 4.46.2's payload: `scrape_options` takes precedence over convenience `formats`. A `ScrapeFormats` container contributes its explicit list and serialized enabled flags, including its default `markdown=true`. The SDK does not serialize its `images` or `json` boolean flags, so those flags add no metadata; explicit format entries still do. + +Search counts include `tools` because SDK 4.46.2 returns a list of `DiscoveredTool` discovery records, alongside web/news/image records. The count includes those records, not pages fetched by a tool. The prior generic `data`/`results`/`web` list fallback remains supported; no result contents are exported. + ### Status - The SDK returns a failed or cancelled crawl job without raising. On a blocking `crawl()`, a job with status `failed` sets the span to ERROR `failed`, and a job with status `cancelled` sets ERROR `cancelled` and `firecrawl.cancelled=true`. A `get_crawl_status` poll that gets an answer stays OK and records the job's state in `firecrawl.status`. - `cancel_crawl` sets `firecrawl.cancelled` to the API's answer; the span itself is OK. - Cancelling an asyncio task that is awaiting a Firecrawl call ends the span ERROR `cancelled` with `firecrawl.cancelled=true`. -- Any exception the SDK raises is recorded on the span, sets ERROR with the exception type and message, and is re-raised unchanged. +- Any exception the SDK raises adds one safe exception event and sets ERROR with a recognized exception type (otherwise `Exception`). Exception free text and serialized tracebacks are omitted; the original exception is re-raised unchanged. ## What it never records -This package has no content-capture setting. It never records page content (markdown, HTML, screenshots, links, titles, descriptions), URL paths or query strings, JSON-format prompts or schemas, or the Firecrawl API key. The key is redacted from every attribute the package sets, including a key taken from `FIRECRAWL_API_KEY`. The search query is recorded by design. On an error, the span status and exception event carry the SDK's exception message as raised. +This package has no content-capture setting. It never records page content (markdown, HTML, screenshots, links, titles, descriptions), URL paths or query strings, JSON-format prompts or schemas, or the Firecrawl API key. The key is redacted from every attribute the package sets, including a key taken from `FIRECRAWL_API_KEY`. The search query is recorded by design. Error events contain a safe type and generic message; status descriptions contain that type or `cancelled`. Only HTTP status and recognized machine codes are retained from API errors; unknown codes are omitted. Instrumentation errors never reach your code: if reading an argument or a result fails, the call still returns the SDK's result or raises the SDK's exception, and the span still ends. +Pass `config=TraceConfig(...)` from `fi_instrumentation.instrumentation` to `instrument()` to use shared privacy controls. `None` creates `TraceConfig()` using its `FI_*` environment settings; other objects raise `TypeError`. `hide_inputs` or `hide_input_text` omits the search query. `pii_redaction` (or `FI_PII_REDACTION=true`) applies the shared regex redactor to query text before truncation, followed by FITracer attribute masking. Other message, image and output hide flags are not applicable because no messages or page bodies are exported. Shared `suppress_tracing()` emits no Firecrawl spans, and `using_session()` supplies the shared session attribute. If API-key discovery fails, free-text attributes are omitted while safe format/count metadata remains. Error events and status use the separate safe metadata policy described above. + ## Client construction order `Firecrawl` and `AsyncFirecrawl` copy the v2 client's methods when they are constructed. Call `instrument()` before you construct them: diff --git a/python/frameworks/firecrawl/pyproject.toml b/python/frameworks/firecrawl/pyproject.toml index 496435756..4e9fcb696 100644 --- a/python/frameworks/firecrawl/pyproject.toml +++ b/python/frameworks/firecrawl/pyproject.toml @@ -14,6 +14,7 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Topic :: Software Development :: Libraries :: Python Modules", ] diff --git a/python/frameworks/firecrawl/tests/_firecrawl_fake.py b/python/frameworks/firecrawl/tests/_firecrawl_fake.py index 546c6350f..abd9ea3fb 100644 --- a/python/frameworks/firecrawl/tests/_firecrawl_fake.py +++ b/python/frameworks/firecrawl/tests/_firecrawl_fake.py @@ -18,7 +18,7 @@ PAGE = {"markdown": PAGE_BODY, "metadata": {"sourceURL": "https://example.com/1", "statusCode": 200}} -Payload = Union[Dict[str, Any], Callable[[Dict[str, Any]], Dict[str, Any]]] +Payload = Union[bytes, Dict[str, Any], Callable[[Dict[str, Any]], Dict[str, Any]]] Route = Tuple[int, Payload] @@ -91,7 +91,7 @@ def _respond(self, method: str) -> None: ) if callable(payload): payload = payload(body) - data = json.dumps(payload).encode() + data = payload if isinstance(payload, bytes) else json.dumps(payload).encode() self.send_response(status) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(data))) diff --git a/python/frameworks/firecrawl/tests/test_contract_harness.py b/python/frameworks/firecrawl/tests/test_contract_harness.py index c8b793d28..bd65e8fab 100644 --- a/python/frameworks/firecrawl/tests/test_contract_harness.py +++ b/python/frameworks/firecrawl/tests/test_contract_harness.py @@ -249,12 +249,15 @@ def test_no_key_path_or_content_is_exported(fake_firecrawl, monkeypatch): _EXIT_SCRIPT = """ import os import sys +from pathlib import Path +import fi_instrumentation from firecrawl import Firecrawl from fi_instrumentation import register from fi_instrumentation.fi_types import ProjectType from traceai_firecrawl import FirecrawlInstrumentor +print("FI_CORE_FILE=" + str(Path(fi_instrumentation.__file__).resolve()), flush=True) provider = register(project_type=ProjectType.OBSERVE, project_name=os.environ["PROJECT"], verbose=False) FirecrawlInstrumentor().instrument(tracer_provider=provider) Firecrawl(api_key=os.environ["FIRECRAWL_KEY"], api_url=os.environ["FIRECRAWL_BASE_URL"], max_retries=1).scrape( @@ -273,15 +276,18 @@ def test_scrape_span_is_exported_at_process_exit(fake_firecrawl, exit_mode, expo exit hooks and must export nothing, which shows the receiver is not fed any other way.""" import os + import fi_instrumentation from harness import run with Receiver() as receiver: env = dict(os.environ) here = Path(__file__).resolve() - # The package and the in-repo fi_instrumentation, whatever the cwd. + core_file = Path(fi_instrumentation.__file__).resolve() + # Test the same core the parent imported, including the published wheel. env["PYTHONPATH"] = os.pathsep.join( - [str(here.parents[1]), str(here.parents[3])] + [p for p in [env.get("PYTHONPATH")] if p] + [str(here.parents[1]), str(core_file.parent.parent)] + + [p for p in [env.get("PYTHONPATH")] if p] ) env.update( FI_BASE_URL=receiver.origin, @@ -296,5 +302,9 @@ def test_scrape_span_is_exported_at_process_exit(fake_firecrawl, exit_mode, expo assert not result.timed_out, result.stderr.decode(errors="replace") assert result.returncode == 0, result.stderr.decode(errors="replace") + assert "FI_CORE_FILE=" + str(core_file) in result.stdout.decode() + core_root = core_file.parent.parent + if core_root != Path(__file__).resolve().parents[3]: + assert "site-packages" in core_root.parts assert fake_firecrawl.calls == [("POST", "/v2/scrape")] assert [span["name"] for span in spans] == (["firecrawl.scrape"] if exported else []) diff --git a/python/frameworks/firecrawl/tests/test_packaging.py b/python/frameworks/firecrawl/tests/test_packaging.py index e56a50432..8298ae671 100644 --- a/python/frameworks/firecrawl/tests/test_packaging.py +++ b/python/frameworks/firecrawl/tests/test_packaging.py @@ -86,5 +86,15 @@ def test_pyproject_matches_the_supported_range() -> None: def test_classifiers_list_only_tested_pythons() -> None: classifiers = re.findall(r'"Programming Language :: Python :: (3\.\d+)"', PYPROJECT.read_text()) - assert classifiers == ["3.10", "3.11", "3.13"] + # Include the already admitted 3.12 minor; match the declared constraint. + assert classifiers == ["3.10", "3.11", "3.12", "3.13"] assert _poetry_value("python") == ">=3.10,<3.14" + + +def test_python_constraint_matches_classifiers() -> None: + from packaging.specifiers import SpecifierSet + + classifiers = set(re.findall(r'"Programming Language :: Python :: (3\.\d+)"', PYPROJECT.read_text())) + constraint = SpecifierSet(_poetry_value("python")) + admitted = {"3." + str(minor) for minor in range(20) if constraint.contains("3." + str(minor))} + assert admitted == classifiers diff --git a/python/frameworks/firecrawl/tests/test_review_regressions.py b/python/frameworks/firecrawl/tests/test_review_regressions.py new file mode 100644 index 000000000..ae7a09661 --- /dev/null +++ b/python/frameworks/firecrawl/tests/test_review_regressions.py @@ -0,0 +1,517 @@ +"""Bounded review regressions through the real SDK and OTLP Receiver. + +All network traffic goes to loopback fixtures; every credential is synthetic. +""" +from __future__ import annotations + +import asyncio +from functools import wraps +from importlib import import_module +import json + +import pytest + +from _firecrawl_fake import FakeFirecrawl, PAGE_BODY +from harness import Receiver, _flatten_attributes +from traceai_firecrawl import FirecrawlInstrumentor + +KEY = "fc-regression-placeholder-key" +PRIVATE_URL = "https://example.com/private-review-sentinel?secret=review" +HTML = "PRIVATE-PAGE-REVIEW-SENTINEL" + + +@pytest.fixture() +def fake(): + server = FakeFirecrawl() + try: + yield server + finally: + server.close() + + +@pytest.fixture() +def telemetry(monkeypatch): + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv("FI_API_KEY", "placeholder-fi-api-key") + monkeypatch.setenv("FI_SECRET_KEY", "placeholder-fi-secret-key") + provider = register(project_name="firecrawl-review", project_type=ProjectType.OBSERVE, + verbose=False) + instrumentor = FirecrawlInstrumentor() + try: + yield receiver, provider, instrumentor + finally: + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + provider.shutdown() + + +def invoke(fake, use_async, method="scrape", target=PRIVATE_URL, env_key=False, **kwargs): + from firecrawl import AsyncFirecrawl, Firecrawl + + options = dict(api_url=fake.origin, max_retries=1) + if not env_key: + options["api_key"] = KEY + if use_async: + async def journey(): + client = AsyncFirecrawl(**options) + return await getattr(client, method)(target, **kwargs) + return asyncio.run(journey()) + return getattr(Firecrawl(**options), method)(target, **kwargs) + + +def exported(telemetry): + receiver, provider, _ = telemetry + assert provider.force_flush(timeout_millis=10_000) + spans = receiver.spans() + return spans, json.dumps(spans) + json.dumps(receiver.requests()) + + +def format_names(span): + value = _flatten_attributes(span["attributes"]).get("firecrawl.formats") + if value is None: + return None + return [item["stringValue"] for item in value["arrayValue"]["values"]] + + +def observe_vendor(monkeypatch, use_async, method): + """Observe the real SDK's exact result/exception without replacing it.""" + module = import_module("firecrawl.v2.client_async" if use_async else "firecrawl.v2.client") + cls = getattr(module, "AsyncFirecrawlClient" if use_async else "FirecrawlClient") + original = getattr(cls, method) + observed = [] + if use_async: + @wraps(original) + async def call(*args, **kwargs): + try: + result = await original(*args, **kwargs) + except BaseException as error: + observed.append(error) + raise + observed.append(result) + return result + else: + @wraps(original) + def call(*args, **kwargs): + try: + result = original(*args, **kwargs) + except BaseException as error: + observed.append(error) + raise + observed.append(result) + return result + monkeypatch.setattr(cls, method, call) + return observed + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("error_shape", ["error", "details", "nonjson", "validation", "unsafe-code"]) +def test_f1_exception_wire_omits_untrusted_text(fake, telemetry, monkeypatch, use_async, error_shape): + from pydantic import ValidationError + from firecrawl.v2.utils.error_handler import FirecrawlError + + sentinel = " ".join((KEY, PRIVATE_URL, HTML)) + options = {} + if error_shape == "validation": + options["timeout"] = sentinel + expected = ValidationError + else: + body = {"success": False, "error": "failed", "code": "RATE_LIMIT_EXCEEDED"} + if error_shape == "nonjson": + body = sentinel.encode() + elif error_shape == "unsafe-code": + body["code"] = sentinel + else: + body[error_shape] = sentinel + fake.routes[("POST", "/v2/scrape")] = (400, body) + expected = FirecrawlError + observed = observe_vendor(monkeypatch, use_async, "scrape") + telemetry[2].instrument(tracer_provider=telemetry[1]) + with pytest.raises(expected) as caught: + invoke(fake, use_async, **options) + assert caught.value is observed[0] + # A control demonstrates that the vendor actually received the markers. + if error_shape == "validation": + assert caught.value.errors()[0]["input"] == sentinel + elif error_shape == "unsafe-code": + assert caught.value.code == sentinel + else: + assert sentinel in str(caught.value) + spans, wire = exported(telemetry) + assert len(spans) == 1 + span = spans[0] + assert span["status"]["code"] == "STATUS_CODE_ERROR" + assert len(span["events"]) == 1 and span["events"][0]["name"] == "exception" + for secret in (KEY, KEY[:16], PRIVATE_URL, HTML, "REVIEW-SENTINEL"): + assert secret not in wire + event = _flatten_attributes(span["events"][0]["attributes"]) + assert "exception.stacktrace" not in event + assert event["exception.type"] == type(caught.value).__name__ + assert span["status"]["message"] == type(caught.value).__name__ + attributes = _flatten_attributes(span["attributes"]) + if error_shape not in ("validation", "nonjson", "unsafe-code"): + assert attributes["firecrawl.error.code"] == "RATE_LIMIT_EXCEEDED" + else: + assert "firecrawl.error.code" not in attributes + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("env_key", [False, True], ids=["explicit-key", "env-key"]) +@pytest.mark.parametrize("shape", ["dict", "boundary-dict", "string"]) +@pytest.mark.parametrize("failed", [False, True], ids=["success", "failure"]) +def test_f2_format_labels_are_canonical(fake, telemetry, monkeypatch, use_async, env_key, shape, failed): + from firecrawl.v2.types import JsonFormat + + monkeypatch.setenv("FIRECRAWL_API_KEY", KEY) + name = ("x" * 60 if shape == "boundary-dict" else "") + KEY + invalid = name if shape == "string" else {"type": name, "prompt": HTML, "schema": {"title": KEY}} + observed = observe_vendor(monkeypatch, use_async, "scrape") + telemetry[2].instrument(tracer_provider=telemetry[1]) + options = dict(formats=["html", invalid, {"type": "person@example.test"}, + JsonFormat(prompt=HTML, schema={"title": KEY}), "html"]) + if failed: + fake.routes[("POST", "/v2/scrape")] = (400, {"success": False, "error": HTML}) + if failed or shape == "string": + with pytest.raises(Exception) as caught: + invoke(fake, use_async, env_key=env_key, **options) + assert caught.value is observed[0] + else: + result = invoke(fake, use_async, env_key=env_key, **options) + assert result is observed[0] and result.markdown == PAGE_BODY + spans, wire = exported(telemetry) + assert len(spans) == 1 + assert format_names(spans[0]) == ["html", "json"] + for secret in (KEY, KEY[:4], "x" * 60, "person@example.test", HTML): + assert secret not in wire + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +def test_f2_format_aliases_match_vendor_payload(fake, telemetry, use_async): + from firecrawl.v2.types import Format, QuestionFormat, HighlightsFormat + + payloads = [] + def response(body): + payloads.append(body) + return {"success": True, "data": {"markdown": PAGE_BODY}} + fake.routes[("POST", "/v2/scrape")] = (200, response) + telemetry[2].instrument(tracer_provider=telemetry[1]) + invoke(fake, use_async, formats=["raw_html", "rawHtml", Format(type="audio"), + {"type": "change_tracking", "modes": ["git-diff"]}, + QuestionFormat(question=HTML), HighlightsFormat(query=HTML)]) + spans, wire = exported(telemetry) + assert format_names(spans[0]) == [ + "rawHtml", "audio", "changeTracking", "question", "highlights"] + actual = [item if isinstance(item, str) else item["type"] for item in payloads[0]["formats"]] + assert list(dict.fromkeys(actual)) == ["rawHtml", "audio", "changeTracking", "question", "highlights"] + assert HTML not in wire + + +def test_f5_readme_usage_registers_observe(fake, telemetry): + import ast + from pathlib import Path + + readme = Path(__file__).resolve().parents[1] / "README.md" + snippet = readme.read_text().split("```python\n", 1)[1].split("```", 1)[0] + tree = ast.parse(snippet) + redirected = 0 + for node in ast.walk(tree): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "Firecrawl": + # Execute the documented public path with just the network endpoint + # redirected to loopback. Preserve its register/import arguments. + node.keywords.append(ast.keyword(arg="api_url", value=ast.Constant(fake.origin))) + redirected += 1 + assert redirected == 1 + namespace = {} + try: + exec(compile(ast.fix_missing_locations(tree), "README.md usage", "exec"), namespace) + assert namespace["tracer_provider"].force_flush(timeout_millis=10_000) + finally: + if telemetry[2].is_instrumented_by_opentelemetry: + telemetry[2].uninstrument() + if "tracer_provider" in namespace: + namespace["tracer_provider"].shutdown() + assert fake.calls == [("POST", "/v2/scrape")] + exports = telemetry[0].requests() + assert exports and len(telemetry[0].spans()) == 1 + for export in exports: + for resource in export["resource_attributes"]: + assert resource["project_name"] == "firecrawl" + assert resource["project_type"] == "observe" + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("method", ["search", "start_crawl"]) +@pytest.mark.parametrize("shape", ["flags", "explicit-mapping", "defaults"]) +@pytest.mark.parametrize("failed", [False, True], ids=["success", "failure"]) +def test_f6_scrapeformats_match_actual_payload(fake, telemetry, monkeypatch, use_async, method, shape, failed): + from firecrawl.v2.types import ScrapeOptions, ScrapeFormats, JsonFormat + + if shape == "flags": + formats = ScrapeFormats(markdown=False, html=True, images=True, json=True) + expected = ["html"] # SDK 4.46.2 does not serialize the images/json flags. + elif shape == "explicit-mapping": + formats = ScrapeFormats(formats=[JsonFormat(prompt=HTML, schema={"title": KEY}), "links"], + markdown=False, html=True, raw_html=True) + expected = ["json", "links", "html", "rawHtml"] + else: + formats = ScrapeFormats(formats=["html"]) + expected = ["html", "markdown"] # default markdown is actually serialized. + scrape_options = {"formats": formats} if shape == "explicit-mapping" else ScrapeOptions(formats=formats) + route = "/v2/search" if method == "search" else "/v2/crawl" + _, response = fake.routes[("POST", route)] + payloads = [] + def respond(body): + payloads.append(body) + return {"success": False, "error": HTML} if failed else response + fake.routes[("POST", route)] = (400 if failed else 200, respond) + observed = observe_vendor(monkeypatch, use_async, method) + telemetry[2].instrument(tracer_provider=telemetry[1]) + if failed: + with pytest.raises(Exception) as caught: + invoke(fake, use_async, method=method, target="search" if method == "search" else PRIVATE_URL, + scrape_options=scrape_options) + assert caught.value is observed[0] + else: + result = invoke(fake, use_async, method=method, target="search" if method == "search" else PRIVATE_URL, + scrape_options=scrape_options) + assert result is observed[0] + actual = payloads[0]["scrapeOptions"]["formats"] + actual_names = [item if isinstance(item, str) else item["type"] for item in actual] + assert list(dict.fromkeys(actual_names)) == expected + spans, wire = exported(telemetry) + assert len(spans) == 1 and format_names(spans[0]) == expected + for secret in (HTML, KEY, PRIVATE_URL, PAGE_BODY): + assert secret not in wire + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("nested", [None, ["html"]], ids=["no-nested-formats", "nested-formats"]) +def test_f6_scrape_options_override_convenience_formats(fake, telemetry, use_async, nested): + from firecrawl.v2.types import ScrapeOptions + + payloads = [] + def respond(body): + payloads.append(body) + return {"success": True, "id": "review-job", "url": "unused"} + fake.routes[("POST", "/v2/crawl")] = (200, respond) + telemetry[2].instrument(tracer_provider=telemetry[1]) + invoke(fake, use_async, method="start_crawl", formats=["markdown"], + scrape_options=ScrapeOptions(formats=nested)) + spans, _ = exported(telemetry) + assert payloads[0].get("scrapeOptions", {}).get("formats") == nested + assert format_names(spans[0]) == nested + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +def test_f3_default_query_uses_shared_session_context(fake, telemetry, monkeypatch, use_async): + from fi_instrumentation.instrumentation import using_session + from fi_instrumentation.fi_types import SpanAttributes + + observed = observe_vendor(monkeypatch, use_async, "search") + telemetry[2].instrument(tracer_provider=telemetry[1], config=None) + query = "find person@example.test" + with using_session("review-session"): + result = invoke(fake, use_async, method="search", target=query) + assert result is observed[0] + spans, _ = exported(telemetry) + assert len(spans) == 1 + attributes = _flatten_attributes(spans[0]["attributes"]) + assert attributes["fi.retrieval.query"] == query + assert attributes[SpanAttributes.GEN_AI_CONVERSATION_ID] == "review-session" + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("policy", ["config", "env"]) +@pytest.mark.parametrize("query", ["find person@example.test 415-555-2671", + "x " * 507 + "boundary.person@example.test", + "x " * 508 + "415-555-2671"], + ids=["ordinary", "email-boundary", "phone-boundary"]) +def test_f3_query_pii_redaction_precedes_cap(fake, telemetry, monkeypatch, use_async, policy, query): + from fi_instrumentation.instrumentation import TraceConfig + from fi_instrumentation.instrumentation.pii_redaction import redact_pii_in_string + + if policy == "env": + monkeypatch.setenv("FI_PII_REDACTION", "true") + config = None + else: + config = TraceConfig(pii_redaction=True) + telemetry[2].instrument(tracer_provider=telemetry[1], config=config) + result = invoke(fake, use_async, method="search", target=query) + assert len(result.web) == 2 + spans, wire = exported(telemetry) + assert len(spans) == 1 + expected = redact_pii_in_string(query)[:1024] + assert _flatten_attributes(spans[0]["attributes"])["fi.retrieval.query"] == expected + for secret in ("person@example.test", "boundary.person", "415-555"): + assert secret not in wire + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_input_text"]) +@pytest.mark.parametrize("policy", ["config", "env"]) +def test_f3_hidden_inputs_omit_search_query(fake, telemetry, monkeypatch, use_async, flag, policy): + from fi_instrumentation.instrumentation import TraceConfig + + if policy == "env": + monkeypatch.setenv("FI_" + flag.upper(), "true") + config = None + else: + config = TraceConfig(**{flag: True}) + telemetry[2].instrument(tracer_provider=telemetry[1], config=config) + result = invoke(fake, use_async, method="search", target="QUERY-TO-HIDE-REVIEW") + assert len(result.web) == 2 + spans, wire = exported(telemetry) + assert len(spans) == 1 + assert "fi.retrieval.query" not in _flatten_attributes(spans[0]["attributes"]) + assert "QUERY-TO-HIDE-REVIEW" not in wire + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("failed", [False, True], ids=["success", "failure"]) +def test_f3_shared_suppression_emits_no_spans(fake, telemetry, monkeypatch, use_async, failed): + from fi_instrumentation.instrumentation import suppress_tracing + + if failed: + fake.routes[("POST", "/v2/search")] = (400, {"success": False, "error": HTML}) + observed = observe_vendor(monkeypatch, use_async, "search") + telemetry[2].instrument(tracer_provider=telemetry[1]) + with suppress_tracing(): + if failed: + with pytest.raises(Exception) as caught: + invoke(fake, use_async, method="search", target="query") + assert caught.value is observed[0] + else: + assert invoke(fake, use_async, method="search", target="query") is observed[0] + assert fake.calls_to("POST", "/v2/search") == 1 + spans, _ = exported(telemetry) + assert spans == [] + # Suppression and the local nested-call guard both reset for the next call. + fake.routes[("POST", "/v2/search")] = (200, {"success": True, "data": {"web": []}}) + invoke(fake, use_async, method="search", target="after-suppression") + spans, _ = exported(telemetry) + assert [span["name"] for span in spans] == ["firecrawl.search"] + + +@pytest.mark.parametrize("invalid", [object(), {}], ids=["object", "dict"]) +def test_f3_rejects_invalid_trace_config(telemetry, invalid): + with pytest.raises(TypeError, match="TraceConfig"): + telemetry[2].instrument(tracer_provider=telemetry[1], config=invalid) + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +def test_f3_key_discovery_failure_keeps_only_safe_metadata(fake, telemetry, use_async): + from firecrawl import Firecrawl, AsyncFirecrawl + from firecrawl.v2.types import ScrapeOptions + + class UnreadableConfig: + @property + def api_key(self): + raise RuntimeError(HTML) + + telemetry[2].instrument(tracer_provider=telemetry[1]) + async def journey(): + client = AsyncFirecrawl(api_key=KEY, api_url=fake.origin, max_retries=1) + client._v2_client.config = UnreadableConfig() + return await client.search("query " + KEY, scrape_options=ScrapeOptions(formats=["html"])) + if use_async: + result = asyncio.run(journey()) + else: + client = Firecrawl(api_key=KEY, api_url=fake.origin, max_retries=1) + client._v2_client.config = UnreadableConfig() + result = client.search("query " + KEY, scrape_options=ScrapeOptions(formats=["html"])) + assert len(result.web) == 2 + spans, wire = exported(telemetry) + attributes = _flatten_attributes(spans[0]["attributes"]) + assert "fi.retrieval.query" not in attributes + assert int(attributes["fi.retrieval.document_count"]) == 2 + assert format_names(spans[0]) == ["html"] + assert KEY not in wire and HTML not in wire + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("groups,expected", [ + ({"web": 2}, 2), ({"news": 2}, 2), ({"images": 2}, 2), + ({"web": 1, "news": 2, "images": 3}, 6), ({"web": 0, "news": 2}, 2), + ({"web": 0, "news": 0, "images": 0, "tools": 0}, 0), ({}, 0), + ({"tools": 2}, 2), ({"web": 1, "tools": 2}, 3), +], ids=["web", "news", "images", "mixed", "empty-web-news", "all-empty", "defaults-empty", + "tools", "web-tools"]) +def test_f4_counts_all_sdk_search_groups(fake, telemetry, monkeypatch, use_async, groups, expected): + from firecrawl.v2.types import SearchData, DiscoveredTool + + data = {} + for group, count in groups.items(): + record = ({"provider": "fixture", "capability": "search", "description": HTML} + if group == "tools" else {"url": PRIVATE_URL, "title": HTML}) + data[group] = [record.copy() for _ in range(count)] + fake.routes[("POST", "/v2/search")] = (200, {"success": True, "data": data}) + observed = observe_vendor(monkeypatch, use_async, "search") + telemetry[2].instrument(tracer_provider=telemetry[1]) + result = invoke(fake, use_async, method="search", target="search") + assert result is observed[0] and isinstance(result, SearchData) + assert sum(len(getattr(result, group) or []) for group in ("web", "news", "images", "tools")) == expected + if result.tools: + assert all(isinstance(item, DiscoveredTool) for item in result.tools) + spans, wire = exported(telemetry) + assert len(spans) == 1 + assert int(_flatten_attributes(spans[0]["attributes"])["fi.retrieval.document_count"]) == expected + assert HTML not in wire and PRIVATE_URL not in wire + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("shape,expected", [("unknown", None), ("malformed", None), + ("data", 2), ("results", 2)]) +def test_f4_unknown_counts_are_omitted_and_legacy_lists_survive(fake, telemetry, monkeypatch, use_async, shape, expected): + from types import SimpleNamespace + from firecrawl.v2.types import SearchData, Document + + # Unrecognized/malformed return shapes cannot come through validated normal + # HTTP responses. Inject only these compatibility/error probes, using real + # SDK models for list records and checking identity of the returned object. + if shape == "unknown": + result = object() + elif shape == "malformed": + result = SearchData.model_construct(news=HTML) + else: + result = SimpleNamespace(**{shape: [Document(markdown=HTML), Document(markdown=HTML)]}) + module = import_module("firecrawl.v2.client_async" if use_async else "firecrawl.v2.client") + cls = getattr(module, "AsyncFirecrawlClient" if use_async else "FirecrawlClient") + def search(*args, **kwargs): + return result + async def asearch(*args, **kwargs): + return result + monkeypatch.setattr(cls, "search", asearch if use_async else search) + telemetry[2].instrument(tracer_provider=telemetry[1]) + assert invoke(fake, use_async, method="search", target="search") is result + spans, wire = exported(telemetry) + attributes = _flatten_attributes(spans[0]["attributes"]) + if expected is None: + assert "fi.retrieval.document_count" not in attributes + else: + assert int(attributes["fi.retrieval.document_count"]) == expected + assert HTML not in wire + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +def test_invalid_job_status_is_rejected_without_exporting_validation_input(fake, telemetry, monkeypatch, use_async): + from pydantic import ValidationError + + sentinel = " ".join((KEY, PRIVATE_URL, HTML)) + fake.routes[("GET", "/v2/crawl/review-job")] = ( + 200, {"success": True, "status": sentinel, "total": 1, "completed": 0, + "creditsUsed": 0, "data": []}, + ) + observed = observe_vendor(monkeypatch, use_async, "get_crawl_status") + telemetry[2].instrument(tracer_provider=telemetry[1]) + with pytest.raises(ValidationError) as caught: + invoke(fake, use_async, method="get_crawl_status", target="review-job") + assert caught.value is observed[0] and caught.value.errors()[0]["input"] == sentinel + spans, wire = exported(telemetry) + assert len(spans) == 1 and spans[0]["status"]["code"] == "STATUS_CODE_ERROR" + assert len(spans[0]["events"]) == 1 + assert "firecrawl.status" not in _flatten_attributes(spans[0]["attributes"]) + for secret in (KEY, PRIVATE_URL, HTML): + assert secret not in wire diff --git a/python/frameworks/firecrawl/traceai_firecrawl/__init__.py b/python/frameworks/firecrawl/traceai_firecrawl/__init__.py index 09846debd..fba738bd6 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/__init__.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/__init__.py @@ -6,6 +6,8 @@ from opentelemetry import trace as trace_api from opentelemetry.instrumentation.instrumentor import BaseInstrumentor from wrapt import wrap_function_wrapper +from fi_instrumentation import FITracer +from fi_instrumentation.instrumentation import TraceConfig from traceai_firecrawl._wrappers import AsyncOperationWrapper, OperationWrapper from traceai_firecrawl.package import _instruments @@ -44,8 +46,13 @@ def instrumentation_dependencies(self) -> Collection[str]: return _instruments def _instrument(self, **kwargs: Any) -> None: + config = kwargs.get("config") + if config is None: + config = TraceConfig() + elif not isinstance(config, TraceConfig): + raise TypeError("config must be a TraceConfig") tracer_provider = kwargs.get("tracer_provider") or trace_api.get_tracer_provider() - tracer = trace_api.get_tracer(__name__, __version__, tracer_provider) + tracer = FITracer(trace_api.get_tracer(__name__, __version__, tracer_provider), config=config) self._original_methods: Dict[Tuple[str, str, str], Any] = {} for module_name, class_name, is_async in ( @@ -54,9 +61,9 @@ def _instrument(self, **kwargs: Any) -> None: ): for method_name, span_name in _OPERATIONS.items(): wrapper = ( - AsyncOperationWrapper(tracer, span_name, method_name) + AsyncOperationWrapper(tracer, span_name, method_name, config) if is_async - else OperationWrapper(tracer, span_name, method_name) + else OperationWrapper(tracer, span_name, method_name, config) ) self._wrap(module_name, class_name, method_name, wrapper) diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index 1c30ec2fe..a45dcfe67 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -11,6 +11,8 @@ from opentelemetry import trace as trace_api from opentelemetry.trace import Span, Status, StatusCode, Tracer +from fi_instrumentation.instrumentation import TraceConfig +from fi_instrumentation.instrumentation.pii_redaction import redact_pii_in_string logger = logging.getLogger(__name__) @@ -41,8 +43,29 @@ _ERROR_JOB_STATUSES = {"failed", "cancelled"} # Methods whose arguments carry the requested output formats. _FORMAT_METHODS = {"scrape", "search", "crawl", "start_crawl"} -_MAX_FORMAT_NAME_LENGTH = 64 +# SDK 4.46.2 FormatString/model Literals and validation aliases. The finite +# vocabulary bounds both label length and the deduplicated attribute list. +_FORMAT_NAMES = {name: name for name in ( + "markdown", "html", "rawHtml", "links", "images", "screenshot", "summary", + "changeTracking", "json", "attributes", "branding", "product", "menu", + "query", "audio", "video", "question", "highlights", "screenshot@fullPage", +)} +_FORMAT_NAMES.update(raw_html="rawHtml", change_tracking="changeTracking", + screenshot_full_page="screenshot@fullPage") _CANCELLATION_ERRORS = (asyncio.CancelledError, concurrent.futures.CancelledError) +# Fixed metadata vocabulary: arbitrary exception names/codes are caller content. +_SAFE_EXCEPTION_TYPES = frozenset({ + "Exception", "RuntimeError", "ValueError", "TypeError", "TimeoutError", + "CancelledError", "ValidationError", "FirecrawlError", "BadRequestError", + "UnauthorizedError", "PaymentRequiredError", "WebsiteNotSupportedError", + "ProviderTermsRequiredError", "RequestTimeoutError", "CrawlJobTimeoutError", + "RateLimitError", "InternalServerError", +}) +_SAFE_ERROR_CODES = frozenset({ + "RATE_LIMIT_EXCEEDED", "THIRD_PARTY_DATA_TERMS_REQUIRED", "duplicate_request", + "request_in_flight", "request_unresolved", "unknown_provider", + "insufficient_credits", "billing_unavailable", +}) # True while a traced Firecrawl call runs in this context. firecrawl-py 4.46.2 # AsyncFirecrawlClient.crawl awaits self.start_crawl, which is wrapped as well; @@ -71,9 +94,16 @@ def _api_keys(instance: Any) -> list[str]: return keys -def _redact(value: str, instance: Any) -> str: - for api_key in _api_keys(instance): +def _redact(value: str, instance: Any, config: Optional[TraceConfig] = None) -> Optional[str]: + try: + keys = _api_keys(instance) + except Exception: + logger.debug("Could not discover Firecrawl API keys; omitting free text") + return None + for api_key in keys: value = value.replace(api_key, "[redacted]") + if config is not None and config.pii_redaction: + value = redact_pii_in_string(value) return value[:_MAX_QUERY_LENGTH] @@ -82,18 +112,28 @@ def _url_host(value: Any, instance: Any) -> str: if not isinstance(value, str) or not value: return "" host = urlsplit(value).hostname or "" - return _redact(host, instance) + return _redact(host, instance) or "" -def _document_count(result: Any) -> int: +def _document_count(result: Any) -> Optional[int]: + from firecrawl.v2.types import SearchData + + if isinstance(result, SearchData): + count = 0 + for group in ("web", "news", "images", "tools"): + records = getattr(result, group, None) + if records is not None: + if not isinstance(records, (list, tuple)): + return None + count += len(records) + return count + # Preserve the prior generic list-result contract; strings, mappings and + # unrecognized objects do not establish a measured result count. for attribute in ("data", "results", "web"): value = getattr(result, attribute, None) if value is not None: - try: - return len(value) - except TypeError: - return 0 - return 0 + return len(value) if isinstance(value, (list, tuple)) else None + return None def _page_count(result: Any) -> int: @@ -131,16 +171,30 @@ def _call_job_id(method_name: str, args: Sequence[Any], kwargs: Mapping[str, Any return _job_id(None, kwargs) -def _formats(kwargs: Mapping[str, Any]) -> list[str]: +def _formats(kwargs: Mapping[str, Any], method_name: str) -> list[str]: """Names of the requested formats. A format object's prompt or schema is content, so only its ``type`` is kept.""" - formats = kwargs.get("formats") - if formats is None: - scrape_options = kwargs.get("scrape_options") + from firecrawl.v2.types import ScrapeFormats + + scrape_options = kwargs.get("scrape_options") + # The SDK ignores convenience formats when scrape_options is supplied. + # Search has no supported top-level formats argument. + if scrape_options is not None or method_name == "search": if isinstance(scrape_options, Mapping): formats = scrape_options.get("formats") else: formats = getattr(scrape_options, "formats", None) + else: + formats = kwargs.get("formats") + if isinstance(formats, ScrapeFormats): + container = formats + formats = list(container.formats or ()) + # Match prepare_scrape_options in 4.46.2: images/json flags are not + # serialized. Its markdown default is serialized, unless disabled. + for flag in ("markdown", "html", "raw_html", "summary", "links", + "screenshot", "change_tracking"): + if getattr(container, flag) is True: + formats.append(flag) if not isinstance(formats, (list, tuple)): return [] names = [] @@ -151,8 +205,10 @@ def _formats(kwargs: Mapping[str, Any]) -> list[str]: name = item.get("type") else: name = getattr(item, "type", None) - if isinstance(name, str) and name: - names.append(name[:_MAX_FORMAT_NAME_LENGTH]) + if isinstance(name, str): + canonical = _FORMAT_NAMES.get(name) + if canonical is not None and canonical not in names: + names.append(canonical) return names @@ -168,7 +224,7 @@ def _vendor_error_fields(error: BaseException) -> tuple[Optional[int], Optional[ code = getattr(error, "code", None) return ( status_code if isinstance(status_code, int) and not isinstance(status_code, bool) else None, - code if isinstance(code, str) and code else None, + code if isinstance(code, str) and code in _SAFE_ERROR_CODES else None, ) @@ -181,25 +237,29 @@ class _BaseWrapper: that starts is ended exactly once. """ - def __init__(self, tracer: Tracer, span_name: str, method_name: str) -> None: + def __init__(self, tracer: Tracer, span_name: str, method_name: str, config: TraceConfig) -> None: self._tracer = tracer self._span_name = span_name self._method_name = method_name + self._config = config def _attributes( self, instance: Any, args: Sequence[Any], kwargs: Mapping[str, Any] ) -> dict[str, Any]: attributes = {_FI_SPAN_KIND: _TOOL} if self._method_name == "search": - query = kwargs.get("query", args[0] if args else "") - attributes[_RETRIEVAL_QUERY] = _redact(str(query or ""), instance) + if not (self._config.hide_inputs or self._config.hide_input_text): + query = kwargs.get("query", args[0] if args else "") + query_text = _redact(str(query or ""), instance, self._config) + if query_text is not None: + attributes[_RETRIEVAL_QUERY] = query_text elif self._method_name in ("scrape", "map", "crawl", "start_crawl"): url = kwargs.get("url", args[0] if args else "") host = _url_host(url, instance) if host: attributes["server.address"] = host if self._method_name in _FORMAT_METHODS: - formats = _formats(kwargs) + formats = _formats(kwargs, self._method_name) if formats: attributes[_FORMATS] = formats if self._method_name in _CRAWL_METHODS: @@ -208,7 +268,9 @@ def _attributes( attributes[_LIMIT] = limit job_id = _call_job_id(self._method_name, args, kwargs) if job_id: - attributes[_JOB_ID] = _redact(job_id, instance) + safe_id = _redact(job_id, instance) + if safe_id is not None: + attributes[_JOB_ID] = safe_id return attributes def _start( @@ -229,11 +291,15 @@ def _record_result( self, span: Span, instance: Any, result: Any, kwargs: Mapping[str, Any] ) -> None: if self._method_name == "search": - span.set_attribute(_RETRIEVAL_DOCUMENT_COUNT, _document_count(result)) + count = _document_count(result) + if count is not None: + span.set_attribute(_RETRIEVAL_DOCUMENT_COUNT, count) if self._method_name in _CRAWL_METHODS: job_id = _job_id(result, kwargs) if job_id: - span.set_attribute(_JOB_ID, _redact(job_id, instance)) + safe_id = _redact(job_id, instance) + if safe_id is not None: + span.set_attribute(_JOB_ID, safe_id) if self._method_name == "crawl": span.set_attribute(_PAGE_COUNT, _page_count(result)) if self._method_name == "cancel_crawl" and isinstance(result, bool): @@ -275,15 +341,20 @@ def _finish_error(self, span: Span, error: BaseException, instance: Any = None) if status_code is not None: span.set_attribute(_ERROR_STATUS_CODE, status_code) if code is not None: - span.set_attribute(_ERROR_CODE, _redact(code, instance)) + safe_code = _redact(code, instance) + if safe_code is not None: + span.set_attribute(_ERROR_CODE, safe_code) except Exception: logger.debug("Could not read %s error", self._span_name, exc_info=True) try: - span.record_exception(error) + span.add_event("exception", { + "exception.type": _exception_type(error), + "exception.message": "Firecrawl call failed", + }) except Exception: logger.debug("Could not record %s exception", self._span_name, exc_info=True) try: - description = "cancelled" if cancelled else _describe(error) + description = "cancelled" if cancelled else _exception_type(error) span.set_status(Status(StatusCode.ERROR, description)) except Exception: logger.debug("Could not set %s status", self._span_name, exc_info=True) @@ -291,11 +362,9 @@ def _finish_error(self, span: Span, error: BaseException, instance: Any = None) _end(span) -def _describe(error: BaseException) -> str: - try: - return "{0}: {1}".format(type(error).__name__, error) - except Exception: - return type(error).__name__ +def _exception_type(error: BaseException) -> str: + name = type(error).__name__ + return name if name in _SAFE_EXCEPTION_TYPES else "Exception" def _current(span: Span) -> Any: From cabf3a6576e0e3f72f12d4204fd2af9b1c8770f7 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 11 Oct 2026 20:27:40 -0700 Subject: [PATCH 19/22] fix(firecrawl): record the search query in input.value The search query was only written to fi.retrieval.query, which the Future AGI backend does not map, so the Observe span panel showed no input for firecrawl.search. Write the same redacted, capped query to input.value as the sibling tool tracers do. Hidden inputs still omit it. Co-Authored-By: Claude Opus 5.5 --- python/frameworks/firecrawl/README.md | 2 +- .../frameworks/firecrawl/tests/test_contract_harness.py | 1 + python/frameworks/firecrawl/tests/test_instrumentor.py | 2 ++ .../frameworks/firecrawl/tests/test_review_regressions.py | 8 ++++++-- .../frameworks/firecrawl/traceai_firecrawl/_wrappers.py | 3 +++ 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/python/frameworks/firecrawl/README.md b/python/frameworks/firecrawl/README.md index da5682b9b..cec2c78b3 100644 --- a/python/frameworks/firecrawl/README.md +++ b/python/frameworks/firecrawl/README.md @@ -35,7 +35,7 @@ Each call to `scrape`, `search`, `map`, `crawl`, `start_crawl`, `get_crawl_statu | Attribute | Methods | Value | |---|---|---| | `server.address` | `scrape`, `map`, `crawl`, `start_crawl` | Host of the target URL. Never the path or query. | -| `fi.retrieval.query` | `search` | The query, with the API key redacted, capped at 1024 characters. Omitted with `hide_inputs` or `hide_input_text`; shared PII redaction applies when enabled. | +| `fi.retrieval.query`, `input.value` | `search` | The query, with the API key redacted, capped at 1024 characters. Omitted with `hide_inputs` or `hide_input_text`; shared PII redaction applies when enabled. | | `fi.retrieval.document_count` | `search` | Total returned records across SDK `web`, `news`, `images` and `tools` groups. Known empty groups count as zero; unknown or malformed shapes omit the attribute. | | `firecrawl.formats` | `scrape`, `search`, `crawl`, `start_crawl` | Bounded, deduplicated canonical SDK format names from `formats=` or `scrape_options.formats`. Unknown names, prompts and schemas are omitted. Snake-case aliases use the SDK's API spelling. | | `firecrawl.job_id` | `crawl`, `start_crawl`, `get_crawl_status`, `cancel_crawl` | The crawl job id, positional or keyword. The start, status and cancel spans of one job share it. | diff --git a/python/frameworks/firecrawl/tests/test_contract_harness.py b/python/frameworks/firecrawl/tests/test_contract_harness.py index bd65e8fab..d54b45cdc 100644 --- a/python/frameworks/firecrawl/tests/test_contract_harness.py +++ b/python/frameworks/firecrawl/tests/test_contract_harness.py @@ -187,6 +187,7 @@ def test_real_client_calls_reach_the_collector_contract(fake_firecrawl, monkeypa search = _flatten_attributes(by_name["firecrawl.search"][0]["attributes"]) assert search["fi.retrieval.query"] == "open telemetry" + assert search["input.value"] == "open telemetry" assert int(search["fi.retrieval.document_count"]) == 2 ok_scrape, failed_scrape = by_name["firecrawl.scrape"] diff --git a/python/frameworks/firecrawl/tests/test_instrumentor.py b/python/frameworks/firecrawl/tests/test_instrumentor.py index 1c648f37f..c8368581d 100644 --- a/python/frameworks/firecrawl/tests/test_instrumentor.py +++ b/python/frameworks/firecrawl/tests/test_instrumentor.py @@ -95,5 +95,7 @@ def search(_self: FirecrawlClient, _query: str, **_kwargs: Any) -> Any: attrs = dict(spans[0].attributes or {}) assert spans[0].name == "firecrawl.search" assert attrs["fi.retrieval.query"] == "retrieval query" + # The Observe input panel reads input.value; the query must reach it too. + assert attrs["input.value"] == "retrieval query" assert attrs["fi.retrieval.document_count"] == 2 assert all(_API_KEY not in str(value) for value in attrs.values()) diff --git a/python/frameworks/firecrawl/tests/test_review_regressions.py b/python/frameworks/firecrawl/tests/test_review_regressions.py index ae7a09661..1ead0d33e 100644 --- a/python/frameworks/firecrawl/tests/test_review_regressions.py +++ b/python/frameworks/firecrawl/tests/test_review_regressions.py @@ -344,7 +344,9 @@ def test_f3_query_pii_redaction_precedes_cap(fake, telemetry, monkeypatch, use_a spans, wire = exported(telemetry) assert len(spans) == 1 expected = redact_pii_in_string(query)[:1024] - assert _flatten_attributes(spans[0]["attributes"])["fi.retrieval.query"] == expected + attributes = _flatten_attributes(spans[0]["attributes"]) + assert attributes["fi.retrieval.query"] == expected + assert attributes["input.value"] == expected for secret in ("person@example.test", "boundary.person", "415-555"): assert secret not in wire @@ -365,7 +367,9 @@ def test_f3_hidden_inputs_omit_search_query(fake, telemetry, monkeypatch, use_as assert len(result.web) == 2 spans, wire = exported(telemetry) assert len(spans) == 1 - assert "fi.retrieval.query" not in _flatten_attributes(spans[0]["attributes"]) + attributes = _flatten_attributes(spans[0]["attributes"]) + assert "fi.retrieval.query" not in attributes + assert attributes.get("input.value", "__REDACTED__") == "__REDACTED__" assert "QUERY-TO-HIDE-REVIEW" not in wire diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index a45dcfe67..e1ba81dde 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -18,6 +18,7 @@ _FI_SPAN_KIND = "fi.span.kind" _TOOL = "TOOL" +_INPUT_VALUE = "input.value" _RETRIEVAL_QUERY = "fi.retrieval.query" _RETRIEVAL_DOCUMENT_COUNT = "fi.retrieval.document_count" _JOB_ID = "firecrawl.job_id" @@ -253,6 +254,8 @@ def _attributes( query_text = _redact(str(query or ""), instance, self._config) if query_text is not None: attributes[_RETRIEVAL_QUERY] = query_text + # The Observe input panel reads input.value; keep both keys. + attributes[_INPUT_VALUE] = query_text elif self._method_name in ("scrape", "map", "crawl", "start_crawl"): url = kwargs.get("url", args[0] if args else "") host = _url_host(url, instance) From 0fc99e5b1038a90b908ae2a305ef9b00bee9d72c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 11 Oct 2026 20:29:18 -0700 Subject: [PATCH 20/22] docs(firecrawl): add the pip install line to the README Co-Authored-By: Claude Opus 5.5 --- python/frameworks/firecrawl/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/frameworks/firecrawl/README.md b/python/frameworks/firecrawl/README.md index cec2c78b3..8af851806 100644 --- a/python/frameworks/firecrawl/README.md +++ b/python/frameworks/firecrawl/README.md @@ -8,6 +8,12 @@ OpenTelemetry instrumentation for the Firecrawl v2 client. This package wraps the v2 client only. It does not wrap `FirecrawlApp`, and it does not import or disable OpenLIT. OpenLIT's `FireCrawlInstrumentor` wraps `FirecrawlApp.scrape_url` on the v1 client. A user who constructs both clients and enables both instrumentors gets two spans. Pick one. +## Installation + +```bash +pip install traceAI-firecrawl +``` + ## Supported versions `firecrawl-py >=4.46.2,<5` on Python 3.10, 3.11, 3.12 and 3.13. The tests run on `firecrawl-py==4.46.2`. Outside that range `instrument()` logs an error and wraps nothing. From 8c1289f2eaa2d0012e9c11e79e24eae90995aed9 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 11 Oct 2026 20:59:04 -0700 Subject: [PATCH 21/22] fix(firecrawl): align span keys with the tool tracer family - Record the search query in gen_ai.retrieval.query (the repo constant the backend reads) and the count in firecrawl.result_count instead of the unmapped fi.retrieval.* keys. - gen_ai.span.kind: RETRIEVER for search; scrape, map and the crawl calls stay TOOL. - Wrap only methods defined on each class (vars). - uninstrument() now also stops facades built while instrumented (shared enabled flag), so a Firecrawl() built before uninstrument() stops tracing. - Add a CHANGELOG. Co-Authored-By: Claude Opus 5.5 --- python/frameworks/firecrawl/CHANGELOG.md | 16 +++++++ python/frameworks/firecrawl/README.md | 8 ++-- .../firecrawl/tests/test_contract_harness.py | 9 ++-- .../firecrawl/tests/test_instrumentor.py | 7 +-- .../tests/test_review_regressions.py | 16 +++---- .../frameworks/firecrawl/tests/test_spans.py | 11 ++--- .../firecrawl/traceai_firecrawl/__init__.py | 20 +++++--- .../firecrawl/traceai_firecrawl/_wrappers.py | 47 +++++++++++++++---- 8 files changed, 94 insertions(+), 40 deletions(-) create mode 100644 python/frameworks/firecrawl/CHANGELOG.md diff --git a/python/frameworks/firecrawl/CHANGELOG.md b/python/frameworks/firecrawl/CHANGELOG.md new file mode 100644 index 000000000..43f307d8e --- /dev/null +++ b/python/frameworks/firecrawl/CHANGELOG.md @@ -0,0 +1,16 @@ +# Changelog + +All notable changes to `traceAI-firecrawl` will be documented here. + +## [0.1.0] - Unreleased + +### Added +- `FirecrawlInstrumentor` for the Firecrawl v2 sync and async clients (`firecrawl-py>=4.46.2,<5`): + `scrape`, `search`, `map`, `crawl`, `start_crawl`, `get_crawl_status` and `cancel_crawl`, one span per call. +- `gen_ai.span.kind` is `RETRIEVER` for `search` and `TOOL` for the other calls. +- `search` records the key-redacted query (capped at 1024 characters) in `gen_ai.retrieval.query` and + `input.value`, and the result count in `firecrawl.result_count`; other calls record only the target host. +- Crawl job id, status, page count and credits; a crawl job that ends `failed` or `cancelled` sets ERROR. +- Fixed-vocabulary error metadata: no raw SDK error text or traceback is recorded. +- `TraceConfig` support (`hide_inputs`, `hide_input_text`, PII redaction, `using_session` and friends). +- `uninstrument()` also stops facades built while instrumented. diff --git a/python/frameworks/firecrawl/README.md b/python/frameworks/firecrawl/README.md index 8af851806..1bb600049 100644 --- a/python/frameworks/firecrawl/README.md +++ b/python/frameworks/firecrawl/README.md @@ -36,13 +36,13 @@ client.scrape("https://example.com") ## What it records -Each call to `scrape`, `search`, `map`, `crawl`, `start_crawl`, `get_crawl_status` or `cancel_crawl`, sync or async, emits one span named `firecrawl.` with `fi.span.kind=TOOL`. A blocking `crawl()` is one span for the whole call: not one per page and not one per status poll. When the async `crawl()` calls `start_crawl()` internally, that inner call adds no span. The span is current while the SDK runs, so HTTP client spans nest under it. +Each call to `scrape`, `search`, `map`, `crawl`, `start_crawl`, `get_crawl_status` or `cancel_crawl`, sync or async, emits one span named `firecrawl.`. `gen_ai.span.kind` is `RETRIEVER` for `search` and `TOOL` for the others, which fetch the pages they are given. A blocking `crawl()` is one span for the whole call: not one per page and not one per status poll. When the async `crawl()` calls `start_crawl()` internally, that inner call adds no span. The span is current while the SDK runs, so HTTP client spans nest under it. | Attribute | Methods | Value | |---|---|---| | `server.address` | `scrape`, `map`, `crawl`, `start_crawl` | Host of the target URL. Never the path or query. | -| `fi.retrieval.query`, `input.value` | `search` | The query, with the API key redacted, capped at 1024 characters. Omitted with `hide_inputs` or `hide_input_text`; shared PII redaction applies when enabled. | -| `fi.retrieval.document_count` | `search` | Total returned records across SDK `web`, `news`, `images` and `tools` groups. Known empty groups count as zero; unknown or malformed shapes omit the attribute. | +| `gen_ai.retrieval.query`, `input.value` | `search` | The query, with the API key redacted, capped at 1024 characters. Omitted with `hide_inputs` or `hide_input_text`; shared PII redaction applies when enabled. | +| `firecrawl.result_count` | `search` | Total returned records across SDK `web`, `news`, `images` and `tools` groups. Known empty groups count as zero; unknown or malformed shapes omit the attribute. | | `firecrawl.formats` | `scrape`, `search`, `crawl`, `start_crawl` | Bounded, deduplicated canonical SDK format names from `formats=` or `scrape_options.formats`. Unknown names, prompts and schemas are omitted. Snake-case aliases use the SDK's API spelling. | | `firecrawl.job_id` | `crawl`, `start_crawl`, `get_crawl_status`, `cancel_crawl` | The crawl job id, positional or keyword. The start, status and cancel spans of one job share it. | | `firecrawl.limit` | `crawl`, `start_crawl` | The `limit` argument, when passed. | @@ -78,7 +78,7 @@ Pass `config=TraceConfig(...)` from `fi_instrumentation.instrumentation` to `ins `Firecrawl` and `AsyncFirecrawl` copy the v2 client's methods when they are constructed. Call `instrument()` before you construct them: - A client constructed before `instrument()` is not traced. -- A client constructed while instrumented keeps tracing after `uninstrument()`. Construct a new client to stop. +- A client constructed while instrumented stops tracing at `uninstrument()`. `FirecrawlClient` and `AsyncFirecrawlClient` used directly look the methods up on every call, so they follow `instrument()` and `uninstrument()` immediately. diff --git a/python/frameworks/firecrawl/tests/test_contract_harness.py b/python/frameworks/firecrawl/tests/test_contract_harness.py index d54b45cdc..5f77149e7 100644 --- a/python/frameworks/firecrawl/tests/test_contract_harness.py +++ b/python/frameworks/firecrawl/tests/test_contract_harness.py @@ -177,7 +177,10 @@ def test_real_client_calls_reach_the_collector_contract(fake_firecrawl, monkeypa by_name.setdefault(span["name"], []).append(span) for span in spans: - assert _flatten_attributes(span["attributes"])["fi.span.kind"] == "TOOL" + # search is a retrieval; scrape, map and crawl fetch pages they are given. + expected = "RETRIEVER" if span["name"] == "firecrawl.search" else "TOOL" + assert _flatten_attributes(span["attributes"])["gen_ai.span.kind"] == expected + assert "fi.span.kind" not in _flatten_attributes(span["attributes"]) crawl = _flatten_attributes(by_name["firecrawl.crawl"][0]["attributes"]) assert int(crawl["firecrawl.page_count"]) == 3 @@ -186,9 +189,9 @@ def test_real_client_calls_reach_the_collector_contract(fake_firecrawl, monkeypa assert crawl["server.address"] == "example.com" search = _flatten_attributes(by_name["firecrawl.search"][0]["attributes"]) - assert search["fi.retrieval.query"] == "open telemetry" + assert search["gen_ai.retrieval.query"] == "open telemetry" assert search["input.value"] == "open telemetry" - assert int(search["fi.retrieval.document_count"]) == 2 + assert int(search["firecrawl.result_count"]) == 2 ok_scrape, failed_scrape = by_name["firecrawl.scrape"] assert _flatten_attributes(ok_scrape["attributes"])["server.address"] == "example.com" diff --git a/python/frameworks/firecrawl/tests/test_instrumentor.py b/python/frameworks/firecrawl/tests/test_instrumentor.py index c8368581d..926fb3903 100644 --- a/python/frameworks/firecrawl/tests/test_instrumentor.py +++ b/python/frameworks/firecrawl/tests/test_instrumentor.py @@ -51,7 +51,7 @@ def crawl(_self: FirecrawlClient, _url: str, **_kwargs: Any) -> Any: span = spans[0] attrs = dict(span.attributes or {}) assert span.name == "firecrawl.crawl" - assert attrs["fi.span.kind"] == "TOOL" + assert attrs["gen_ai.span.kind"] == "TOOL" assert attrs["firecrawl.page_count"] == 3 assert attrs["firecrawl.limit"] == 10 assert attrs["firecrawl.job_id"] == "job-1" @@ -94,8 +94,9 @@ def search(_self: FirecrawlClient, _query: str, **_kwargs: Any) -> Any: assert len(spans) == 1 attrs = dict(spans[0].attributes or {}) assert spans[0].name == "firecrawl.search" - assert attrs["fi.retrieval.query"] == "retrieval query" + assert attrs["gen_ai.span.kind"] == "RETRIEVER" + assert attrs["gen_ai.retrieval.query"] == "retrieval query" # The Observe input panel reads input.value; the query must reach it too. assert attrs["input.value"] == "retrieval query" - assert attrs["fi.retrieval.document_count"] == 2 + assert attrs["firecrawl.result_count"] == 2 assert all(_API_KEY not in str(value) for value in attrs.values()) diff --git a/python/frameworks/firecrawl/tests/test_review_regressions.py b/python/frameworks/firecrawl/tests/test_review_regressions.py index 1ead0d33e..0a5af4eca 100644 --- a/python/frameworks/firecrawl/tests/test_review_regressions.py +++ b/python/frameworks/firecrawl/tests/test_review_regressions.py @@ -319,7 +319,7 @@ def test_f3_default_query_uses_shared_session_context(fake, telemetry, monkeypat spans, _ = exported(telemetry) assert len(spans) == 1 attributes = _flatten_attributes(spans[0]["attributes"]) - assert attributes["fi.retrieval.query"] == query + assert attributes["gen_ai.retrieval.query"] == query assert attributes[SpanAttributes.GEN_AI_CONVERSATION_ID] == "review-session" @@ -345,7 +345,7 @@ def test_f3_query_pii_redaction_precedes_cap(fake, telemetry, monkeypatch, use_a assert len(spans) == 1 expected = redact_pii_in_string(query)[:1024] attributes = _flatten_attributes(spans[0]["attributes"]) - assert attributes["fi.retrieval.query"] == expected + assert attributes["gen_ai.retrieval.query"] == expected assert attributes["input.value"] == expected for secret in ("person@example.test", "boundary.person", "415-555"): assert secret not in wire @@ -368,7 +368,7 @@ def test_f3_hidden_inputs_omit_search_query(fake, telemetry, monkeypatch, use_as spans, wire = exported(telemetry) assert len(spans) == 1 attributes = _flatten_attributes(spans[0]["attributes"]) - assert "fi.retrieval.query" not in attributes + assert "gen_ai.retrieval.query" not in attributes assert attributes.get("input.value", "__REDACTED__") == "__REDACTED__" assert "QUERY-TO-HIDE-REVIEW" not in wire @@ -429,8 +429,8 @@ async def journey(): assert len(result.web) == 2 spans, wire = exported(telemetry) attributes = _flatten_attributes(spans[0]["attributes"]) - assert "fi.retrieval.query" not in attributes - assert int(attributes["fi.retrieval.document_count"]) == 2 + assert "gen_ai.retrieval.query" not in attributes + assert int(attributes["firecrawl.result_count"]) == 2 assert format_names(spans[0]) == ["html"] assert KEY not in wire and HTML not in wire @@ -461,7 +461,7 @@ def test_f4_counts_all_sdk_search_groups(fake, telemetry, monkeypatch, use_async assert all(isinstance(item, DiscoveredTool) for item in result.tools) spans, wire = exported(telemetry) assert len(spans) == 1 - assert int(_flatten_attributes(spans[0]["attributes"])["fi.retrieval.document_count"]) == expected + assert int(_flatten_attributes(spans[0]["attributes"])["firecrawl.result_count"]) == expected assert HTML not in wire and PRIVATE_URL not in wire @@ -493,9 +493,9 @@ async def asearch(*args, **kwargs): spans, wire = exported(telemetry) attributes = _flatten_attributes(spans[0]["attributes"]) if expected is None: - assert "fi.retrieval.document_count" not in attributes + assert "firecrawl.result_count" not in attributes else: - assert int(attributes["fi.retrieval.document_count"]) == expected + assert int(attributes["firecrawl.result_count"]) == expected assert HTML not in wire diff --git a/python/frameworks/firecrawl/tests/test_spans.py b/python/frameworks/firecrawl/tests/test_spans.py index 6a8335585..a2085023c 100644 --- a/python/frameworks/firecrawl/tests/test_spans.py +++ b/python/frameworks/firecrawl/tests/test_spans.py @@ -248,7 +248,7 @@ async def journey() -> Any: assert document.markdown and async_document.markdown finished = spans(exporter) assert [span.name for span in finished] == ["firecrawl.scrape"] * 2 - assert all(attrs(span)["fi.span.kind"] == "TOOL" for span in finished) + assert all(attrs(span)["gen_ai.span.kind"] == "TOOL" for span in finished) class _ExplodingResult: @@ -540,7 +540,7 @@ async def journey() -> None: ] for span in finished: assert all(API_KEY not in str(value) for value in attrs(span).values()), span.name - assert attrs(finished[0])["fi.retrieval.query"] == "find [redacted]" + assert attrs(finished[0])["gen_ai.retrieval.query"] == "find [redacted]" assert attrs(finished[1])["firecrawl.job_id"] == "job-[redacted]" @@ -573,7 +573,7 @@ async def journey() -> Any: assert [span.name for span in finished] == ["firecrawl.map"] * 2 for span in finished: assert span.status.status_code is StatusCode.OK - assert attrs(span) == {"fi.span.kind": "TOOL", "server.address": "example.com"} + assert attrs(span) == {"gen_ai.span.kind": "TOOL", "server.address": "example.com"} assert "/private" not in str(attrs(span)) and RESULT_TITLE not in str(attrs(span)) @@ -641,8 +641,8 @@ def test_uninstrument_restores_methods_and_stops_new_spans(fake: FakeFirecrawl) def test_facade_methods_bind_at_construction(fake: FakeFirecrawl) -> None: """R4, documented in the README: Firecrawl/AsyncFirecrawl copy the v2 client's bound methods in __init__. Build the client after instrument(); a client built - before is not traced, and one built while instrumented keeps tracing after - uninstrument().""" + before is not traced. One built while instrumented stops tracing at + uninstrument(), like every other client.""" exporter = InMemorySpanExporter() provider = TracerProvider() provider.add_span_processor(SimpleSpanProcessor(exporter)) @@ -661,5 +661,4 @@ def test_facade_methods_bind_at_construction(fake: FakeFirecrawl) -> None: assert [attrs(span)["server.address"] for span in spans(exporter)] == [ "during.example.com", - "during-after-uninstrument.example.com", ] diff --git a/python/frameworks/firecrawl/traceai_firecrawl/__init__.py b/python/frameworks/firecrawl/traceai_firecrawl/__init__.py index fba738bd6..9b1ca9b1f 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/__init__.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/__init__.py @@ -9,7 +9,7 @@ from fi_instrumentation import FITracer from fi_instrumentation.instrumentation import TraceConfig -from traceai_firecrawl._wrappers import AsyncOperationWrapper, OperationWrapper +from traceai_firecrawl._wrappers import AsyncOperationWrapper, OperationWrapper, _State from traceai_firecrawl.package import _instruments from traceai_firecrawl.version import __version__ @@ -54,6 +54,7 @@ def _instrument(self, **kwargs: Any) -> None: tracer_provider = kwargs.get("tracer_provider") or trace_api.get_tracer_provider() tracer = FITracer(trace_api.get_tracer(__name__, __version__, tracer_provider), config=config) self._original_methods: Dict[Tuple[str, str, str], Any] = {} + self._state = _State() for module_name, class_name, is_async in ( (_SYNC_MODULE, _SYNC_CLASS, False), @@ -61,9 +62,9 @@ def _instrument(self, **kwargs: Any) -> None: ): for method_name, span_name in _OPERATIONS.items(): wrapper = ( - AsyncOperationWrapper(tracer, span_name, method_name, config) + AsyncOperationWrapper(tracer, span_name, method_name, config, self._state) if is_async - else OperationWrapper(tracer, span_name, method_name, config) + else OperationWrapper(tracer, span_name, method_name, config, self._state) ) self._wrap(module_name, class_name, method_name, wrapper) @@ -74,18 +75,23 @@ def _wrap(self, module_name: str, class_name: str, method_name: str, wrapper: An module = import_module(module_name) client = getattr(module, class_name) - if not hasattr(client, method_name): + # Only a method defined on this class itself, so an inherited one is + # never wrapped twice and uninstrument() restores exactly. + original = vars(client).get(method_name) + if original is None: return - self._original_methods[(module_name, class_name, method_name)] = getattr( - client, method_name - ) + self._original_methods[(module_name, class_name, method_name)] = original # wrapt 2.5.0 rejects keyword arguments, so these are positional. wrap_function_wrapper(module, "{0}.{1}".format(class_name, method_name), wrapper) def _uninstrument(self, **kwargs: Any) -> None: from importlib import import_module + state = getattr(self, "_state", None) + if state is not None: + # A facade built while instrumented holds bound wrappers; stop them. + state.enabled = False for (module_name, class_name, method_name), original in self._original_methods.items(): setattr(getattr(import_module(module_name), class_name), method_name, original) self._original_methods.clear() diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index e1ba81dde..90d5beb64 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -16,11 +16,12 @@ logger = logging.getLogger(__name__) -_FI_SPAN_KIND = "fi.span.kind" +_SPAN_KIND = "gen_ai.span.kind" _TOOL = "TOOL" +_RETRIEVER = "RETRIEVER" _INPUT_VALUE = "input.value" -_RETRIEVAL_QUERY = "fi.retrieval.query" -_RETRIEVAL_DOCUMENT_COUNT = "fi.retrieval.document_count" +_RETRIEVAL_QUERY = "gen_ai.retrieval.query" +_RESULT_COUNT = "firecrawl.result_count" _JOB_ID = "firecrawl.job_id" _LIMIT = "firecrawl.limit" _PAGE_COUNT = "firecrawl.page_count" @@ -75,6 +76,23 @@ _ACTIVE: ContextVar[bool] = ContextVar("traceai_firecrawl_active", default=False) +class _State: + """Shared by every wrapper of one instrument() call; uninstrument() disables it. + + The Firecrawl facade copies the v2 client's bound methods when it is + built; a facade built while instrumented must stop tracing once the + instrumentor is removed. + """ + + def __init__(self) -> None: + self.enabled = True + + +def _span_kind(method_name: str) -> str: + """search is a retrieval; scrape, map and the crawl calls fetch pages they are given.""" + return _RETRIEVER if method_name == "search" else _TOOL + + def _api_keys(instance: Any) -> list[str]: """API keys the client holds. firecrawl-py 4.46.2 keeps the key on ``http_client`` (sync and async), ``async_http_client`` (async) and @@ -238,16 +256,27 @@ class _BaseWrapper: that starts is ended exactly once. """ - def __init__(self, tracer: Tracer, span_name: str, method_name: str, config: TraceConfig) -> None: + def __init__( + self, + tracer: Tracer, + span_name: str, + method_name: str, + config: TraceConfig, + state: Optional[_State] = None, + ) -> None: self._tracer = tracer self._span_name = span_name self._method_name = method_name self._config = config + self._state = state or _State() + + def _untraced(self) -> bool: + return not self._state.enabled or _ACTIVE.get() def _attributes( self, instance: Any, args: Sequence[Any], kwargs: Mapping[str, Any] ) -> dict[str, Any]: - attributes = {_FI_SPAN_KIND: _TOOL} + attributes = {_SPAN_KIND: _span_kind(self._method_name)} if self._method_name == "search": if not (self._config.hide_inputs or self._config.hide_input_text): query = kwargs.get("query", args[0] if args else "") @@ -283,7 +312,7 @@ def _start( attributes = self._attributes(instance, args, kwargs) except Exception: logger.debug("Could not read %s arguments", self._span_name, exc_info=True) - attributes = {_FI_SPAN_KIND: _TOOL} + attributes = {_SPAN_KIND: _span_kind(self._method_name)} try: return self._tracer.start_span(self._span_name, attributes=attributes) except Exception: @@ -296,7 +325,7 @@ def _record_result( if self._method_name == "search": count = _document_count(result) if count is not None: - span.set_attribute(_RETRIEVAL_DOCUMENT_COUNT, count) + span.set_attribute(_RESULT_COUNT, count) if self._method_name in _CRAWL_METHODS: job_id = _job_id(result, kwargs) if job_id: @@ -395,7 +424,7 @@ def __call__( args: tuple, kwargs: Mapping[str, Any], ) -> Any: - if _ACTIVE.get(): + if self._untraced(): return wrapped(*args, **kwargs) span = self._start(instance, args, kwargs) if span is None: @@ -423,7 +452,7 @@ async def __call__( args: tuple, kwargs: Mapping[str, Any], ) -> Any: - if _ACTIVE.get(): + if self._untraced(): return await wrapped(*args, **kwargs) span = self._start(instance, args, kwargs) if span is None: From 4c9f68e69f738a0b299c4696e189ac4642003c0b Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 17 Oct 2026 03:42:39 -0700 Subject: [PATCH 22/22] docs(firecrawl): live vendor verification 2026-10-17 Co-Authored-By: Claude Sonnet 5.5 --- python/frameworks/firecrawl/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/frameworks/firecrawl/README.md b/python/frameworks/firecrawl/README.md index 1bb600049..5107a854e 100644 --- a/python/frameworks/firecrawl/README.md +++ b/python/frameworks/firecrawl/README.md @@ -85,3 +85,7 @@ Pass `config=TraceConfig(...)` from `fi_instrumentation.instrumentation` to `ins ## Watchers `Firecrawl.watcher()` returns a `Watcher` that polls `get_crawl_status` from its own thread. Each poll is a separate root `firecrawl.get_crawl_status` span with no parent, and a poll error the watcher swallows still shows as an ERROR span. `AsyncFirecrawl.watcher()` polls in the task that iterates it, so its poll spans are children of the span current there. + +## Live vendor verification (2026-10-17) + +Run as documented above against the real `api.firecrawl.dev` with `firecrawl-py` 4.50.0 and `fi-instrumentation-otel` 1.1.0 on Python 3.12, exporting to Future AGI. `scrape`, `search`, `map`, `start_crawl`, `get_crawl_status`, `cancel_crawl` and a blocking `crawl` each produced one span under an AGENT parent (`fi_verify` PASS, trace `fc0dbe7b-cf70-494e-a5ed-d3b06b91e4df`). A wrong key ended the span ERROR `UnauthorizedError` (HTTP 401), and a free-plan HTTP 429 ended `start_crawl` ERROR `RateLimitError` (trace `0e189c43-63d1-4420-b7c1-42bdb92dbca5`). A scan of every stored column found no page content, URL path or query string, or API key; `FI_HIDE_INPUTS=true` removed the search query. The async client, `watcher()` and failed or cancelled job statuses were not run live and are covered by the test suite only.