diff --git a/.github/release/complete_release.py b/.github/release/complete_release.py index eeaa90d4d..5ae123d21 100644 --- a/.github/release/complete_release.py +++ b/.github/release/complete_release.py @@ -49,7 +49,9 @@ def unreleased(packages, tags, skip): for package in packages: if package["name"] in skip: continue - versions = published_versions(package) + # A historical Java reactor is an atomic Central deployment. New modules in today's + # inventory did not exist at the release version and cannot be uploaded retroactively. + versions = published_versions(package, strict=False) if package["ecosystem"] == "java" else published_versions(package) if versions is None or package["version"] not in versions: return f"{package['name']} {package['version']} is not served by its registry" return None @@ -76,8 +78,18 @@ def main(): if changed is None: print(f"Still pending: release PR #{pr['number']}: could not read the versions it released") continue - required = [with_version(p, changed[path]) for p in packages if (path := p.get("release_path", p["dir"])) in changed] - reason = unreleased(required, tags, skip) + known = {p.get("release_path", p["dir"]): p for p in packages} + unknown = changed.keys() - known.keys() + if unknown: + print(f"Still pending: release PR #{pr['number']}: release paths missing from current inventory: {', '.join(sorted(unknown))}") + continue + required = [with_version(known[path], version) for path, version in changed.items()] + try: + reason = unreleased(required, tags, skip) + except Exception as exc: + # A registry outage must not acknowledge this PR or block an unrelated one. + print(f"Still pending: release PR #{pr['number']}: registry check failed ({type(exc).__name__})") + continue if reason: print(f"Still pending: release PR #{pr['number']}: {reason}") else: diff --git a/.github/release/release_plan.py b/.github/release/release_plan.py index e4c2199cb..ddd8ddfaa 100644 --- a/.github/release/release_plan.py +++ b/.github/release/release_plan.py @@ -221,7 +221,7 @@ def published_versions(p: dict, *, strict: bool = True) -> set[str] | None: eco = p["ecosystem"] if eco == "python": d = _get_json(f"https://pypi.org/pypi/{p['name']}/json") - return set(d["releases"]) if d else None + return {version for version, files in d["releases"].items() if files} if d else None if eco == "typescript": d = _get_json("https://registry.npmjs.org/" + p["name"].replace("/", "%2F")) return set(d.get("versions", {})) if d else None @@ -251,11 +251,10 @@ def on_central(artifact): parent, *modules = p["artifacts"] # other_packages lists the reactor parent first if not on_central(parent): return None - later = [a for a in modules if not on_central(a)] - if later and strict: + if not strict: + return {p["version"]} + if any(not on_central(a) for a in modules): return None - if later: - print(f"{p['group']}:{parent} {p['version']} is published; {', '.join(later)} ship with the next reactor version", file=sys.stderr) return {p["version"]} raise ValueError(f"Unknown ecosystem: {eco}") diff --git a/.github/release/test_release_followup.py b/.github/release/test_release_followup.py new file mode 100644 index 000000000..17e7f0a90 --- /dev/null +++ b/.github/release/test_release_followup.py @@ -0,0 +1,82 @@ +"""Regression coverage for historical release completion and Java reconciliation.""" +import os +import sys +import unittest +import urllib.error +from pathlib import Path +from unittest.mock import Mock, patch + +sys.path.insert(0, str(Path(__file__).parent)) +import complete_release +import release_plan + + +class ReleaseFollowupTests(unittest.TestCase): + def test_pypi_empty_release_has_no_distribution(self): + p = {'ecosystem': 'python', 'name': 'traceai-test'} + with patch.object(release_plan, '_get_json', return_value={'releases': {'1.0.0': [], '1.0.1': [{'filename': 'a.whl'}]}}): + self.assertEqual(release_plan.published_versions(p), {'1.0.1'}) + + def test_java_parent_only_does_not_probe_modules(self): + p = {'ecosystem': 'java', 'group': 'com.futureagi', 'version': '1.0.0', 'artifacts': ['parent', 'later']} + response = Mock(); response.__enter__ = Mock(return_value=response); response.__exit__ = Mock(return_value=False) + with patch.object(release_plan.urllib.request, 'urlopen', side_effect=[response, TimeoutError('later module unavailable')]) as get: + self.assertEqual(release_plan.published_versions(p, strict=False), {'1.0.0'}) + self.assertEqual(get.call_count, 1) + + def test_historical_java_release_requires_parent_not_new_module(self): + p = {'ecosystem': 'java', 'name': 'com.futureagi:parent', 'group': 'com.futureagi', + 'version': '2.0.0', 'tag': 'java-v2.0.0', 'dir': 'java', 'artifacts': ['parent', 'new-module']} + pending = [{'number': 7, 'headRefName': 'release-please--branches--main', 'mergeCommit': {'oid': 'm'}}] + response = Mock(); response.__enter__ = Mock(return_value=response); response.__exit__ = Mock(return_value=False) + with patch.dict(os.environ, {'GITHUB_REPOSITORY': 'owner/repo'}), \ + patch.object(complete_release, 'gh_json', side_effect=[pending, [[{'tag_name': 'java-v1.0.0'}]]]), \ + patch.object(complete_release, 'changed_release_paths', return_value={'java': '1.0.0'}), \ + patch.object(complete_release, 'python_packages', return_value=[]), \ + patch.object(complete_release, 'typescript_packages', return_value=[]), \ + patch.object(complete_release, 'other_packages', return_value=[p]), \ + patch.object(complete_release, 'load_config', return_value={'skip': {}}), \ + patch.object(release_plan.urllib.request, 'urlopen', side_effect=[response, TimeoutError('new module does not exist')]) as get, \ + patch('subprocess.run') as run: + complete_release.main() + self.assertEqual(get.call_count, 1) + self.assertTrue(any(c.args[0][:3] == ['gh', 'pr', 'edit'] for c in run.call_args_list)) + + def test_unknown_historical_path_remains_pending(self): + p = {'ecosystem': 'python', 'name': 'a', 'version': '2.0.0', 'tag': 'a-v2.0.0', 'dir': 'python/a'} + pending = [{'number': 7, 'headRefName': 'release-please--branches--main', 'mergeCommit': {'oid': 'm'}}] + with patch.dict(os.environ, {'GITHUB_REPOSITORY': 'owner/repo'}), \ + patch.object(complete_release, 'gh_json', side_effect=[pending, [[]]]), \ + patch.object(complete_release, 'changed_release_paths', return_value={'python/removed': '1.0.0'}), \ + patch.object(complete_release, 'python_packages', return_value=[p]), \ + patch.object(complete_release, 'typescript_packages', return_value=[]), \ + patch.object(complete_release, 'other_packages', return_value=[]), \ + patch.object(complete_release, 'load_config', return_value={'skip': {}}), \ + patch('subprocess.run') as run: + complete_release.main() + run.assert_not_called() + + def test_one_registry_failure_does_not_block_other_pending_pr(self): + packages = [{'ecosystem': 'python', 'name': name, 'version': '1.0.0', 'tag': f'{name}-v1.0.0', 'dir': f'python/{name}'} for name in ('a', 'b')] + pending = [{'number': number, 'headRefName': 'release-please--branches--main', 'mergeCommit': {'oid': str(number)}} for number in (7, 8)] + error = urllib.error.HTTPError('https://pypi.org', 503, 'unavailable', {}, None) + def published(p, **kwargs): + if p['name'] == 'a': + raise error + return {'1.0.0'} + with patch.dict(os.environ, {'GITHUB_REPOSITORY': 'owner/repo'}), \ + patch.object(complete_release, 'gh_json', side_effect=[pending, [[{'tag_name': p['tag']} for p in packages]]]), \ + patch.object(complete_release, 'changed_release_paths', side_effect=lambda repo, pr: {f"python/{'a' if pr['number'] == 7 else 'b'}": '1.0.0'}), \ + patch.object(complete_release, 'python_packages', return_value=packages), \ + patch.object(complete_release, 'typescript_packages', return_value=[]), \ + patch.object(complete_release, 'other_packages', return_value=[]), \ + patch.object(complete_release, 'load_config', return_value={'skip': {}}), \ + patch.object(complete_release, 'published_versions', side_effect=published), \ + patch('subprocess.run') as run: + complete_release.main() + edits = [c.args[0][3] for c in run.call_args_list if c.args[0][:3] == ['gh', 'pr', 'edit']] + self.assertEqual(edits, ['8']) + + def test_java_reconciliation_polls_parent_after_skipping_deploy(self): + workflow = (Path(__file__).parents[1] / 'workflows/release-java.yml').read_text() + self.assertIn("strict=not p['registry_published']", workflow) diff --git a/.github/workflows/firecrawl-tests.yml b/.github/workflows/firecrawl-tests.yml new file mode 100644 index 000000000..f67d5d526 --- /dev/null +++ b/.github/workflows/firecrawl-tests.yml @@ -0,0 +1,80 @@ +name: Firecrawl compatibility + +on: + pull_request: + branches: [main, dev] + paths: + - 'python/frameworks/firecrawl/**' + - '.github/workflows/firecrawl-tests.yml' + push: + branches: [dev] + paths: + - 'python/frameworks/firecrawl/**' + - '.github/workflows/firecrawl-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +defaults: + run: + shell: bash + +env: + PYTHON_DOTENV_DISABLED: '1' + FI_API_KEY: synthetic-fi-key + FI_SECRET_KEY: synthetic-fi-secret + FI_BASE_URL: http://127.0.0.1:9 + FI_GRPC_URL: http://127.0.0.1:9 + +jobs: + compatibility: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - python: '3.10' + firecrawl: '4.46.2' + distribution: wheel + - python: '3.10' + firecrawl: latest + distribution: sdist + - python: '3.13' + firecrawl: '4.46.2' + distribution: sdist + - python: '3.13' + firecrawl: latest + distribution: wheel + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Build and install a distribution + env: + DISTRIBUTION: ${{ matrix.distribution }} + FIRECRAWL_VERSION: ${{ matrix.firecrawl }} + run: | + python -m pip install 'build>=1.2,<2' poetry-core + python -m build python/frameworks/firecrawl --outdir "$RUNNER_TEMP/dist" + if [ "$DISTRIBUTION" = wheel ]; then + artifact=$(find "$RUNNER_TEMP/dist" -name '*.whl' -print -quit) + else + artifact=$(find "$RUNNER_TEMP/dist" -name '*.tar.gz' -print -quit) + fi + if [ "$FIRECRAWL_VERSION" = latest ]; then + upstream='firecrawl-py>=4.46.2,<5' + else + upstream="firecrawl-py==$FIRECRAWL_VERSION" + fi + python -m pip install "$artifact" "$upstream" 'pytest>=8,<9' pytest-asyncio + - name: Run installed package tests with only loopback network + run: | + sudo iptables -I OUTPUT 1 ! -o lo -j REJECT + sudo ip6tables -I OUTPUT 1 ! -o lo -j REJECT + trap 'sudo iptables -D OUTPUT ! -o lo -j REJECT; sudo ip6tables -D OUTPUT ! -o lo -j REJECT' EXIT + python -I -m pytest --noconftest python/frameworks/firecrawl/tests -q diff --git a/.github/workflows/release-java.yml b/.github/workflows/release-java.yml index 33273486f..aa6253a26 100644 --- a/.github/workflows/release-java.yml +++ b/.github/workflows/release-java.yml @@ -53,7 +53,7 @@ jobs: MAVEN_GPG_PRIVATE_KEY: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} run: mvn -B -ntp -P release deploy - - name: Verify every reactor artifact on Central + - name: Verify reactor on Central if: inputs.publish env: PACKAGE: ${{ toJson(matrix) }} @@ -64,7 +64,9 @@ jobs: plan = importlib.util.module_from_spec(spec); spec.loader.exec_module(plan) p = json.loads(os.environ['PACKAGE']); p['ecosystem'] = 'java' for attempt in range(60): - if plan.published_versions(p): break + # A reconciliation has no deploy: its parent proves the old reactor bundle shipped. + # After a fresh deploy, verify every module in the current reactor. + if plan.published_versions(p, strict=not p['registry_published']): break time.sleep(30) else: raise SystemExit('Central has not served every reactor artifact after 30 minutes; rerun after checking the deployment portal') PY diff --git a/python/frameworks/firecrawl/README.md b/python/frameworks/firecrawl/README.md index 5107a854e..03d710941 100644 --- a/python/frameworks/firecrawl/README.md +++ b/python/frameworks/firecrawl/README.md @@ -1,26 +1,28 @@ # traceAI-firecrawl -OpenTelemetry instrumentation for the Firecrawl v2 client. +Trace Firecrawl v2 scraping, extraction, agent and browser operations in Future AGI without exporting page bodies, prompts or browser code. -## v1 and v2 are different clients - -`firecrawl-py` 4.x ships two clients. The v2 client is `from firecrawl import Firecrawl` (and `AsyncFirecrawl`), and it delegates `scrape`, `search`, `map`, and the crawl methods to `firecrawl.v2.client.FirecrawlClient` (and `firecrawl.v2.client_async.AsyncFirecrawlClient`). The v1 client is `FirecrawlApp`, and it is feature-frozen. - -This package wraps the v2 client only. It does not wrap `FirecrawlApp`, and it does not import or disable OpenLIT. OpenLIT's `FireCrawlInstrumentor` wraps `FirecrawlApp.scrape_url` on the v1 client. A user who constructs both clients and enables both instrumentors gets two spans. Pick one. - -## Installation +## Install ```bash pip install traceAI-firecrawl ``` -## Supported versions +Supports `firecrawl-py >=4.46.2,<5` on Python 3.10–3.13. Use `Firecrawl` or `AsyncFirecrawl`, not the feature-frozen v1 `FirecrawlApp`. Outside the supported SDK range, `instrument()` logs an error and wraps nothing. + +## Configure and run -`firecrawl-py >=4.46.2,<5` on Python 3.10, 3.11, 3.12 and 3.13. The tests run on `firecrawl-py==4.46.2`. Outside that range `instrument()` logs an error and wraps nothing. +Set your Future AGI credentials and Firecrawl API key in your environment: -## Usage +```bash +export FI_API_KEY="your-futureagi-api-key" +export FI_SECRET_KEY="your-futureagi-secret-key" +export FIRECRAWL_API_KEY="your-firecrawl-api-key" +``` ```python +import os + from firecrawl import Firecrawl from fi_instrumentation import register from fi_instrumentation.fi_types import ProjectType @@ -29,63 +31,86 @@ from traceai_firecrawl import FirecrawlInstrumentor tracer_provider = register(project_name="firecrawl", project_type=ProjectType.OBSERVE) FirecrawlInstrumentor().instrument(tracer_provider=tracer_provider) -# Construct the client after instrument(); see "Client construction order". -client = Firecrawl(api_key="your-firecrawl-api-key") +# Construct facade clients after instrumentation. +client = Firecrawl(api_key=os.environ["FIRECRAWL_API_KEY"]) client.scrape("https://example.com") +client.batch_scrape(["https://example.com"], formats=["markdown"]) + +# Flush before a short-lived script exits. +tracer_provider.force_flush() ``` -## What it records +Open the `firecrawl` project in Future AGI's **Observe** view. You will see `firecrawl.scrape` and `firecrawl.batch_scrape` spans with duration, status and available job metadata. Use the same instrumentation setup with `AsyncFirecrawl` and await its operations. -Each call to `scrape`, `search`, `map`, `crawl`, `start_crawl`, `get_crawl_status` or `cancel_crawl`, sync or async, emits one span named `firecrawl.`. `gen_ai.span.kind` is `RETRIEVER` for `search` and `TOOL` for the others, which fetch the pages they are given. A blocking `crawl()` is one span for the whole call: not one per page and not one per status poll. When the async `crawl()` calls `start_crawl()` internally, that inner call adds no span. The span is current while the SDK runs, so HTTP client spans nest under it. +## Supported operations -| Attribute | Methods | Value | -|---|---|---| -| `server.address` | `scrape`, `map`, `crawl`, `start_crawl` | Host of the target URL. Never the path or query. | -| `gen_ai.retrieval.query`, `input.value` | `search` | The query, with the API key redacted, capped at 1024 characters. Omitted with `hide_inputs` or `hide_input_text`; shared PII redaction applies when enabled. | -| `firecrawl.result_count` | `search` | Total returned records across SDK `web`, `news`, `images` and `tools` groups. Known empty groups count as zero; unknown or malformed shapes omit the attribute. | -| `firecrawl.formats` | `scrape`, `search`, `crawl`, `start_crawl` | Bounded, deduplicated canonical SDK format names from `formats=` or `scrape_options.formats`. Unknown names, prompts and schemas are omitted. Snake-case aliases use the SDK's API spelling. | -| `firecrawl.job_id` | `crawl`, `start_crawl`, `get_crawl_status`, `cancel_crawl` | The crawl job id, positional or keyword. The start, status and cancel spans of one job share it. | -| `firecrawl.limit` | `crawl`, `start_crawl` | The `limit` argument, when passed. | -| `firecrawl.page_count` | `crawl` | Pages the job completed. | -| `firecrawl.status` | `crawl`, `get_crawl_status` | Job status from the response (`scraping`, `completed`, `failed`, `cancelled`). | -| `firecrawl.cancelled` | see "Status" | `true` when the job or the call was cancelled. | -| `firecrawl.credits_used` | any method whose result carries it | Credits the response reports. Not converted to dollars. | -| `firecrawl.error.status_code`, `firecrawl.error.code` | any, on a Firecrawl API error | HTTP status and the vendor's machine-readable error code, when present. | +| API | Methods | +|---|---| +| Scrape, search and map | `scrape`, `search`, `map` | +| Crawl | `crawl`, `start_crawl`, `get_crawl_status`, `cancel_crawl` | +| Extract | `extract`, `start_extract`, `get_extract_status` | +| Batch scrape | `batch_scrape`, `start_batch_scrape`, `get_batch_scrape_status`, `get_batch_scrape_status_page`, `get_batch_scrape_errors`, `cancel_batch_scrape` | +| Agent | `agent`, `start_agent`, `get_agent_status`, `cancel_agent` | +| Browser | `browser`, `browser_execute`, `delete_browser`, `list_browsers` | -`firecrawl.credits_used` caveat: for crawl jobs, firecrawl-py fills `credits_used` with `0` when the API response omits `creditsUsed`, so a recorded `0` on `crawl` or `get_crawl_status` can mean either "no credits" or "not reported". +These methods are covered on both sync and async clients. The direct `AsyncFirecrawlClient.wait_batch_scrape` method is also covered. Firecrawl marks its extract endpoint as maintenance-only; existing extract calls remain traceable. For new applications, follow [Firecrawl's extractor selection guide](https://docs.firecrawl.dev/developer-guides/usage-guides/choosing-the-data-extractor). -Format metadata follows SDK 4.46.2's payload: `scrape_options` takes precedence over convenience `formats`. A `ScrapeFormats` container contributes its explicit list and serialized enabled flags, including its default `markdown=true`. The SDK does not serialize its `images` or `json` boolean flags, so those flags add no metadata; explicit format entries still do. +Each user call emits one span named `firecrawl.`. Blocking calls include the whole wait, not a span per page or internal poll. The span is current while the SDK runs, so separately enabled HTTP client instrumentation nests beneath it. The package does not trace remote agent steps, browser activity between API calls, or other SDK APIs outside this table. -Search counts include `tools` because SDK 4.46.2 returns a list of `DiscoveredTool` discovery records, alongside web/news/image records. The count includes those records, not pages fetched by a tool. The prior generic `data`/`results`/`web` list fallback remains supported; no result contents are exported. +## Span metadata -### Status +`gen_ai.span.kind` is `RETRIEVER` for `search` and `TOOL` for the other operations. -- The SDK returns a failed or cancelled crawl job without raising. On a blocking `crawl()`, a job with status `failed` sets the span to ERROR `failed`, and a job with status `cancelled` sets ERROR `cancelled` and `firecrawl.cancelled=true`. A `get_crawl_status` poll that gets an answer stays OK and records the job's state in `firecrawl.status`. -- `cancel_crawl` sets `firecrawl.cancelled` to the API's answer; the span itself is OK. -- Cancelling an asyncio task that is awaiting a Firecrawl call ends the span ERROR `cancelled` with `firecrawl.cancelled=true`. -- Any exception the SDK raises adds one safe exception event and sets ERROR with a recognized exception type (otherwise `Exception`). Exception free text and serialized tracebacks are omitted; the original exception is re-raised unchanged. +| Attribute | Value | +|---|---| +| `server.address` | Target URL host for `scrape`, `map`, `crawl` and `start_crawl`. No path or query. URL lists are not exported. | +| `gen_ai.retrieval.query`, `input.value` | Search query, API-key redacted and capped at 1024 characters. Omitted with `hide_inputs` or `hide_input_text`. | +| `firecrawl.result_count` | Search records across SDK `web`, `news`, `images` and `tools` groups. Known empty groups count as zero; unknown shapes omit the attribute. | +| `firecrawl.formats` | Bounded, deduplicated format names for scrape, search, crawl, batch and extract inputs. No prompts or schemas. | +| `firecrawl.job_id` | Job ID supplied to a lifecycle call or returned by the SDK. Start, status and cancel calls can be correlated using this value. | +| `firecrawl.session_id` | Browser session ID returned by `browser` or supplied to `browser_execute` and `delete_browser`. Separate from the application tracing `session.id`. | +| `firecrawl.limit` | Crawl `limit`, when passed. | +| `firecrawl.page_count` | Completed pages for `crawl`, `batch_scrape` and `wait_batch_scrape`. | +| `firecrawl.status` | Recognized job state: `scraping`, `processing`, `completed`, `failed` or `cancelled`. | +| `firecrawl.cancelled` | Cancellation result for cancel APIs, or `true` for a cancelled blocking job or asyncio call. | +| `firecrawl.credits_used` | Credits reported by the SDK, including `credits_billed` on browser deletion. Not converted to dollars. | +| `firecrawl.error.status_code`, `firecrawl.error.code` | HTTP status and recognized machine-readable API error code, when present. | -## What it never records +Job IDs are omitted when neither the caller nor the SDK result provides one. In particular, the SDK's blocking batch result and pagination-page result do not contain an ID. To correlate separate batch operations, use `start_batch_scrape` followed by status and cancel calls with its ID. -This package has no content-capture setting. It never records page content (markdown, HTML, screenshots, links, titles, descriptions), URL paths or query strings, JSON-format prompts or schemas, or the Firecrawl API key. The key is redacted from every attribute the package sets, including a key taken from `FIRECRAWL_API_KEY`. The search query is recorded by design. Error events contain a safe type and generic message; status descriptions contain that type or `cancelled`. Only HTTP status and recognized machine codes are retained from API errors; unknown codes are omitted. +Credits are recorded only when available on the SDK result. The SDK defaults some crawl/batch credit fields to zero, so zero may mean unreported usage. Its async extract implementation can omit the API's `creditsUsed` field; this integration does not infer missing credits. -Instrumentation errors never reach your code: if reading an argument or a result fails, the call still returns the SDK's result or raises the SDK's exception, and the span still ends. +Format metadata follows SDK serialization: `scrape_options` takes precedence over convenience `formats`. A `ScrapeFormats` container contributes its explicit list and serialized enabled flags, including its default `markdown=true`. The SDK does not serialize its `images` or `json` boolean flags; explicit format entries still contribute metadata. Search counts include tool discovery records, not pages fetched by those tools. -Pass `config=TraceConfig(...)` from `fi_instrumentation.instrumentation` to `instrument()` to use shared privacy controls. `None` creates `TraceConfig()` using its `FI_*` environment settings; other objects raise `TypeError`. `hide_inputs` or `hide_input_text` omits the search query. `pii_redaction` (or `FI_PII_REDACTION=true`) applies the shared regex redactor to query text before truncation, followed by FITracer attribute masking. Other message, image and output hide flags are not applicable because no messages or page bodies are exported. Shared `suppress_tracing()` emits no Firecrawl spans, and `using_session()` supplies the shared session attribute. If API-key discovery fails, free-text attributes are omitted while safe format/count metadata remains. Error events and status use the separate safe metadata policy described above. +### Status and errors -## Client construction order +- Blocking crawl, batch, extract and agent calls that return a failed or cancelled job end with ERROR. A successful status poll stays OK and records the job state; a rejected request is an error. +- Cancel APIs record their Boolean result in `firecrawl.cancelled`; the API call itself stays OK. +- Extract, agent and browser responses with `success=false` end with ERROR `failed`. A browser execution with a nonzero exit code ends with ERROR `execution_failed`. +- Cancelling an asyncio task ends its span with ERROR `cancelled` and `firecrawl.cancelled=true`. +- SDK exceptions add one safe exception event and end with ERROR. Exception text and tracebacks are omitted; your code receives the original exception unchanged. -`Firecrawl` and `AsyncFirecrawl` copy the v2 client's methods when they are constructed. Call `instrument()` before you construct them: +## Privacy -- A client constructed before `instrument()` is not traced. -- A client constructed while instrumented stops tracing at `uninstrument()`. +This package has no content-capture setting. It does not export page bodies, screenshots, titles, descriptions, extracted data, agent prompts or results, schemas, browser code or output, CDP/live-view URLs, request URL paths or query strings, or the Firecrawl API key. Search query text is the deliberate exception. The API key is redacted from free-text metadata, including keys resolved from `FIRECRAWL_API_KEY`. -`FirecrawlClient` and `AsyncFirecrawlClient` used directly look the methods up on every call, so they follow `instrument()` and `uninstrument()` immediately. +Pass `config=TraceConfig(...)` from `fi_instrumentation.instrumentation` to `instrument()` for shared privacy controls: -## Watchers +- `hide_inputs` or `hide_input_text` omits search query text. +- `pii_redaction` (or `FI_PII_REDACTION=true`) redacts PII in query text before truncation. +- `None` uses the shared `FI_*` environment settings. Other config types raise `TypeError`. + +Message, image and output hide flags do not add content capture. Shared `suppress_tracing()` suppresses Firecrawl spans; `using_session()` supplies the application session attribute. If API-key discovery fails, free-text metadata is omitted. Instrumentation metadata errors do not replace the SDK's result or exception. + +## Client construction order + +`Firecrawl` and `AsyncFirecrawl` copy the v2 client's bound methods during construction. Call `instrument()` first: -`Firecrawl.watcher()` returns a `Watcher` that polls `get_crawl_status` from its own thread. Each poll is a separate root `firecrawl.get_crawl_status` span with no parent, and a poll error the watcher swallows still shows as an ERROR span. `AsyncFirecrawl.watcher()` polls in the task that iterates it, so its poll spans are children of the span current there. +- A facade client constructed before instrumentation is not traced; construct a new one afterward. +- A facade client constructed while instrumented stops tracing at `uninstrument()`. -## Live vendor verification (2026-10-17) +Direct `FirecrawlClient` and `AsyncFirecrawlClient` instances look methods up on each call and follow instrumentation changes immediately. This package does not instrument v1 `FirecrawlApp` or enable or disable other vendors' instrumentors. + +## Watchers -Run as documented above against the real `api.firecrawl.dev` with `firecrawl-py` 4.50.0 and `fi-instrumentation-otel` 1.1.0 on Python 3.12, exporting to Future AGI. `scrape`, `search`, `map`, `start_crawl`, `get_crawl_status`, `cancel_crawl` and a blocking `crawl` each produced one span under an AGENT parent (`fi_verify` PASS, trace `fc0dbe7b-cf70-494e-a5ed-d3b06b91e4df`). A wrong key ended the span ERROR `UnauthorizedError` (HTTP 401), and a free-plan HTTP 429 ended `start_crawl` ERROR `RateLimitError` (trace `0e189c43-63d1-4420-b7c1-42bdb92dbca5`). A scan of every stored column found no page content, URL path or query string, or API key; `FI_HIDE_INPUTS=true` removed the search query. The async client, `watcher()` and failed or cancelled job statuses were not run live and are covered by the test suite only. +`Firecrawl.watcher()` polls `get_crawl_status` from its own thread. Each poll produces a separate root span; a swallowed poll error still appears as ERROR. `AsyncFirecrawl.watcher()` polls in the iterating task, so its spans inherit that task's current parent. diff --git a/python/frameworks/firecrawl/tests/test_extended_operations.py b/python/frameworks/firecrawl/tests/test_extended_operations.py new file mode 100644 index 000000000..97975e075 --- /dev/null +++ b/python/frameworks/firecrawl/tests/test_extended_operations.py @@ -0,0 +1,420 @@ +"""Real v2 SDK operations against a loopback API; no vendor credentials needed.""" + +from __future__ import annotations + +import asyncio +from contextlib import contextmanager + +import pytest +from firecrawl import AsyncFirecrawl, Firecrawl +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from _firecrawl_fake import FakeFirecrawl, PAGE_BODY, crawl_status +from traceai_firecrawl import FirecrawlInstrumentor + +API_KEY = "fc-synthetic-private-key" +PRIVATE = "PRIVATE-CONTENT-MUST-NOT-LEAVE" +URLS = [f"https://example.com/{PRIVATE}?key={API_KEY}"] + + +@pytest.fixture() +def api(): + server = FakeFirecrawl() + for family, job in (("extract", "extract-1"), ("agent", "agent-1")): + server.routes[("POST", f"/v2/{family}")] = (200, { + "success": True, "id": job, "status": "processing", + }) + server.routes[("GET", f"/v2/{family}/{job}")] = (200, { + "success": True, "id": job, "status": "completed", "creditsUsed": 2, + "data": {"secret": PRIVATE}, "error": PRIVATE, + }) + server.routes.update({ + ("DELETE", "/v2/agent/agent-1"): (200, {"success": True}), + ("POST", "/v2/batch/scrape"): (200, { + "success": True, "id": "batch-1", "url": URLS[0], + }), + ("GET", "/v2/batch/scrape/batch-1"): (200, crawl_status("completed", pages=2)), + ("GET", "/v2/batch/scrape/batch-1/errors"): (200, { + "success": True, "errors": [], "robotsBlocked": URLS, + }), + ("DELETE", "/v2/batch/scrape/batch-1"): (200, { + "success": True, "status": "cancelled", + }), + ("POST", "/v2/browser"): (200, { + "success": True, "id": "browser-1", "cdpUrl": URLS[0], + "liveViewUrl": URLS[0], "interactiveLiveViewUrl": URLS[0], + }), + ("POST", "/v2/browser/browser-1/execute"): (200, { + "success": True, "stdout": PRIVATE, "output": PRIVATE, "exitCode": 0, + }), + ("DELETE", "/v2/browser/browser-1"): (200, { + "success": True, "status": "destroyed", "creditsBilled": 3, + }), + ("GET", "/v2/browser"): (200, {"success": True, "sessions": []}), + }) + try: + yield server + finally: + server.close() + + +@contextmanager +def trace_calls(config=None): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = FirecrawlInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=config) + try: + yield exporter, provider, instrumentor + finally: + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + provider.shutdown() + + +def invoke(api, is_async, calls): + """Use the public facade, constructed after instrumentation.""" + if not is_async: + client = Firecrawl(api_key=API_KEY, api_url=api.origin, max_retries=1) + return [getattr(client, name)(*args, **kwargs) for name, args, kwargs in calls] + + async def run(): + client = AsyncFirecrawl(api_key=API_KEY, api_url=api.origin, max_retries=1) + try: + results = [] + for name, args, kwargs in calls: + # wait_batch_scrape is public on the direct v2 client only. + target = client._v2_client if name == "wait_batch_scrape" else client + results.append(await getattr(target, name)(*args, **kwargs)) + return results + finally: + await client._v2_client.async_http_client.close() + + return asyncio.run(run()) + + +OPERATIONS = [ + ("extract", [URLS], {"prompt": PRIVATE, "poll_interval": 0}), + ("start_extract", [URLS], {"prompt": PRIVATE}), + ("get_extract_status", ["extract-1"], {}), + ("batch_scrape", [URLS], {"poll_interval": 0}), + ("start_batch_scrape", [URLS], {}), + ("get_batch_scrape_status", ["batch-1"], {}), + ("get_batch_scrape_status_page", [], {}), + ("get_batch_scrape_errors", ["batch-1"], {}), + ("cancel_batch_scrape", ["batch-1"], {}), + ("agent", [URLS], {"prompt": PRIVATE, "poll_interval": 0}), + ("start_agent", [URLS], {"prompt": PRIVATE}), + ("get_agent_status", ["agent-1"], {}), + ("cancel_agent", ["agent-1"], {}), + ("browser", [], {}), + ("browser_execute", ["browser-1", PRIVATE], {}), + ("delete_browser", ["browser-1"], {}), + ("list_browsers", [], {}), +] + + +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("operation", OPERATIONS, ids=[item[0] for item in OPERATIONS]) +def test_real_sdk_operation_emits_one_private_span(api, is_async, operation): + name, args, kwargs = operation + if name == "get_batch_scrape_status_page": + args = [api.origin + "/v2/batch/scrape/batch-1?cursor=" + PRIVATE] + with trace_calls() as (exporter, provider, _): + with provider.get_tracer(__name__).start_as_current_span("parent") as parent: + result = invoke(api, is_async, [(name, args, kwargs)])[0] + spans = [span for span in exporter.get_finished_spans() if span.name != "parent"] + assert api.calls, "The installed SDK must reach the loopback transport" + assert result is not None + assert len(spans) == 1 + span = spans[0] + assert span.name == "firecrawl." + name + assert span.parent.span_id == parent.get_span_context().span_id + assert span.attributes["gen_ai.span.kind"] == "TOOL" + assert span.status.status_code == StatusCode.OK + exported = span.to_json() + for secret in (PRIVATE, API_KEY, PAGE_BODY, "https://example.com/"): + assert secret not in exported + + +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("keyword_ids", [False, True], ids=["positional", "keyword"]) +@pytest.mark.parametrize("family", ["extract", "batch_scrape", "agent", "browser"]) +def test_lifecycle_ids_and_cancellation(api, is_async, keyword_ids, family): + if family == "browser": + calls = [("browser", [], {}), ("browser_execute", ["browser-1"], {"code": PRIVATE}), + ("delete_browser", ["browser-1"], {})] + expected_id, attribute, arg_name = "browser-1", "firecrawl.session_id", "session_id" + else: + expected_id = {"extract": "extract-1", "batch_scrape": "batch-1", "agent": "agent-1"}[family] + calls = [("start_" + family, [URLS], {} if family == "batch_scrape" else {"prompt": PRIVATE}), + ("get_" + family + "_status", [expected_id], {})] + if family != "extract": + calls.append(("cancel_" + family, [expected_id], {})) + attribute, arg_name = "firecrawl.job_id", "job_id" + if keyword_ids: + calls = [calls[0]] + [(name, [], {arg_name: args[0], **kwargs}) for name, args, kwargs in calls[1:]] + with trace_calls() as (exporter, _, _): + invoke(api, is_async, calls) + spans = exporter.get_finished_spans() + assert [span.attributes.get(attribute) for span in spans] == [expected_id] * len(calls) + if family in ("batch_scrape", "agent"): + assert spans[-1].attributes["firecrawl.cancelled"] is True + if family == "browser": + assert spans[-1].attributes["firecrawl.credits_used"] == 3 + else: + # The async extract SDK drops the API's camel-case creditsUsed field. + # Do not invent a zero or read raw transport payloads to recover it. + if family == "extract" and is_async: + assert "firecrawl.credits_used" not in spans[1].attributes + else: + assert spans[1].attributes["firecrawl.credits_used"] == 2 + assert spans[1].attributes["firecrawl.status"] == "completed" + + +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.parametrize("family", ["extract", "batch_scrape", "agent"]) +def test_failed_job_outcome_but_successful_status_poll(api, is_async, family): + route, job = {"extract": ("extract", "extract-1"), "agent": ("agent", "agent-1"), + "batch_scrape": ("batch/scrape", "batch-1")}[family] + status, payload = api.routes[("GET", f"/v2/{route}/{job}")] + api.routes[("GET", f"/v2/{route}/{job}")] = (status, {**payload, "status": "failed"}) + kwargs = {"poll_interval": 0} + if family != "batch_scrape": + kwargs["prompt"] = PRIVATE + with trace_calls() as (exporter, _, _): + results = invoke(api, is_async, [(family, [URLS], kwargs), (f"get_{family}_status", [job], {})]) + spans = exporter.get_finished_spans() + assert all(result.status == "failed" for result in results) + assert len(spans) == 2 + assert spans[0].status.status_code == StatusCode.ERROR + assert spans[0].status.description == "failed" + assert spans[1].status.status_code == StatusCode.OK + assert spans[1].attributes["firecrawl.status"] == "failed" + + +@pytest.mark.parametrize("is_async", [False, True]) +def test_browser_failure_is_safe_and_preserves_response(api, is_async): + api.routes[("POST", "/v2/browser/browser-1/execute")] = (200, { + "success": False, "error": PRIVATE, "stderr": PRIVATE, + }) + with trace_calls() as (exporter, _, _): + result = invoke(api, is_async, [("browser_execute", ["browser-1", PRIVATE], {})])[0] + (span,) = exporter.get_finished_spans() + assert result.error == PRIVATE + assert span.status.status_code == StatusCode.ERROR + assert span.status.description == "failed" + assert PRIVATE not in span.to_json() + + +def test_async_wait_batch_scrape_is_one_span(api): + with trace_calls() as (exporter, _, _): + invoke(api, True, [("wait_batch_scrape", ["batch-1"], {"poll_interval": 0})]) + (span,) = exporter.get_finished_spans() + assert span.name == "firecrawl.wait_batch_scrape" + assert span.attributes["firecrawl.job_id"] == "batch-1" + assert span.attributes["firecrawl.page_count"] == 2 + + +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.parametrize("method", ["start_agent", "start_batch_scrape", "browser_execute"]) +def test_api_error_keeps_safe_metadata(api, is_async, method): + from firecrawl.v2.utils.error_handler import UnauthorizedError + + path, args, kwargs = { + "start_agent": ("/v2/agent", [URLS], {"prompt": PRIVATE}), + "start_batch_scrape": ("/v2/batch/scrape", [URLS], {}), + "browser_execute": ("/v2/browser/browser-1/execute", ["browser-1", PRIVATE], {}), + }[method] + api.routes[("POST", path)] = (401, {"success": False, "error": PRIVATE + API_KEY}) + with trace_calls() as (exporter, _, _): + if is_async and method == "browser_execute": + # The SDK returns a failure model rather than raising for this API. + result = invoke(api, is_async, [(method, args, kwargs)])[0] + assert result.success is False and result.error == PRIVATE + API_KEY + else: + with pytest.raises(UnauthorizedError): + invoke(api, is_async, [(method, args, kwargs)]) + (span,) = exporter.get_finished_spans() + assert span.status.status_code == StatusCode.ERROR + if is_async and method == "browser_execute": + assert span.status.description == "failed" + assert "firecrawl.error.status_code" not in span.attributes + assert not span.events + else: + assert span.status.description == "UnauthorizedError" + assert span.attributes["firecrawl.error.status_code"] == 401 + assert len(span.events) == 1 + assert PRIVATE not in span.to_json() and API_KEY not in span.to_json() + + +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.parametrize("method,route,job", [ + ("extract", "extract", "extract-1"), ("batch_scrape", "batch/scrape", "batch-1"), +]) +def test_cancelled_job_is_not_a_success(api, is_async, method, route, job): + key = ("GET", f"/v2/{route}/{job}") + status, payload = api.routes[key] + api.routes[key] = status, {**payload, "status": "cancelled"} + kwargs = {"poll_interval": 0} + if method == "extract": + kwargs["prompt"] = PRIVATE + with trace_calls() as (exporter, _, _): + invoke(api, is_async, [(method, [URLS], kwargs)]) + (span,) = exporter.get_finished_spans() + assert span.status.description == "cancelled" + assert span.status.status_code == StatusCode.ERROR + assert span.attributes["firecrawl.cancelled"] is True + + +@pytest.mark.parametrize("is_async", [False, True]) +def test_browser_nonzero_exit_is_not_a_success(api, is_async): + api.routes[("POST", "/v2/browser/browser-1/execute")] = (200, { + "success": True, "exitCode": 7, "stderr": PRIVATE, + }) + with trace_calls() as (exporter, _, _): + invoke(api, is_async, [("browser_execute", ["browser-1", PRIVATE], {})]) + (span,) = exporter.get_finished_spans() + assert span.status.status_code == StatusCode.ERROR + assert span.status.description == "execution_failed" + assert PRIVATE not in span.to_json() + + +@pytest.mark.parametrize("is_async", [False, True]) +def test_suppression_and_facade_uninstrument(api, is_async): + from fi_instrumentation import suppress_tracing + + with trace_calls() as (exporter, _, instrumentor): + with suppress_tracing(): + invoke(api, is_async, OPERATIONS[:6]) + assert not exporter.get_finished_spans() + # A facade holds bound wrapper references even after class restoration. + async def run(): + client_cls = AsyncFirecrawl if is_async else Firecrawl + client = client_cls(api_key=API_KEY, api_url=api.origin, max_retries=1) + instrumentor.uninstrument() + try: + result = client.start_agent(URLS, prompt=PRIVATE) + if is_async: + result = await result + assert result.id == "agent-1" + finally: + if is_async: + await client._v2_client.async_http_client.close() + asyncio.run(run()) + assert not exporter.get_finished_spans() + + +def test_async_cancel_and_concurrent_calls_do_not_leak_active_guard(api, monkeypatch): + from firecrawl.v2.utils.http_client_async import AsyncHttpClient + + original_post = AsyncHttpClient.post + + async def journey(): + entered = asyncio.Event() + client = AsyncFirecrawl(api_key=API_KEY, api_url=api.origin, max_retries=1) + + async def post(transport, endpoint, *args, **kwargs): + if endpoint == "/v2/browser": + entered.set() + await asyncio.Event().wait() + return await original_post(transport, endpoint, *args, **kwargs) + + monkeypatch.setattr(AsyncHttpClient, "post", post) + try: + task = asyncio.create_task(client.browser()) + await asyncio.wait_for(entered.wait(), timeout=2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + await asyncio.gather( + client.agent(URLS, prompt=PRIVATE, poll_interval=0), + client.batch_scrape(URLS, poll_interval=0), + ) + finally: + await client._v2_client.async_http_client.close() + + with trace_calls() as (exporter, _, _): + asyncio.run(journey()) + spans = exporter.get_finished_spans() + assert sorted(span.name for span in spans) == ["firecrawl.agent", "firecrawl.batch_scrape", "firecrawl.browser"] + cancelled = next(span for span in spans if span.name == "firecrawl.browser") + assert cancelled.attributes["firecrawl.cancelled"] is True + assert cancelled.status.description == "cancelled" + + +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.parametrize("method", ["start_batch_scrape", "batch_scrape", "start_extract", "extract"]) +def test_formats_only_not_prompt_or_schema(api, is_async, method): + from firecrawl.v2.types import ScrapeOptions + from fi_instrumentation.instrumentation import TraceConfig + + formats = [{"type": "json", "prompt": PRIVATE, "schema": {"type": "object"}}] + payloads = [] + if "batch" in method: + api.routes[("POST", "/v2/batch/scrape")] = (200, lambda body: ( + payloads.append(body) or {"success": True, "id": "batch-1", "url": URLS[0]} + )) + # Sync exposes convenience formats; async forwards only options to _prepare. + kwargs = {"options": ScrapeOptions(formats=formats), "formats": ["html"]} if is_async else {"formats": formats} + else: + kwargs = {"prompt": PRIVATE, "scrape_options": ScrapeOptions(formats=formats)} + with trace_calls(TraceConfig(hide_inputs=True, hide_outputs=True)) as (exporter, _, _): + invoke(api, is_async, [(method, [URLS], kwargs)]) + (span,) = exporter.get_finished_spans() + assert span.attributes["firecrawl.formats"] == ("json",) + if payloads: + assert [item["type"] if isinstance(item, dict) else item for item in payloads[0]["formats"]] == ["json"] + assert "input.value" not in span.attributes and "output.value" not in span.attributes + assert PRIVATE not in span.to_json() + + +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +def test_killed_browser_execution_is_failure_even_without_exit_code(api, is_async): + api.routes[("POST", "/v2/browser/browser-1/execute")] = (200, { + "success": True, "killed": True, "exitCode": None, "output": PRIVATE, + }) + with trace_calls() as (exporter, _, _): + result = invoke(api, is_async, [("browser_execute", ["browser-1", PRIVATE], {})])[0] + (span,) = exporter.get_finished_spans() + assert result.success is True and result.killed is True and result.exit_code is None + assert span.status.status_code == StatusCode.ERROR + assert span.status.description == "execution_failed" + assert PRIVATE not in span.to_json() + + +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("method,job", [("extract", "extract-1"), ("agent", "agent-1")]) +def test_blocking_job_exhausted_wait_is_not_success(api, is_async, method, job): + api.routes[("GET", f"/v2/{method}/{job}")] = (200, { + "success": True, "id": job, "status": "processing", "data": {"secret": PRIVATE}, + }) + with trace_calls() as (exporter, _, _): + result, poll = invoke(api, is_async, [ + (method, [URLS], {"prompt": PRIVATE, "timeout": 0}), + (f"get_{method}_status", [job], {}), + ]) + span, poll_span = exporter.get_finished_spans() + assert result.status == poll.status == "processing" + assert span.attributes["firecrawl.status"] == "processing" + assert span.status.status_code == StatusCode.ERROR + assert span.status.description == "processing" + assert poll_span.status.status_code == StatusCode.OK + assert PRIVATE not in span.to_json() and PRIVATE not in poll_span.to_json() + + +@pytest.mark.parametrize("method", ["start_batch_scrape", "batch_scrape"]) +def test_async_batch_ignores_top_level_formats_without_options(api, method): + payloads = [] + api.routes[("POST", "/v2/batch/scrape")] = (200, lambda body: ( + payloads.append(body) or {"success": True, "id": "batch-1", "url": URLS[0]} + )) + with trace_calls() as (exporter, _, _): + invoke(api, True, [(method, [URLS], {"formats": ["json"], "poll_interval": 0})]) + (span,) = exporter.get_finished_spans() + assert "formats" not in payloads[0] + assert "firecrawl.formats" not in span.attributes diff --git a/python/frameworks/firecrawl/tests/test_extended_wire.py b/python/frameworks/firecrawl/tests/test_extended_wire.py new file mode 100644 index 000000000..01f865e1b --- /dev/null +++ b/python/frameworks/firecrawl/tests/test_extended_wire.py @@ -0,0 +1,178 @@ +"""OTLP protobuf export and custom JSON transport, plain and gzip, over loopback.""" + +from __future__ import annotations + +import base64 +import gzip +import json +import re +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from threading import Thread +from urllib.request import Request, urlopen + +import pytest +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from fi_instrumentation.otel import HTTPSpanExporter +from google.protobuf.json_format import MessageToDict, ParseDict +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.exporter.otlp.proto.http import Compression +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult + +from test_extended_operations import API_KEY, PRIVATE, OPERATIONS, invoke +from test_extended_operations import api as api # Re-export the loopback fixture. +from _firecrawl_fake import PAGE_BODY +from traceai_firecrawl import FirecrawlInstrumentor + + +def _ids(value, decode=False): + # OTLP JSON IDs are hex, unlike protobuf's default base64 JSON mapping. + if isinstance(value, dict): + return {key: ( + (base64.b64encode(bytes.fromhex(item)).decode() if decode else base64.b64decode(item).hex()) + if key in {"traceId", "spanId", "parentSpanId"} else _ids(item, decode) + ) for key, item in value.items()} + if isinstance(value, list): + return [_ids(item, decode) for item in value] + return value + + +class JsonExporter(SpanExporter): + """Test-only transport; the package continues to use the standard exporter.""" + + def __init__(self, endpoint, compressed): + self.endpoint, self.compressed = endpoint, compressed + + def export(self, spans): + body = json.dumps(_ids(MessageToDict(encode_spans(spans), use_integers_for_enums=True))).encode() + headers = {"Content-Type": "application/json"} + if self.compressed: + body = gzip.compress(body) + headers["Content-Encoding"] = "gzip" + with urlopen(Request(self.endpoint, data=body, headers=headers), timeout=2) as response: + assert response.status == 200 + return SpanExportResult.SUCCESS + + def shutdown(self): + pass + + +@pytest.fixture() +def receiver(): + requests = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): # noqa: N802 + requests.append((self.path, self.headers, self.rfile.read(int(self.headers["Content-Length"])))) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = Thread(target=server.serve_forever, args=(0.05,), daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", requests + finally: + server.shutdown() + server.server_close() + thread.join() + + +def decode_exports(requests, encoding, compressed): + spans = [] + for path, headers, body in requests: + assert path == "/tracer/v1/traces" + assert (headers.get("Content-Encoding") == "gzip") is compressed + if compressed: + body = gzip.decompress(body) + request = ExportTraceServiceRequest() + if encoding == "json": + assert headers["Content-Type"] == "application/json" + ParseDict(_ids(json.loads(body), decode=True), request) + else: + assert headers["Content-Type"] == "application/x-protobuf" + request.ParseFromString(body) + for secret in (PRIVATE, API_KEY, PAGE_BODY): + assert secret not in str(request) + for resource in request.resource_spans: + attributes = {a.key: a.value.string_value for a in resource.resource.attributes} + assert attributes["project_name"] == "firecrawl" + assert attributes["project_type"] == "observe" + for scope in resource.scope_spans: + spans.extend(scope.spans) + return spans + + +@pytest.mark.parametrize("encoding", ["json", "protobuf"]) +@pytest.mark.parametrize("compressed", [False, True]) +@pytest.mark.parametrize("is_async", [False, True]) +def test_extended_operations_over_otlp(api, receiver, encoding, compressed, is_async, monkeypatch): + origin, requests = receiver + monkeypatch.setenv("FI_API_KEY", "synthetic-fi-key") + monkeypatch.setenv("FI_SECRET_KEY", "synthetic-fi-secret") + monkeypatch.setenv("FI_BASE_URL", origin) + endpoint = origin + "/tracer/v1/traces" + exporter = JsonExporter(endpoint, compressed) if encoding == "json" else HTTPSpanExporter( + endpoint=endpoint, headers={"X-Api-Key": "synthetic-fi-key", "X-Secret-Key": "synthetic-fi-secret"}, + compression=Compression.Gzip if compressed else Compression.NoCompression, + ) + provider = register(project_name="firecrawl", project_type=ProjectType.OBSERVE, + span_exporter=exporter, batch=False, verbose=False, set_global_tracer_provider=False) + instrumentor = FirecrawlInstrumentor() + instrumentor.instrument(tracer_provider=provider) + operations = [(name, [api.origin + "/v2/batch/scrape/batch-1?cursor=" + PRIVATE] if name == "get_batch_scrape_status_page" else args, kwargs) + for name, args, kwargs in OPERATIONS] + try: + with provider.get_tracer(__name__).start_as_current_span("parent"): + invoke(api, is_async, operations) + assert provider.force_flush() + finally: + instrumentor.uninstrument() + provider.shutdown() + spans = decode_exports(requests, encoding, compressed) + assert len(spans) == len(operations) + 1 + parent = next(span for span in spans if span.name == "parent") + assert {span.name for span in spans} == {"parent"} | {"firecrawl." + name for name, _, _ in operations} + for span in spans: + if span is parent: + continue + assert span.parent_span_id == parent.span_id + assert span.trace_id == parent.trace_id + assert span.status.code == 1 # OK + attributes = {a.key: a.value for a in span.attributes} + assert attributes["gen_ai.span.kind"].string_value == "TOOL" + + +def test_readme_python_snippet_uses_real_register_and_sdk(api, receiver, monkeypatch): + import firecrawl + from functools import partial + + origin, requests = receiver + monkeypatch.setenv("FI_BASE_URL", origin) + monkeypatch.setenv("FI_API_KEY", "synthetic-fi-key") + monkeypatch.setenv("FI_SECRET_KEY", "synthetic-fi-secret") + monkeypatch.setenv("FIRECRAWL_API_KEY", API_KEY) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_COMPRESSION", "gzip") + # Only the vendor base URL is substituted; the snippet and SDK run unchanged. + monkeypatch.setattr(firecrawl, "Firecrawl", partial(firecrawl.Firecrawl, api_url=api.origin)) + readme = (Path(__file__).resolve().parents[1] / "README.md").read_text() + snippet = re.search(r"```python\n(.*?)```", readme, re.S).group(1) + namespace = {} + try: + exec(compile(snippet, "README.md", "exec"), namespace) + assert namespace["tracer_provider"].force_flush() + finally: + FirecrawlInstrumentor().uninstrument() + if "tracer_provider" in namespace: + namespace["tracer_provider"].shutdown() + spans = decode_exports(requests, "protobuf", True) + assert [span.name for span in spans] == ["firecrawl.scrape", "firecrawl.batch_scrape"] + assert api.calls == [("POST", "/v2/scrape"), ("POST", "/v2/batch/scrape"), ("GET", "/v2/batch/scrape/batch-1")] + assert all(headers["X-Api-Key"] == "synthetic-fi-key" and headers["X-Secret-Key"] == "synthetic-fi-secret" + for _, headers, _ in requests) diff --git a/python/frameworks/firecrawl/tests/test_review_regressions.py b/python/frameworks/firecrawl/tests/test_review_regressions.py index 0a5af4eca..f4bf305c7 100644 --- a/python/frameworks/firecrawl/tests/test_review_regressions.py +++ b/python/frameworks/firecrawl/tests/test_review_regressions.py @@ -209,10 +209,15 @@ def response(body): assert HTML not in wire -def test_f5_readme_usage_registers_observe(fake, telemetry): +def test_f5_readme_usage_registers_observe(fake, telemetry, monkeypatch): import ast from pathlib import Path + from _firecrawl_fake import crawl_status + monkeypatch.setenv("FIRECRAWL_API_KEY", KEY) + fake.routes[("POST", "/v2/batch/scrape")] = ( + 200, {"success": True, "id": "batch-1", "url": "https://example.com"}) + fake.routes[("GET", "/v2/batch/scrape/batch-1")] = (200, crawl_status("completed")) readme = Path(__file__).resolve().parents[1] / "README.md" snippet = readme.read_text().split("```python\n", 1)[1].split("```", 1)[0] tree = ast.parse(snippet) @@ -233,9 +238,11 @@ def test_f5_readme_usage_registers_observe(fake, telemetry): telemetry[2].uninstrument() if "tracer_provider" in namespace: namespace["tracer_provider"].shutdown() - assert fake.calls == [("POST", "/v2/scrape")] + assert fake.calls == [ + ("POST", "/v2/scrape"), ("POST", "/v2/batch/scrape"), ("GET", "/v2/batch/scrape/batch-1")] exports = telemetry[0].requests() - assert exports and len(telemetry[0].spans()) == 1 + assert exports and len(telemetry[0].spans()) == 2 + assert [span["name"] for span in telemetry[0].spans()] == ["firecrawl.scrape", "firecrawl.batch_scrape"] for export in exports: for resource in export["resource_attributes"]: assert resource["project_name"] == "firecrawl" diff --git a/python/frameworks/firecrawl/traceai_firecrawl/__init__.py b/python/frameworks/firecrawl/traceai_firecrawl/__init__.py index 9b1ca9b1f..389ff8edf 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/__init__.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/__init__.py @@ -34,11 +34,30 @@ "start_crawl": "firecrawl.start_crawl", "get_crawl_status": "firecrawl.get_crawl_status", "cancel_crawl": "firecrawl.cancel_crawl", + "extract": "firecrawl.extract", + "start_extract": "firecrawl.start_extract", + "get_extract_status": "firecrawl.get_extract_status", + "batch_scrape": "firecrawl.batch_scrape", + "start_batch_scrape": "firecrawl.start_batch_scrape", + # Only exposed by AsyncFirecrawlClient; _wrap skips absent methods. + "wait_batch_scrape": "firecrawl.wait_batch_scrape", + "get_batch_scrape_status": "firecrawl.get_batch_scrape_status", + "get_batch_scrape_status_page": "firecrawl.get_batch_scrape_status_page", + "get_batch_scrape_errors": "firecrawl.get_batch_scrape_errors", + "cancel_batch_scrape": "firecrawl.cancel_batch_scrape", + "agent": "firecrawl.agent", + "start_agent": "firecrawl.start_agent", + "get_agent_status": "firecrawl.get_agent_status", + "cancel_agent": "firecrawl.cancel_agent", + "browser": "firecrawl.browser", + "browser_execute": "firecrawl.browser_execute", + "delete_browser": "firecrawl.delete_browser", + "list_browsers": "firecrawl.list_browsers", } class FirecrawlInstrumentor(BaseInstrumentor): - """Instrument the Firecrawl v2 client's scrape, search, map, and crawl APIs.""" + """Instrument Firecrawl v2 scraping, extraction, agent and browser operations.""" __slots__ = ("_original_methods",) diff --git a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py index 90d5beb64..4e4f6b75e 100644 --- a/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py +++ b/python/frameworks/firecrawl/traceai_firecrawl/_wrappers.py @@ -33,18 +33,41 @@ _ERROR_CODE = "firecrawl.error.code" _MAX_QUERY_LENGTH = 1024 -# One span per call. crawl polls internally, so it must not emit one span per page. +# One span per user operation, not per internal poll or page. _CRAWL_METHODS = {"crawl", "start_crawl", "get_crawl_status", "cancel_crawl"} -_JOB_ID_FIRST_ARG_METHODS = {"get_crawl_status", "cancel_crawl"} -# Methods that return a CrawlJob. The SDK returns failed and cancelled jobs -# without raising (methods/crawl.py wait_for_crawl_completion), so the span -# status comes from the job. -_JOB_STATUS_METHODS = {"crawl", "get_crawl_status"} -# Of those, the methods whose span status follows the job (crawl waits for it). -_JOB_OUTCOME_METHODS = {"crawl"} +_EXTRACT_METHODS = {"extract", "start_extract", "get_extract_status"} +_BATCH_METHODS = { + "batch_scrape", "start_batch_scrape", "wait_batch_scrape", + "get_batch_scrape_status", "get_batch_scrape_status_page", + "get_batch_scrape_errors", "cancel_batch_scrape", +} +_AGENT_METHODS = {"agent", "start_agent", "get_agent_status", "cancel_agent"} +_BROWSER_METHODS = {"browser", "browser_execute", "delete_browser", "list_browsers"} +_JOB_METHODS = _CRAWL_METHODS | _EXTRACT_METHODS | _BATCH_METHODS | _AGENT_METHODS +_JOB_ID_FIRST_ARG_METHODS = { + "get_crawl_status", "cancel_crawl", "get_extract_status", "wait_batch_scrape", + "get_batch_scrape_status", "get_batch_scrape_errors", "cancel_batch_scrape", + "get_agent_status", "cancel_agent", +} +_CANCEL_METHODS = {"cancel_crawl", "cancel_batch_scrape", "cancel_agent"} +# Blocking operations return failed/cancelled jobs without necessarily raising. +# A successful status poll reports the job state but is not itself a failure. +_JOB_OUTCOME_METHODS = {"crawl", "extract", "batch_scrape", "wait_batch_scrape", "agent"} +_JOB_STATUS_METHODS = _JOB_OUTCOME_METHODS | { + "get_crawl_status", "start_extract", "get_extract_status", + "get_batch_scrape_status", "get_batch_scrape_status_page", "start_agent", "get_agent_status", +} _ERROR_JOB_STATUSES = {"failed", "cancelled"} +_SAFE_JOB_STATUSES = _ERROR_JOB_STATUSES | {"scraping", "processing", "completed"} +# These responses can reject an operation without raising an SDK exception. +_SUCCESS_METHODS = _EXTRACT_METHODS | _AGENT_METHODS | _BROWSER_METHODS +_PAGE_COUNT_METHODS = {"crawl", "batch_scrape", "wait_batch_scrape"} +_SESSION_ID = "firecrawl.session_id" # Methods whose arguments carry the requested output formats. -_FORMAT_METHODS = {"scrape", "search", "crawl", "start_crawl"} +_FORMAT_METHODS = { + "scrape", "search", "crawl", "start_crawl", "batch_scrape", "start_batch_scrape", + "extract", "start_extract", +} # SDK 4.46.2 FormatString/model Literals and validation aliases. The finite # vocabulary bounds both label length and the deduplicated attribute list. _FORMAT_NAMES = {name: name for name in ( @@ -190,15 +213,17 @@ def _call_job_id(method_name: str, args: Sequence[Any], kwargs: Mapping[str, Any return _job_id(None, kwargs) -def _formats(kwargs: Mapping[str, Any], method_name: str) -> list[str]: +def _formats(kwargs: Mapping[str, Any], method_name: str, async_batch: bool = False) -> list[str]: """Names of the requested formats. A format object's prompt or schema is content, so only its ``type`` is kept.""" from firecrawl.v2.types import ScrapeFormats - scrape_options = kwargs.get("scrape_options") + # Async batch accepts options=ScrapeOptions and ignores top-level formats; + # the sync batch facade instead exposes formats directly. + scrape_options = kwargs.get("options" if async_batch else "scrape_options") # The SDK ignores convenience formats when scrape_options is supplied. # Search has no supported top-level formats argument. - if scrape_options is not None or method_name == "search": + if scrape_options is not None or method_name == "search" or async_batch: if isinstance(scrape_options, Mapping): formats = scrape_options.get("formats") else: @@ -291,13 +316,23 @@ def _attributes( if host: attributes["server.address"] = host if self._method_name in _FORMAT_METHODS: - formats = _formats(kwargs, self._method_name) + formats = _formats( + kwargs, self._method_name, + async_batch=isinstance(self, AsyncOperationWrapper) and self._method_name in _BATCH_METHODS, + ) if formats: attributes[_FORMATS] = formats if self._method_name in _CRAWL_METHODS: limit = kwargs.get("limit") if isinstance(limit, int): attributes[_LIMIT] = limit + if self._method_name in ("browser_execute", "delete_browser"): + session_id = kwargs.get("session_id", args[0] if args else None) + if isinstance(session_id, str) and session_id: + safe_id = _redact(session_id, instance) + if safe_id is not None: + attributes[_SESSION_ID] = safe_id + if self._method_name in _JOB_METHODS: job_id = _call_job_id(self._method_name, args, kwargs) if job_id: safe_id = _redact(job_id, instance) @@ -326,31 +361,43 @@ def _record_result( count = _document_count(result) if count is not None: span.set_attribute(_RESULT_COUNT, count) - if self._method_name in _CRAWL_METHODS: + if self._method_name in _JOB_METHODS or self._method_name == "browser": job_id = _job_id(result, kwargs) if job_id: safe_id = _redact(job_id, instance) if safe_id is not None: - span.set_attribute(_JOB_ID, safe_id) - if self._method_name == "crawl": - span.set_attribute(_PAGE_COUNT, _page_count(result)) - if self._method_name == "cancel_crawl" and isinstance(result, bool): + attribute = _SESSION_ID if self._method_name == "browser" else _JOB_ID + span.set_attribute(attribute, safe_id) + if self._method_name in _PAGE_COUNT_METHODS: + span.set_attribute(_PAGE_COUNT, _page_count(result)) + if self._method_name in _CANCEL_METHODS and isinstance(result, bool): span.set_attribute(_CANCELLED, result) - credits = getattr(result, "credits_used", None) - if isinstance(credits, int): + credits_attribute = "credits_billed" if self._method_name == "delete_browser" else "credits_used" + credits = getattr(result, credits_attribute, None) + if isinstance(credits, int) and not isinstance(credits, bool): span.set_attribute(_CREDITS_USED, credits) job_status = ( getattr(result, "status", None) if self._method_name in _JOB_STATUS_METHODS else None ) - if isinstance(job_status, str) and job_status: + # A bounded vocabulary prevents arbitrary response text entering metadata. + if not isinstance(job_status, str) or job_status not in _SAFE_JOB_STATUSES: + job_status = None + if job_status is not None: span.set_attribute(_STATUS, job_status) - # Only the blocking crawl() call's outcome is the job's outcome. A - # get_crawl_status poll that got an answer succeeded; it records the state. job_failed = self._method_name in _JOB_OUTCOME_METHODS and job_status in _ERROR_JOB_STATUSES + # Unlike status polls, blocking extract/agent return processing only + # when their SDK wait expires; the operation did not complete. + incomplete = self._method_name in {"extract", "agent"} and job_status in {"processing", "scraping"} if job_failed and job_status == "cancelled": span.set_attribute(_CANCELLED, True) - if job_failed: - span.set_status(Status(StatusCode.ERROR, job_status)) + rejected = self._method_name in _SUCCESS_METHODS and getattr(result, "success", None) is False + exit_code = getattr(result, "exit_code", None) if self._method_name == "browser_execute" else None + execution_failed = ( + self._method_name == "browser_execute" and getattr(result, "killed", None) is True + ) or (isinstance(exit_code, int) and not isinstance(exit_code, bool) and exit_code != 0) + if job_failed or incomplete or rejected or execution_failed: + reason = job_status if job_failed or incomplete else ("failed" if rejected else "execution_failed") + span.set_status(Status(StatusCode.ERROR, reason)) else: span.set_status(Status(StatusCode.OK))